Changeset 70977 in vbox for trunk/src/VBox/VMM/VMMR3

Timestamp:

Feb 12, 2018 8:45:31 PM (7 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

120824

Message:

NEM: Working on PGM notifications. bugref:9044

Location:

trunk/src/VBox/VMM/VMMR3

Files:

• NEMR3.cpp (modified) (3 diffs)
• NEMR3Native-win.cpp (modified) (22 diffs)
• PGMHandler.cpp (modified) (2 diffs)
• PGMPhys.cpp (modified) (21 diffs)
• PGMSavedState.cpp (modified) (2 diffs)

trunk/src/VBox/VMM/VMMR3/NEMR3.cpp

@@ -249 +249 @@
 
 
-VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
-{
-    int rc = VINF_SUCCESS;
-#ifdef VBOX_WITH_NATIVE_NEM
-    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
-        rc = nemR3NativeNotifyPhysMmioExMap(pVM, GCPhys, cb, fFlags);
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysMmioExMap(pVM, GCPhys, cb, fFlags, pvMmio2);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
+#endif
+    return rc;
+}
+
+
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysMmioExUnmap(pVM, GCPhys, cb, fFlags);
 #else
     NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
@@ -262 +275 @@
 
 
-VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
-{
-    int rc = VINF_SUCCESS;
-#ifdef VBOX_WITH_NATIVE_NEM
-    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
-        rc = nemR3NativeNotifyPhysMmioExUnmap(pVM, GCPhys, cb, fFlags);
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fFlags);
 #else
     NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
@@ -275 +288 @@
 
 
-VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTUINT cb, uint32_t fFlags)
-{
-    int rc = VINF_SUCCESS;
-#ifdef VBOX_WITH_NATIVE_NEM
-    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
-        rc = nemR3NativeNotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fFlags);
-#else
-    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
-#endif
-    return rc;
-}
-
-
-VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTUINT cb, uint32_t fFlags)
+/**
+ * Called after the ROM range has been fully completed.
+ *
+ * This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
+ * number of NEMHCNotifyPhysPageProtChanged calls.
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
 {
     int rc = VINF_SUCCESS;

trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp

@@ -21 +21 @@
 *********************************************************************************************************************************/
 #define LOG_GROUP LOG_GROUP_NEM
+#include <iprt/nt/nt-and-windows.h>
 #include <WinHvPlatform.h>
 
@@ -42 +43 @@
 #include <iprt/path.h>
 #include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef LOG_ENABLED
+# define NEM_WIN_INTERCEPT_NT_IO_CTLS
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** @name Our two-bit physical page state for PGMPAGE
+ * @{ */
+#define NEM_WIN_PAGE_STATE_NOT_SET      0
+#define NEM_WIN_PAGE_STATE_UNMAPPED     1
+#define NEM_WIN_PAGE_STATE_READABLE     2
+#define NEM_WIN_PAGE_STATE_WRITABLE     3
+/** @} */
 
 
@@ -123 +144 @@
 
 
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+
+/** The real NtDeviceIoControlFile API in NTDLL.   */
+static decltype(NtDeviceIoControlFile) *g_pfnNtDeviceIoControlFile;
+
+/**
+ * Wrapper that logs the call from VID.DLL.
+ *
+ * This is very handy for figuring out why an API call fails.
+ */
+static NTSTATUS WINAPI
+nemR3WinLogWrapper_NtDeviceIoControlFile(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                         PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                         PVOID pvOutput, ULONG cbOutput)
+{
+    NTSTATUS rcNt = g_pfnNtDeviceIoControlFile(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, uFunction,
+                                               pvInput, cbInput, pvOutput, cbOutput);
+    if (!hEvt && !pfnApcCallback && !pvApcCtx)
+        LogRel(("VID!NtDeviceIoControlFile: hFile=%#zx pIos=%p->{s:%#x, i:%#zx} uFunction=%#x Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
+                hFile, pIos, pIos->Status, pIos->Information, uFunction, pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
+    else
+        LogRel(("VID!NtDeviceIoControlFile: hFile=%#zx hEvt=%#zx Apc=%p/%p pIos=%p->{s:%#x, i:%#zx} uFunction=%#x Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
+                hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pIos->Status, pIos->Information, uFunction,
+                pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
+    return rcNt;
+}
+
+
+/**
+ * Patches the call table of VID.DLL so we can intercept and log
+ * NtDeviceIoControlFile.
+ *
+ * This is for DEBUGGING only.
+ *
+ * @param   hLdrModVid      The VID module handle.
+ */
+static void nemR3WinInitVidIntercepts(RTLDRMOD hLdrModVid)
+{
+    /*
+     * Locate the real API.
+     */
+    g_pfnNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) *)RTLdrGetSystemSymbol("NTDLL.DLL", "NtDeviceIoControlFile");
+    AssertReturnVoid(g_pfnNtDeviceIoControlFile > 0);
+
+    /*
+     * Locate the PE header and get what we need from it.
+     */
+    uint8_t const *pbImage = (uint8_t const *)RTLdrGetNativeHandle(hLdrModVid);
+    IMAGE_DOS_HEADER const *pMzHdr  = (IMAGE_DOS_HEADER const *)pbImage;
+    AssertReturnVoid(pMzHdr->e_magic == IMAGE_DOS_SIGNATURE);
+    IMAGE_NT_HEADERS const *pNtHdrs = (IMAGE_NT_HEADERS const *)&pbImage[pMzHdr->e_lfanew];
+    AssertReturnVoid(pNtHdrs->Signature == IMAGE_NT_SIGNATURE);
+
+    uint32_t const             cbImage   = pNtHdrs->OptionalHeader.SizeOfImage;
+    IMAGE_DATA_DIRECTORY const ImportDir = pNtHdrs->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
+
+    /*
+     * Walk the import descriptor table looking for NTDLL.DLL.
+     */
+    bool fSuccess = true;
+    AssertReturnVoid(ImportDir.Size > 0);
+    AssertReturnVoid(ImportDir.Size < cbImage);
+    AssertReturnVoid(ImportDir.VirtualAddress > 0);
+    AssertReturnVoid(ImportDir.VirtualAddress < cbImage);
+    for (PIMAGE_IMPORT_DESCRIPTOR pImps = (PIMAGE_IMPORT_DESCRIPTOR)&pbImage[ImportDir.VirtualAddress];
+         pImps->Name != 0 && pImps->FirstThunk != 0;
+         pImps++)
+    {
+        AssertReturnVoid(pImps->Name < cbImage);
+        const char *pszModName = (const char *)&pbImage[pImps->Name];
+        if (RTStrICmpAscii(pszModName, "ntdll.dll"))
+            continue;
+        AssertReturnVoid(pImps->FirstThunk < cbImage);
+        AssertReturnVoid(pImps->OriginalFirstThunk < cbImage);
+
+        /*
+         * Walk the thunks table(s) looking for NtDeviceIoControlFile.
+         */
+        PIMAGE_THUNK_DATA pFirstThunk = (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]; /* update this. */
+        PIMAGE_THUNK_DATA pThunk      = pImps->OriginalFirstThunk == 0                  /* read from this. */
+                                      ? (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]
+                                      : (PIMAGE_THUNK_DATA)&pbImage[pImps->OriginalFirstThunk];
+        while (pThunk->u1.Ordinal != 0)
+        {
+            if (!(pThunk->u1.Ordinal & IMAGE_ORDINAL_FLAG32))
+            {
+                AssertReturnVoid(pThunk->u1.Ordinal > 0 && pThunk->u1.Ordinal < cbImage);
+
+                const char *pszSymbol = (const char *)&pbImage[(uintptr_t)pThunk->u1.AddressOfData + 2];
+                if (strcmp(pszSymbol, "NtDeviceIoControlFile") == 0)
+                {
+                    DWORD fOldProt = PAGE_EXECUTE;
+                    VirtualProtect(&pFirstThunk->u1.Function, sizeof(uintptr_t), PAGE_EXECUTE_READWRITE, &fOldProt);
+                    pFirstThunk->u1.Function = (uintptr_t)nemR3WinLogWrapper_NtDeviceIoControlFile;
+                    VirtualProtect(&pFirstThunk->u1.Function, sizeof(uintptr_t), fOldProt, &fOldProt);
+                    fSuccess = true;
+                }
+            }
+
+            pThunk++;
+            pFirstThunk++;
+        }
+    }
+    Assert(fSuccess);
+}
+#endif
+
+
 
 /**
@@ -200 +329 @@
     if (RT_SUCCESS(rc))
     {
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+        nemR3WinInitVidIntercepts(ahMods[1]);
+#endif
         for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
         {
@@ -267 +399 @@
     DWORD   rcWin = GetLastError();
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     if (!Caps.HypervisorPresent)
     {
@@ -284 +418 @@
     hrc = WHvGetCapability(WHvCapabilityCodeExtendedVmExits, &Caps, sizeof(Caps));
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     NEM_LOG_REL_CAP_EX("WHvCapabilityCodeExtendedVmExits", "%'#018RX64", Caps.ExtendedVmExits.AsUINT64);
     pVM->nem.s.fExtendedMsrExit   = RT_BOOL(Caps.ExtendedVmExits.X64MsrExit);
@@ -302 +438 @@
     hrc = WHvGetCapability(WHvCapabilityCodeFeatures, &Caps, sizeof(Caps));
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     if (Caps.Features.AsUINT64 & ~(uint64_t)0)
         LogRel(("NEM: Warning! Unknown feature definitions: %#RX64\n", Caps.Features.AsUINT64));
@@ -313 +451 @@
     hrc = WHvGetCapability(WHvCapabilityCodeProcessorVendor, &Caps, sizeof(Caps));
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     switch (Caps.ProcessorVendor)
     {
@@ -336 +476 @@
     hrc = WHvGetCapability(WHvCapabilityCodeProcessorFeatures, &Caps, sizeof(Caps));
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorFeatures", "%'#018RX64", Caps.ProcessorFeatures.AsUINT64);
 #define NEM_LOG_REL_CPU_FEATURE(a_Field)    NEM_LOG_REL_CAP_SUB(#a_Field, Caps.ProcessorFeatures.a_Field)
@@ -393 +535 @@
     hrc = WHvGetCapability(WHvCapabilityCodeProcessorClFlushSize, &Caps, sizeof(Caps));
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorClFlushSize", "2^%u", Caps.ProcessorClFlushSize);
     if (Caps.ProcessorClFlushSize < 8 && Caps.ProcessorClFlushSize > 9)
@@ -450 +594 @@
     HRESULT hrc = WHvCreatePartition(&hPartition);
     if (FAILED(hrc))
-        return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc", hrc);
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
 
     int rc;
@@ -495 +640 @@
     else
         rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
-                           "Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc", pVM->cCpus, hrc);
+                           "Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc (Last=%#x/%u)",
+                           pVM->cCpus, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
     WHvDeletePartition(hPartition);
 
@@ -590 +736 @@
     if (FAILED(hrc))
         return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
-                          "Failed to set WHvPartitionPropertyCodeProcessorVendor to %u: %Rhrc", Property.ProcessorVendor, hrc);
+                          "Failed to set WHvPartitionPropertyCodeProcessorVendor to %u: %Rhrc (Last=%#x/%u)",
+                          Property.ProcessorVendor, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
 
     /* Not sure if we really need to set the cache line flush size. */
@@ -599 +746 @@
     if (FAILED(hrc))
         return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
-                          "Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc",
-                          pVM->nem.s.cCacheLineFlushShift, hrc);
+                          "Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc (Last=%#x/%u)",
+                          pVM->nem.s.cCacheLineFlushShift, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
 
     /*
@@ -614 +761 @@
     if (FAILED(hrc))
         return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
-                          "Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc",
-                          pVM->nem.s.uCpuFeatures.u64, hrc);
+                          "Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc (Last=%#x/%u)",
+                          pVM->nem.s.uCpuFeatures.u64, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
 
     /*
@@ -625 +772 @@
     hrc = WHvSetupPartition(hPartition);
     if (FAILED(hrc))
-        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to WHvSetupPartition failed: %Rhrc", hrc);
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)",
+                          hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue);
 
     /* Get the handle. */
@@ -655 +804 @@
         if (FAILED(hrc))
         {
+            NTSTATUS const rcNtLast  = RTNtCurrentTeb()->LastStatusValue;
+            DWORD const    dwErrLast = RTNtCurrentTeb()->LastErrorValue;
             while (iCpu-- > 0)
             {
                 HRESULT hrc2 = WHvDeleteVirtualProcessor(hPartition, iCpu);
-                AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc\n", hPartition, iCpu, hrc2));
+                AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
+                                                  hPartition, iCpu, hrc2, RTNtCurrentTeb()->LastStatusValue,
+                                                  RTNtCurrentTeb()->LastErrorValue));
             }
-            return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to WHvSetupPartition failed: %Rhrc", hrc);
+            return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                              "Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
         }
     }
@@ -692 +846 @@
         {
             HRESULT hrc = WHvDeleteVirtualProcessor(hPartition, iCpu);
-            AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc\n", hPartition, iCpu, hrc));
+            AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
+                                             hPartition, iCpu, hrc, RTNtCurrentTeb()->LastStatusValue,
+                                             RTNtCurrentTeb()->LastErrorValue));
         }
         WHvDeletePartition(hPartition);
@@ -713 +869 @@
 
 
+
+DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
+{
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
+    if (RT_SUCCESS(rc))
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+    return rc;
+}
+
+DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
+{
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
+    if (RT_SUCCESS(rc))
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+    return rc;
+}
+
+
 int nemR3NativeNotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
 {
+    LogRel(("nemR3NativeNotifyPhysRamRegister: %RGp LB %RGp\n", GCPhys, cb));
     NOREF(pVM); NOREF(GCPhys); NOREF(cb);
     return VINF_SUCCESS;
@@ -720 +897 @@
 
 
-int nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
-{
+int nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
+{
+    LogRel(("nemR3NativeNotifyPhysMmioExMap: %RGp LB %RGp fFlags=%#x pvMmio2=%p\n", GCPhys, cb, fFlags, pvMmio2));
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    LogRel(("nemR3NativeNotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
     NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
     return VINF_SUCCESS;
@@ -727 +913 @@
 
 
-int nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
-{
+/**
+ * Called early during ROM registration, right after the pages have been
+ * allocated and the RAM range updated.
+ *
+ * This will be succeeded by a number of NEMHCNotifyPhysPageProtChanged() calls
+ * and finally a NEMR3NotifyPhysRomRegisterEarly().
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+int nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
+#if 0 /* Let's not do this after all.  We'll protection change notifications for each page and if not we'll map them lazily. */
+    RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
+    for (RTGCPHYS iPage = 0; iPage < cPages; iPage++, GCPhys += X86_PAGE_SIZE)
+    {
+        const void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhys, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
+            if (SUCCEEDED(hrc))
+            { /* likely */ }
+            else
+            {
+                LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                        GCPhys, hrc, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue));
+                return VERR_NEM_INIT_FAILED;
+            }
+        }
+        else
+        {
+            LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+            return rc;
+        }
+    }
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb);
+#endif
+    RT_NOREF_PV(fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called after the ROM range has been fully completed.
+ *
+ * This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
+ * number of NEMHCNotifyPhysPageProtChanged calls.
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+int nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    LogRel(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
     NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
     return VINF_SUCCESS;
@@ -734 +982 @@
 
 
-int nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTUINT cb, uint32_t fFlags)
-{
-    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+void nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled)
+{
+    LogRel(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
+    NOREF(pVCpu); NOREF(fEnabled);
+}
+
+
+void nemR3NativeNotifyHandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    LogRel(("nemR3NativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
+}
+
+
+void nemR3NativeNotifyHandlerPhysicalDeregister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                                int fRestoreAsRAM, bool fRestoreAsRAM2)
+{
+    LogRel(("nemR3NativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
+            GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
+}
+
+
+void nemR3NativeNotifyHandlerPhysicalModify(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                            RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
+{
+    LogRel(("nemR3NativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
+            GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
+}
+
+
+static int nemR3NativeSetPhysPage(PVM pVM, RTGCPHYS GCPhys, uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
+{
+    /*
+     * Looks like we need to unmap a page before we can change the backing
+     * or even modify the protection.  This is going to be *REALLY* efficient.
+     * PGM lends us two bits to keep track of the state here.
+     */
+    uint8_t const u2OldState = *pu2State;
+    uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
+                             : fPageProt & NEM_PAGE_PROT_READ  ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_NOT_SET;
+    if (   fBackingChanged
+        || u2NewState != u2OldState)
+    {
+        if (u2OldState > NEM_WIN_PAGE_STATE_NOT_SET)
+        {
+            HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_NOT_SET;
+                if (u2NewState == NEM_WIN_PAGE_STATE_NOT_SET)
+                    return VINF_SUCCESS;
+            }
+            else
+            {
+                LogRel(("nemR3NativeSetPhysPage/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                        GCPhys, hrc, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue));
+                return VERR_NEM_INIT_FAILED;
+            }
+        }
+    }
+
+    /*
+     * Writeable mapping?
+     */
+    if (fPageProt & NEM_PAGE_PROT_WRITE)
+    {
+        void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhys, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhys, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+                return VINF_SUCCESS;
+            }
+            LogRel(("nemR3NativeSetPhysPage/writable: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhys, hrc, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemR3NativeSetPhysPage/writable: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+        return rc;
+    }
+
+    if (fPageProt & NEM_PAGE_PROT_READ)
+    {
+        const void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhys, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_READABLE;
+                return VINF_SUCCESS;
+            }
+            LogRel(("nemR3NativeSetPhysPage/readonly: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhys, hrc, hrc, RTNtCurrentTeb()->LastStatusValue, RTNtCurrentTeb()->LastErrorValue));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemR3NativeSetPhysPage/readonly: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+        return rc;
+    }
+
+    /* We already unmapped it above. */
+    *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
     return VINF_SUCCESS;
 }
 
 
-int nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTUINT cb, uint32_t fFlags)
-{
-    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
-    return VINF_SUCCESS;
-}
-
-
-void nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled)
-{
-    NOREF(pVCpu); NOREF(fEnabled);
-}
-
+int nemR3NativeNotifyPhysPageAllocated(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                       PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    LogRel(("nemR3NativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+            GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
+
+    return nemR3NativeSetPhysPage(pVM, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+}
+
+
+void nemR3NativeNotifyPhysPageProtChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                          PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    LogRel(("nemR3NativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+            GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
+
+    nemR3NativeSetPhysPage(pVM, GCPhys, fPageProt, pu2State, false /*fBackingChanged*/);
+}
+
+
+void nemR3NativeNotifyPhysPageChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
+                                      uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    LogRel(("nemR3NativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+            GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
+
+    nemR3NativeSetPhysPage(pVM, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+}
+

trunk/src/VBox/VMM/VMMR3/PGMHandler.cpp

@@ -268 +268 @@
         int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
         if (RT_SUCCESS(rc))
+        {
             PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
+
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
         else
             AssertRC(rc);
@@ -300 +312 @@
         int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
         if (RT_SUCCESS(rc))
+        {
             PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
+
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
         else
             AssertRC(rc);

trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp

@@ -288 +288 @@
                     if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
                         && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-                        pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
+                        pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
                     else
                     {
@@ -474 +474 @@
 #endif
                    )
-                    pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
+                    pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
                 else
                 {
@@ -895 +895 @@
  * @param   GCPhys      The address of the first page.
  * @param   GCPhysLast  The address of the last page.
- * @param   uType       The page type to replace then with.
- */
-static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
+ * @param   enmType     The page type to replace then with.
+ */
+static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPAGETYPE enmType)
 {
     PGM_LOCK_ASSERT_OWNER(pVM);
@@ -910 +910 @@
     while (cPagesLeft-- > 0)
     {
-        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
+        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, enmType);
         AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
 
-        PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
+        PGM_PAGE_SET_TYPE(pVM, pPageDst, enmType);
 
         GCPhys += PAGE_SIZE;
@@ -985 +985 @@
             pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
 
-            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
+            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
             if (RT_FAILURE(rc))
             {
@@ -1020 +1020 @@
             Assert(PGM_PAGE_IS_BALLOONED(pPage));
 
-            /* Change back to zero page. */
+            /* Change back to zero page.  (NEM does not need to be informed.) */
             PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
         }
@@ -1854 +1854 @@
                 if (PGM_PAGE_IS_SHARED(pPage))
                 {
-                    uint32_t u32Checksum = pPage->s.u2Unused0 | ((uint32_t)pPage->s.u2Unused1 << 8);
+                    uint32_t u32Checksum = pPage->s.u2Unused0/* | ((uint32_t)pPage->s.u2Unused1 << 8)*/;
                     if (!u32Checksum)
                     {
@@ -1864 +1864 @@
                             uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
 # if 0
-                            AssertMsg((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
+                            AssertMsg((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
 # else
-                            if ((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum)
+                            if ((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum)
                                 LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
                             else
@@ -1974 +1974 @@
                         else if (!PGM_PAGE_IS_ZERO(pPage))
                         {
-                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                 PGMPAGETYPE_RAM);
                             AssertLogRelRCReturn(rc, rc);
                         }
@@ -2123 +2124 @@
                     if (PGM_PAGE_IS_SHARED(pPage))
                     {
-                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                             PGMPAGETYPE_RAM);
                         AssertLogRelRCReturn(rc, rc);
                     }
@@ -3389 +3391 @@
                 PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
                 PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
+                /* (We tell NEM at the end of the function.) */
 
                 pVM->pgm.s.cZeroPages--;
@@ -3516 +3519 @@
     uint32_t const fNemNotify = (pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2   : 0)
                               | (pFirstMmio->fFlags & PGMREGMMIORANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
-    int rc = NEMR3NotifyPhysMmioExMap(pVM, GCPhys, cbRange, fNemNotify);
+    int rc = NEMR3NotifyPhysMmioExMap(pVM, GCPhys, cbRange, fNemNotify, pFirstMmio->pvR3);
+
     pgmUnlock(pVM);
+
 #ifdef VBOX_WITH_REM
     if (!fRamExists && (pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2)) /** @todo this doesn't look right. */
@@ -3899 +3904 @@
  * @param   pszDesc             Pointer to description string. This must not be freed.
  */
-static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
-                                const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                      const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
 {
     /*
@@ -4270 +4275 @@
          pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
     pgmLock(pVM);
-    int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
+    int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
     pgmUnlock(pVM);
     return rc;
@@ -4315 +4320 @@
 
                 for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                    if (    !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
-                        &&  !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
+                    if (   !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
+                        && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
                     {
                         Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
                         rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
-                                             pRom->GCPhys + (iPage << PAGE_SHIFT));
+                                             pRom->GCPhys + (iPage << PAGE_SHIFT),
+                                             (PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
                         AssertLogRelRCReturn(rc, rc);
                     }
@@ -4457 +4463 @@
     for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
     {
-        if (    GCPhys     <= pRom->GCPhysLast
-            &&  GCPhysLast >= pRom->GCPhys
-            &&  (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
+        if (   GCPhys     <= pRom->GCPhysLast
+            && GCPhysLast >= pRom->GCPhys
+            && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
         {
             /*
@@ -4483 +4489 @@
                     if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
                         rc = rc2;
+                    uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
 
                     PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
@@ -4490 +4497 @@
                     *pRamPage = *pNew;
                     /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
+
+                    /* Tell NEM about the backing and protection change. */
+                    if (VM_IS_NEM_ENABLED(pVM))
+                    {
+                        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
+                        NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
+                                                   pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
+                        PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
+                    }
                 }
                 pRomPage->enmProt = enmProt;
@@ -5247 +5263 @@
  * @param   pPage           Pointer to the page structure.
  * @param   GCPhys          The guest physical address of the page, if applicable.
+ * @param   enmNewType      New page type for NEM notification, since several
+ *                          callers will change the type upon successful return.
  *
  * @remarks The caller must own the PGM lock.
  */
-int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
+int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
+                    PGMPAGETYPE enmNewType)
 {
     /*
@@ -5280 +5299 @@
         return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
     }
+    const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
 
     /* update page count stats. */
@@ -5307 +5327 @@
     /* Flush physical page map TLB entry. */
     pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
+
+    /* Notify NEM. */
+    /** @todo consider doing batch NEM notifications.  */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
+                                   pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
 
     /*

trunk/src/VBox/VMM/VMMR3/PGMSavedState.cpp

@@ -2739 +2739 @@
                                  && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED)
                         {
-                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys);
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys, (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
                             AssertRCReturn(rc, rc);
                         }
@@ -2762 +2762 @@
                                                      ("GCPhys=%RGp %R[pgmpage]\n", GCPhys, pPage), VERR_PGM_LOAD_UNEXPECTED_PAGE_TYPE);
 
-                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys);
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys, (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
                             AssertRCReturn(rc, rc);
                         }