Changeset 27955 in vbox

Timestamp:

Apr 2, 2010 9:12:26 AM (15 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

59659

Message:

wddm: vboxtray: abstraction display driver API for passing escape codes (using ExtEscape for XPDM & PFND3DKMT stugg for WDDM); WDDM miniport driver: basics for handling autoresize & seamles

Location:

trunk/src/VBox/Additions/WINNT

Files:

• Graphics/Miniport/VBoxVideo.cpp (modified) (3 diffs)
• Graphics/Miniport/VBoxVideo.h (modified) (1 diff)
• Graphics/Miniport/wddm/VBoxVideoWddm.cpp (modified) (2 diffs)
• VBoxTray/Makefile.kmk (modified) (2 diffs)
• VBoxTray/VBoxDisplay.cpp (modified) (8 diffs)
• VBoxTray/VBoxDisplay.h (modified) (1 diff)
• VBoxTray/VBoxRestore.cpp (modified) (4 diffs)
• VBoxTray/VBoxSeamless.cpp (modified) (4 diffs)
• VBoxTray/VBoxTray.cpp (modified) (2 diffs)
• VBoxTray/VBoxTray.h (modified) (3 diffs)
• include/VBoxDisplay.h (modified) (1 diff)

trunk/src/VBox/Additions/WINNT/Graphics/Miniport/VBoxVideo.cpp

@@ -1231 +1231 @@
 
 #ifdef VBOXWDDM
+static bool g_bModesTableInitialized = false;
 /**
  * Helper function to dynamically build our table of standard video
@@ -1241 +1242 @@
         D3DKMDT_2DREGION **ppResolutions, uint32_t * pcResolutions)
 {
-    static bool bTableInitialized = false;
-    if(bRebuildTable || !bTableInitialized)
+    if(bRebuildTable || !g_bModesTableInitialized)
     {
         VBoxBuildModesTable(DeviceExtension);
-        bTableInitialized = true;
+        g_bModesTableInitialized = true;
     }
 
@@ -1253 +1253 @@
     *ppResolutions = g_VBoxWddmVideoResolutions;
     *pcResolutions = g_VBoxWddmNumResolutions;
+}
+
+VOID VBoxWddmInvalidateModesTable(PDEVICE_EXTENSION DeviceExtension)
+{
+    g_bModesTableInitialized = false;
 }
 

trunk/src/VBox/Additions/WINNT/Graphics/Miniport/VBoxVideo.h

@@ -587 +587 @@
         D3DKMDT_2DREGION **ppResolutions, uint32_t * pcResolutions);
 
+VOID VBoxWddmInvalidateModesTable(PDEVICE_EXTENSION DeviceExtension);
+
 D3DDDIFORMAT vboxWddmCalcPixelFormat(VIDEO_MODE_INFORMATION *pInfo);
 UINT vboxWddmCalcBitsPerPixel(D3DDDIFORMAT format);

trunk/src/VBox/Additions/WINNT/Graphics/Miniport/wddm/VBoxVideoWddm.cpp

@@ -20 +20 @@
 #include <VBox/VBoxGuestLib.h>
 #include <VBox/VBoxVideo.h>
+#include <wingdi.h> /* needed for RGNDATA definition */
+#include <VBoxDisplay.h> /* this is from Additions/WINNT/include/ to include escape codes */
 
 #define VBOXWDDM_MEMTAG 'MDBV'
@@ -2166 +2168 @@
     dfprintf(("==> "__FUNCTION__ ", hAdapter(0x%x)\n", hAdapter));
 
-    AssertBreakpoint();
+    NTSTATUS Status = STATUS_NOT_SUPPORTED;
+    PDEVICE_EXTENSION pDevExt = (PDEVICE_EXTENSION)hAdapter;
+    if (pEscape->PrivateDriverDataSize >= sizeof (VBOXDISPIFESCAPE))
+    {
+        PVBOXDISPIFESCAPE pEscapeHdr = (PVBOXDISPIFESCAPE)pEscape->pPrivateDriverData;
+        switch (pEscapeHdr->escapeCode)
+        {
+            case VBOXESC_SETVISIBLEREGION:
+            {
+                LPRGNDATA lpRgnData = VBOXDISPIFESCAPE_DATA(pEscapeHdr, RGNDATA);
+                uint32_t cbData = VBOXDISPIFESCAPE_DATA_SIZE(pEscape->PrivateDriverDataSize);
+                uint32_t cbRects = cbData - RT_OFFSETOF(RGNDATA, Buffer);
+                /* the lpRgnData->Buffer comes to us as RECT
+                 * to avoid extra memcpy we cast it to PRTRECT assuming
+                 * they are identical */
+                AssertCompile(sizeof(RECT) == sizeof(RTRECT));
+                AssertCompile(RT_OFFSETOF(RECT, left) == RT_OFFSETOF(RTRECT, xLeft));
+                AssertCompile(RT_OFFSETOF(RECT, bottom) == RT_OFFSETOF(RTRECT, yBottom));
+                AssertCompile(RT_OFFSETOF(RECT, right) == RT_OFFSETOF(RTRECT, xRight));
+                AssertCompile(RT_OFFSETOF(RECT, top) == RT_OFFSETOF(RTRECT, yTop));
+                RTRECT   *pRect = (RTRECT *)&lpRgnData->Buffer;
+
+                uint32_t cRects = cbRects/sizeof(RTRECT);
+                int      rc;
+
+                dprintf(("IOCTL_VIDEO_VBOX_SETVISIBLEREGION cRects=%d\n", cRects));
+                Assert(cbRects >= sizeof(RTRECT)
+                    &&  cbRects == cRects*sizeof(RTRECT)
+                    &&  cRects == lpRgnData->rdh.nCount);
+                if (    cbRects >= sizeof(RTRECT)
+                    &&  cbRects == cRects*sizeof(RTRECT)
+                    &&  cRects == lpRgnData->rdh.nCount)
+                {
+                    /*
+                     * Inform the host about the visible region
+                     */
+                    VMMDevVideoSetVisibleRegion *req = NULL;
+
+                    rc = VbglGRAlloc ((VMMDevRequestHeader **)&req,
+                                      sizeof (VMMDevVideoSetVisibleRegion) + (cRects-1)*sizeof(RTRECT),
+                                      VMMDevReq_VideoSetVisibleRegion);
+                    AssertRC(rc);
+                    if (RT_SUCCESS(rc))
+                    {
+                        req->cRect = cRects;
+                        memcpy(&req->Rect, pRect, cRects*sizeof(RTRECT));
+
+                        rc = VbglGRPerform (&req->header);
+                        AssertRC(rc);
+                        if (!RT_SUCCESS(rc))
+                        {
+                            drprintf((__FUNCTION__": VbglGRPerform failed rc (%d)", rc));
+                            Status = STATUS_UNSUCCESSFUL;
+                        }
+                    }
+                    else
+                    {
+                        drprintf((__FUNCTION__": VbglGRAlloc failed rc (%d)", rc));
+                        Status = STATUS_UNSUCCESSFUL;
+                    }
+                }
+                else
+                {
+                    drprintf((__FUNCTION__": VBOXESC_SETVISIBLEREGION: incorrect buffer size (%d), reported count (%d)\n", cbRects, lpRgnData->rdh.nCount));
+                    AssertBreakpoint();
+                    Status = STATUS_INVALID_PARAMETER;
+                }
+                break;
+            }
+            case VBOXESC_ISVRDPACTIVE:
+                /* @todo: implement */
+                Status = STATUS_SUCCESS;
+                break;
+            case VBOXESC_REINITVIDEOMODES:
+                VBoxWddmInvalidateModesTable(pDevExt);
+                Status = STATUS_SUCCESS;
+                break;
+            default:
+                drprintf((__FUNCTION__": unsupported escape code (0x%x)\n", pEscapeHdr->escapeCode));
+                break;
+        }
+    }
+    else
+    {
+        drprintf((__FUNCTION__": pEscape->PrivateDriverDataSize(%d) < (%d)\n", pEscape->PrivateDriverDataSize, sizeof (VBOXDISPIFESCAPE)));
+        Status = STATUS_INVALID_PARAMETER;
+        AssertBreakpoint();
+    }
 
     dfprintf(("<== "__FUNCTION__ ", hAdapter(0x%x)\n", hAdapter));
 
-    return STATUS_INVALID_PARAMETER;
+    return Status;
 }
 

trunk/src/VBox/Additions/WINNT/VBoxTray/Makefile.kmk

@@ -29 +29 @@
 endif
 VBoxTray_DEFS     = VBOX_WITH_HGCM LOG_TO_BACKDOOR
-ifdef VBOXWDDM
- VBoxTray_DEFS   += VBOXWDDM
- VBoxTray_DEFS   += LOG_ENABLED
-endif
 VBoxTray_INCS     = ../include
 VBoxTray_SOURCES  = \
@@ -53 +49 @@
         $(VBOX_LIB_VBGL_R3)
 
+ifdef VBOXWDDM
+ VBoxTray_DEFS   += VBOXWDDM
+ VBoxTray_DEFS   += LOG_ENABLED
+ # we need only includes for PFND3DKMT stuff as all we link dynamicaly to it
+ VBoxTray_INCS   += $(PATH_SDK_WINDDK_INC_API) $(PATH_SDK_WINDDK_INC_DDK)
+endif
 # VBoxTray.cpp uses VBOX_SVN_REV.
 VBoxTray.cpp_DEFS += VBOX_SVN_REV=$(VBOX_SVN_REV)

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxDisplay.cpp

@@ -39 +39 @@
     /* EnumDisplayDevices does not exist in NT. isVBoxDisplayDriverActive et al. are using these functions. */
     BOOL (WINAPI * pfnEnumDisplayDevices)(IN LPCSTR lpDevice, IN DWORD iDevNum, OUT PDISPLAY_DEVICEA lpDisplayDevice, IN DWORD dwFlags);
-
 } VBOXDISPLAYCONTEXT;
 
 static VBOXDISPLAYCONTEXT gCtx = {0};
+
+#ifdef VBOXWDDM
+static bool vboxWddmReinitVideoModes(VBOXDISPLAYCONTEXT *pCtx)
+{
+    VBOXDISPIFESCAPE escape = {0};
+    escape.escapeCode = VBOXESC_REINITVIDEOMODES;
+    DWORD err = VBoxDispIfEscape(&pCtx->pEnv->dispIf, &escape, 0);
+    if (err != NO_ERROR)
+    {
+        Log((__FUNCTION__": VBoxDispIfEscape failed with err (%d)\n", err));
+        return false;
+    }
+    return true;
+}
+
+typedef enum
+{
+    VBOXDISPLAY_DRIVER_TYPE_UNKNOWN = 0,
+    VBOXDISPLAY_DRIVER_TYPE_XPDM    = 1,
+    VBOXDISPLAY_DRIVER_TYPE_WDDM    = 2
+} VBOXDISPLAY_DRIVER_TYPE;
+
+static VBOXDISPLAY_DRIVER_TYPE getVBoxDisplayDriverType (VBOXDISPLAYCONTEXT *pCtx);
+#endif
 
 int VBoxDisplayInit(const VBOXSERVICEENV *pEnv, void **ppInstance, bool *pfStartThread)
@@ -66 +89 @@
         *(uintptr_t *)&gCtx.pfnEnumDisplayDevices = (uintptr_t)GetProcAddress(hUser, "EnumDisplayDevicesA");
         Log(("VBoxTray: pfnEnumDisplayDevices = %p\n", gCtx.pfnEnumDisplayDevices));
+
+#ifdef VBOXWDDM
+        if (OSinfo.dwMajorVersion >= 6)
+        {
+            /* this is vista and up, check if we need to switch the display driver if to WDDM mode */
+            Log(("VBoxTray: this is vista and up\n"));
+            VBOXDISPLAY_DRIVER_TYPE enmType = getVBoxDisplayDriverType (&gCtx);
+            if (enmType == VBOXDISPLAY_DRIVER_TYPE_WDDM)
+            {
+                Log(("VBoxTray: WDDM driver is installed, switching display driver if to WDDM mode\n"));
+                /* this is hacky, but the most easiest way */
+                DWORD err = VBoxDispIfSwitchMode(const_cast<PVBOXDISPIF>(&pEnv->dispIf), VBOXDISPIF_MODE_WDDM, NULL /* old mode, we don't care about it */);
+                if (err == NO_ERROR)
+                    Log(("VBoxTray: DispIf switched to WDDM mode successfully\n"));
+                else
+                    Log(("VBoxTray: failed to switch DispIf to WDDM mode, err (%d)\n", err));
+            }
+        }
+#endif
     }
     else if (OSinfo.dwMajorVersion <= 4)            /* Windows NT 4.0 */
@@ -89 +131 @@
 }
 
+#ifdef VBOXWDDM
+static VBOXDISPLAY_DRIVER_TYPE getVBoxDisplayDriverType (VBOXDISPLAYCONTEXT *pCtx)
+#else
 static bool isVBoxDisplayDriverActive (VBOXDISPLAYCONTEXT *pCtx)
+#endif
 {
+#ifdef VBOXWDDM
+    VBOXDISPLAY_DRIVER_TYPE enmType = VBOXDISPLAY_DRIVER_TYPE_UNKNOWN;
+#else
     bool result = false;
+#endif
 
     if( pCtx->pfnEnumDisplayDevices )
@@ -120 +170 @@
 
                 if (strcmp(&dispDevice.DeviceString[0], "VirtualBox Graphics Adapter") == 0)
+#ifndef VBOXWDDM
                     result = true;
-#ifdef VBOXWDDM
-                if (strcmp(&dispDevice.DeviceString[0], "VirtualBox Graphics Adapter (Microsoft Corporation - WDDM)") == 0)
-                    result = true;
+#else
+                    enmType = VBOXDISPLAY_DRIVER_TYPE_XPDM;
+                else if (strcmp(&dispDevice.DeviceString[0], "VirtualBox Graphics Adapter (Microsoft Corporation - WDDM)") == 0)
+                    enmType = VBOXDISPLAY_DRIVER_TYPE_WDDM;
 #endif
                 break;
@@ -146 +198 @@
         /* Check for the short name, because all long stuff would be truncated */
         if (strcmp((char*)&tempDevMode.dmDeviceName[0], "VBoxDisp") == 0)
+#ifndef VBOXWDDM
             result = true;
-    }
-
+#else
+            enmType = VBOXDISPLAY_DRIVER_TYPE_XPDM;
+#endif
+    }
+
+#ifndef VBOXWDDM
     return result;
+#else
+    return enmType;
+#endif
 }
 
 /* Returns TRUE to try again. */
-static BOOL ResizeDisplayDevice(ULONG Id, DWORD Width, DWORD Height, DWORD BitsPerPixel)
+static BOOL ResizeDisplayDevice(
+#ifdef VBOXWDDM
+        VBOXDISPLAYCONTEXT *pCtx, VBOXDISPLAY_DRIVER_TYPE enmType,
+#endif
+        ULONG Id, DWORD Width, DWORD Height, DWORD BitsPerPixel
+        )
 {
     BOOL fModeReset = (Width == 0 && Height == 0 && BitsPerPixel == 0);
@@ -334 +399 @@
         Log(("ResizeDisplayDevice: EnumDisplaySettings last error %d\n", GetLastError ()));
     }
+
+#ifdef VBOXWDDM
+    if (enmType == VBOXDISPLAY_DRIVER_TYPE_WDDM)
+        vboxWddmReinitVideoModes(pCtx);
+#endif
 
     /* Assign the new rectangles to displays. */
@@ -472 +542 @@
                          * Only try to change video mode if the active display driver is VBox additions.
                          */
+#ifdef VBOXWDDM
+                        VBOXDISPLAY_DRIVER_TYPE enmDriverType = getVBoxDisplayDriverType (pCtx);
+
+                        if (enmDriverType == VBOXDISPLAY_DRIVER_TYPE_WDDM)
+                            Log(("VBoxDisplayThread : Detected WDDM Driver\n"));
+
+                        if (enmDriverType != VBOXDISPLAY_DRIVER_TYPE_UNKNOWN)
+#else
                         if (isVBoxDisplayDriverActive (pCtx))
+#endif
                         {
                             Log(("VBoxDisplayThread : Display driver is active!\n"));
@@ -481 +560 @@
 
                                 /* W2K or later. */
-                                if (!ResizeDisplayDevice(displayChangeRequest.display,
+                                if (!ResizeDisplayDevice(
+#ifdef VBOXWDDM
+                                                         pCtx, enmDriverType ,
+#endif
+                                                         displayChangeRequest.display,
                                                          displayChangeRequest.xres,
                                                          displayChangeRequest.yres,
-                                                         displayChangeRequest.bpp))
+                                                         displayChangeRequest.bpp
+                                                         ))
                                 {
                                     break;

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxDisplay.h

@@ -27 +27 @@
 void               VBoxDisplayDestroy (const VBOXSERVICEENV *pEnv, void *pInstance);
 
+#ifndef VBOXWDDM
 static bool isVBoxDisplayDriverActive (void);
+#else
+/* @misha: getVBoxDisplayDriverType is used instead.
+ * it seems bad to put static function declaration to header,
+ * so it is moved to VBoxDisplay.cpp */
+#endif
 
 #endif /* __VBOXSERVICEDISPLAY__H */

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxRestore.cpp

@@ -33 +33 @@
     const VBOXSERVICEENV *pEnv;
 
-    BOOL  fRDPState;
+    DWORD  fRDPState;
 } VBOXRESTORECONTEXT;
 
@@ -45 +45 @@
 
     gCtx.pEnv      = pEnv;
-    gCtx.fRDPState = FALSE;
+    gCtx.fRDPState = ERROR_NOT_SUPPORTED;
 
     VBoxRestoreCheckVRDP();
@@ -68 +68 @@
 void VBoxRestoreCheckVRDP()
 {
-    HDC  hdc;
-    BOOL ret;
-
+    DWORD ret;
+    VBOXDISPIFESCAPE escape = {0};
+    escape.escapeCode = VBOXESC_ISVRDPACTIVE;
     /* Check VRDP activity */
-    hdc = GetDC(HWND_DESKTOP);
 
     /* send to display driver */
-    ret = ExtEscape(hdc, VBOXESC_ISVRDPACTIVE, 0, NULL, 0, NULL);
+    ret = VBoxDispIfEscape(&gCtx.pEnv->dispIf, &escape, 0);
     Log(("VBoxRestoreSession -> VRDP activate state = %d\n", ret));
-    ReleaseDC(HWND_DESKTOP, hdc);
 
     if (ret != gCtx.fRDPState)
@@ -83 +81 @@
         DWORD cbReturned;
 
-        if (!DeviceIoControl (gCtx.pEnv->hDriver, (ret) ? VBOXGUEST_IOCTL_ENABLE_VRDP_SESSION : VBOXGUEST_IOCTL_DISABLE_VRDP_SESSION, NULL, 0, NULL, 0, &cbReturned, NULL))
+        if (!DeviceIoControl (gCtx.pEnv->hDriver, ret == NO_ERROR ? VBOXGUEST_IOCTL_ENABLE_VRDP_SESSION : VBOXGUEST_IOCTL_DISABLE_VRDP_SESSION, NULL, 0, NULL, 0, &cbReturned, NULL))
         {
             Log(("VBoxRestoreThread: DeviceIOControl(CtlMask) failed, SeamlessChangeThread exited\n"));

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxSeamless.cpp

@@ -38 +38 @@
     BOOL    (* pfnVBoxRemoveHook)();
 
-    LPRGNDATA lpRgnData;
+    PVBOXDISPIFESCAPE lpEscapeData;
 } VBOXSEAMLESSCONTEXT;
 
@@ -144 +144 @@
         gCtx.pfnVBoxRemoveHook();
 
-    if (gCtx.lpRgnData)
-    {
-        free(gCtx.lpRgnData);
-        gCtx.lpRgnData = NULL;
+    if (gCtx.lpEscapeData)
+    {
+        free(gCtx.lpEscapeData);
+        gCtx.lpEscapeData = NULL;
     }
 }
@@ -238 +238 @@
         if (cbSize)
         {
-            LPRGNDATA lpRgnData = (LPRGNDATA)malloc(cbSize);
-            if (lpRgnData)
-            {
+            PVBOXDISPIFESCAPE lpEscapeData = (PVBOXDISPIFESCAPE)malloc(VBOXDISPIFESCAPE_SIZE(cbSize));
+            if (lpEscapeData)
+            {
+                lpEscapeData->escapeCode = VBOXESC_SETVISIBLEREGION;
+                LPRGNDATA lpRgnData = VBOXDISPIFESCAPE_DATA(lpEscapeData, RGNDATA);
                 memset(lpRgnData, 0, cbSize);
                 cbSize = GetRegionData(param.hrgn, cbSize, lpRgnData);
@@ -254 +256 @@
                     }
 #endif
-                    if (    !gCtx.lpRgnData
-                        ||  (gCtx.lpRgnData->rdh.dwSize + gCtx.lpRgnData->rdh.nRgnSize != cbSize)
-                        ||  memcmp(gCtx.lpRgnData, lpRgnData, cbSize))
+                    LPRGNDATA lpCtxRgnData = VBOXDISPIFESCAPE_DATA(gCtx.lpEscapeData, RGNDATA);
+                    if (    !gCtx.lpEscapeData
+                        ||  (lpCtxRgnData->rdh.dwSize + lpCtxRgnData->rdh.nRgnSize != cbSize)
+                        ||  memcmp(lpCtxRgnData, lpRgnData, cbSize))
                     {
                         /* send to display driver */
-                        ExtEscape(param.hdc, VBOXESC_SETVISIBLEREGION, cbSize, (LPCSTR)lpRgnData, 0, NULL);
-
-                        if (gCtx.lpRgnData)
-                            free(gCtx.lpRgnData);
-                        gCtx.lpRgnData = lpRgnData;
+                        VBoxDispIfEscape(&gCtx.pEnv->dispIf, lpEscapeData, cbSize);
+
+                        if (gCtx.lpEscapeData)
+                            free(gCtx.lpEscapeData);
+                        gCtx.lpEscapeData = lpEscapeData;
                     }
                     else
                         Log(("Visible rectangles haven't changed; ignore\n"));
                 }
-                if (lpRgnData != gCtx.lpRgnData)
-                    free(lpRgnData);
+                if (lpEscapeData != gCtx.lpEscapeData)
+                    free(lpEscapeData);
             }
         }

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxTray.cpp

@@ -370 +370 @@
     svcEnv.hDriver    = gVBoxDriver;
 
+    /* initializes disp-if to default (XPDM) mode */
+    status = VBoxDispIfInit(&svcEnv.dispIf);
+#ifdef VBOXWDDM
+    /* for now the display mode will be adjusted to WDDM mode if needed
+     * on display service initialization when it detects the display driver type */
+#endif
+
     if (status == NO_ERROR)
     {
@@ -610 +617 @@
     return 0;
 }
+
+/* display driver interface abstraction for XPDM & WDDM
+ * with WDDM we can not use ExtEscape to communicate with our driver
+ * because we do not have XPDM display driver any more, i.e. escape requests are handled by cdd
+ * that knows nothing about us */
+DWORD VBoxDispIfInit(PVBOXDISPIF pIf)
+{
+    pIf->enmMode = VBOXDISPIF_MODE_XPDM;
+    return NO_ERROR;
+}
+
+DWORD VBoxDispIfSwitchMode(PVBOXDISPIF pIf, VBOXDISPIF_MODE enmMode, VBOXDISPIF_MODE *penmOldMode)
+{
+    /* @todo: may need to addd synchronization in case we want to change modes dynamically
+     * i.e. currently the mode is supposed to be initialized once on service initialization */
+    if (penmOldMode)
+        *penmOldMode = pIf->enmMode;
+
+    if (enmMode == pIf->enmMode)
+        return VINF_ALREADY_INITIALIZED;
+
+    DWORD err = NO_ERROR;
+    switch (enmMode)
+    {
+        case VBOXDISPIF_MODE_XPDM:
+            Log((__FUNCTION__": request to switch to VBOXDISPIF_MODE_XPDM\n"));
+            pIf->enmMode = VBOXDISPIF_MODE_XPDM;
+            break;
+#ifdef VBOXWDDM
+        case VBOXDISPIF_MODE_WDDM:
+        {
+            Log((__FUNCTION__": request to switch to VBOXDISPIF_MODE_WDDM\n"));
+            OSVERSIONINFO OSinfo;
+            OSinfo.dwOSVersionInfoSize = sizeof (OSinfo);
+            GetVersionEx (&OSinfo);
+            if (OSinfo.dwMajorVersion >= 6)
+            {
+                /* this is vista and up */
+                Log((__FUNCTION__": this is vista and up\n"));
+                HMODULE hGdi32 = GetModuleHandle("gdi32");
+                if (hGdi32 != NULL)
+                {
+                    bool bSupported = true;
+                    pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromHdc = (PFND3DKMT_OPENADAPTERFROMHDC)GetProcAddress(hGdi32, "D3DKMTOpenAdapterFromHdc");
+                    Log((__FUNCTION__"pfnD3DKMTOpenAdapterFromHdc = %p\n", pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromHdc));
+                    bSupported &= !!(pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromHdc);
+
+                    pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromGdiDisplayName = (PFND3DKMT_OPENADAPTERFROMGDIDISPLAYNAME)GetProcAddress(hGdi32, "D3DKMTOpenAdapterFromGdiDisplayName");
+                    Log((__FUNCTION__": pfnD3DKMTOpenAdapterFromGdiDisplayName = %p\n", pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromGdiDisplayName));
+                    bSupported &= !!(pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromGdiDisplayName);
+
+                    pIf->modeData.wddm.pfnD3DKMTCloseAdapter = (PFND3DKMT_CLOSEADAPTER)GetProcAddress(hGdi32, "D3DKMTCloseAdapter");
+                    Log((__FUNCTION__": pfnD3DKMTCloseAdapter = %p\n", pIf->modeData.wddm.pfnD3DKMTCloseAdapter));
+                    bSupported &= !!(pIf->modeData.wddm.pfnD3DKMTCloseAdapter);
+
+                    pIf->modeData.wddm.pfnD3DKMTEscape = (PFND3DKMT_ESCAPE)GetProcAddress(hGdi32, "D3DKMTEscape");
+                    Log((__FUNCTION__": pfnD3DKMTEscape = %p\n", pIf->modeData.wddm.pfnD3DKMTEscape));
+                    bSupported &= !!(pIf->modeData.wddm.pfnD3DKMTCloseAdapter);
+
+                    if (bSupported)
+                        pIf->enmMode = VBOXDISPIF_MODE_WDDM;
+                    else
+                    {
+                        Log((__FUNCTION__": one of pfnD3DKMT function pointers failed to initialize\n"));
+                        err = ERROR_NOT_SUPPORTED;
+                    }
+                }
+                else
+                {
+                    Log((__FUNCTION__": GetModuleHandle(gdi32) failed, err(%d)\n", GetLastError()));
+                    err = ERROR_NOT_SUPPORTED;
+                }
+            }
+            else
+            {
+                Log((__FUNCTION__": can not switch to VBOXDISPIF_MODE_WDDM, because os is not Vista or upper\n"));
+                err = ERROR_NOT_SUPPORTED;
+            }
+            break;
+        }
+#endif
+        default:
+            err = ERROR_INVALID_PARAMETER;
+            break;
+    }
+    return err;
+}
+
+DWORD VBoxDispIfTerm(PVBOXDISPIF pIf)
+{
+    pIf->enmMode = VBOXDISPIF_MODE_UNKNOWN;
+    return NO_ERROR;
+}
+
+static DWORD vboxDispIfEscapeXPDM(PCVBOXDISPIF pIf, PVBOXDISPIFESCAPE pEscape, int cbData)
+{
+    HDC  hdc = GetDC(HWND_DESKTOP);
+    VOID *pvData = cbData ? VBOXDISPIFESCAPE_DATA(pEscape, VOID) : NULL;
+    int iRet = ExtEscape(hdc, pEscape->escapeCode, cbData, (LPCSTR)pvData, 0, NULL);
+    ReleaseDC(HWND_DESKTOP, hdc);
+    if (iRet > 0)
+        return VINF_SUCCESS;
+    else if (iRet == 0)
+        return ERROR_NOT_SUPPORTED;
+    /* else */
+    return ERROR_GEN_FAILURE;
+}
+
+#ifdef VBOXWDDM
+static DWORD vboxDispIfEscapeWDDM(PCVBOXDISPIF pIf, PVBOXDISPIFESCAPE pEscape, int cbData)
+{
+    D3DKMT_OPENADAPTERFROMGDIDISPLAYNAME OpenAdapterData = {0};
+    wcscpy(OpenAdapterData.DeviceName, L"\\\\.\\DISPLAY1");
+
+    DWORD err = ERROR_GEN_FAILURE;
+    NTSTATUS Status = pIf->modeData.wddm.pfnD3DKMTOpenAdapterFromGdiDisplayName(&OpenAdapterData);
+    if (!Status)
+    {
+        D3DKMT_ESCAPE EscapeData = {0};
+        EscapeData.hAdapter = OpenAdapterData.hAdapter;
+        //EscapeData.hDevice = NULL;
+        EscapeData.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
+        EscapeData.Flags.HardwareAccess = 1;
+        EscapeData.pPrivateDriverData = pEscape;
+        EscapeData.PrivateDriverDataSize = VBOXDISPIFESCAPE_SIZE(cbData);
+        //EscapeData.hContext = NULL;
+
+        Status = pIf->modeData.wddm.pfnD3DKMTEscape(&EscapeData);
+        if (!Status)
+            err = NO_ERROR;
+        else
+        {
+            if (Status == 0xC00000BBL) /* not supported */
+                err = ERROR_NOT_SUPPORTED;
+            Log((__FUNCTION__": pfnD3DKMTEscape failed, Status (0x%x)\n", Status));
+        }
+
+        D3DKMT_CLOSEADAPTER ClosaAdapterData = {0};
+        ClosaAdapterData.hAdapter = OpenAdapterData.hAdapter;
+        Status = pIf->modeData.wddm.pfnD3DKMTCloseAdapter(&ClosaAdapterData);
+        if (Status)
+            Log((__FUNCTION__": pfnD3DKMTCloseAdapter failed, Status (0x%x)\n", Status));
+    }
+    else
+        Log((__FUNCTION__": pfnD3DKMTOpenAdapterFromGdiDisplayName failed, Status (0x%x)\n", Status));
+
+    return err;
+}
+#endif
+
+DWORD VBoxDispIfEscape(PCVBOXDISPIF pIf, PVBOXDISPIFESCAPE pEscape, int cbData)
+{
+    switch (pIf->enmMode)
+    {
+        case VBOXDISPIF_MODE_XPDM:
+            return vboxDispIfEscapeXPDM(pIf, pEscape, cbData);
+#ifdef VBOXWDDM
+        case VBOXDISPIF_MODE_WDDM:
+            return vboxDispIfEscapeWDDM(pIf, pEscape, cbData);
+#endif
+        default:
+            Log((__FUNCTION__": unknown mode (%d)\n", pIf->enmMode));
+            return ERROR_INVALID_PARAMETER;
+    }
+}

trunk/src/VBox/Additions/WINNT/VBoxTray/VBoxTray.h

@@ -36 +36 @@
 #include <VBox/VBoxGuest.h> /** @todo use the VbglR3 interface! */
 #include <VBox/VBoxGuestLib.h>
+#include <VBoxDisplay.h>
+#ifdef VBOXWDDM
+# include <d3dkmthk.h>
+#endif
 
 #define WM_VBOX_RESTORED                WM_APP + 1
@@ -44 +48 @@
 #define ID_TRAYICON                     2000
 
+typedef enum
+{
+    VBOXDISPIF_MODE_UNKNOWN = 0,
+    VBOXDISPIF_MODE_XPDM    = 1
+#ifdef VBOXWDDM
+    , VBOXDISPIF_MODE_WDDM  = 2
+#endif
+} VBOXDISPIF_MODE;
+/* display driver interface abstraction for XPDM & WDDM
+ * with WDDM we can not use ExtEscape to communicate with our driver
+ * because we do not have XPDM display driver any more, i.e. escape requests are handled by cdd
+ * that knows nothing about us
+ * NOTE: DispIf makes no checks whether the display driver is actually a VBox driver,
+ * it just switches between using different backend OS API based on the VBoxDispIfSwitchMode call
+ * It's caller's responsibility to initiate it to work in the correct mode */
+typedef struct VBOXDISPIF
+{
+    VBOXDISPIF_MODE enmMode;
+#ifdef VBOXWDDM
+    /* with WDDM the approach is to call into WDDM miniport driver via PFND3DKMT API provided by the GDI,
+     * The PFND3DKMT is supposed to be used by the OpenGL ICD according to MSDN, so this approach is a bit hacky */
+    /* open adapter */
+    union
+    {
+        struct
+        {
+        PFND3DKMT_OPENADAPTERFROMHDC pfnD3DKMTOpenAdapterFromHdc;
+        PFND3DKMT_OPENADAPTERFROMGDIDISPLAYNAME pfnD3DKMTOpenAdapterFromGdiDisplayName;
+        /* close adapter */
+        PFND3DKMT_CLOSEADAPTER pfnD3DKMTCloseAdapter;
+        /* escape */
+        PFND3DKMT_ESCAPE pfnD3DKMTEscape;
+        } wddm;
+    } modeData;
+#endif
+} VBOXDISPIF, *PVBOXDISPIF;
+typedef const struct VBOXDISPIF *PCVBOXDISPIF;
+
+/* initializes the DispIf
+ * Initially the DispIf is configured to work in XPDM mode
+ * call VBoxDispIfSwitchMode to switch the mode to WDDM */
+DWORD VBoxDispIfInit(PVBOXDISPIF pIf);
+DWORD VBoxDispIfSwitchMode(PVBOXDISPIF pIf, VBOXDISPIF_MODE enmMode, VBOXDISPIF_MODE *penmOldMode);
+DECLINLINE(VBOXDISPIF_MODE) VBoxDispGetMode(PVBOXDISPIF pIf) { return pIf->enmMode; }
+DWORD VBoxDispIfTerm(PVBOXDISPIF pIf);
+DWORD VBoxDispIfEscape(PCVBOXDISPIF const pIf, PVBOXDISPIFESCAPE pEscape, int cbData);
 
 /* The environment information for services. */
@@ -51 +101 @@
     HANDLE    hDriver;
     HANDLE    hStopEvent;
+    /* display driver interface, XPDM - WDDM abstraction see VBOXDISPIF** definitions above */
+    VBOXDISPIF dispIf;
 } VBOXSERVICEENV;
 

trunk/src/VBox/Additions/WINNT/include/VBoxDisplay.h

@@ -22 +22 @@
 #define VBOXESC_SETVISIBLEREGION            0xABCD9001
 #define VBOXESC_ISVRDPACTIVE                0xABCD9002
+#ifdef VBOXWDDM
+# define VBOXESC_REINITVIDEOMODES           0xABCD9003
+#endif /* #ifdef VBOXWDDM */
 
+typedef struct VBOXDISPIFESCAPE
+{
+    int escapeCode;
+} VBOXDISPIFESCAPE, *PVBOXDISPIFESCAPE;
+
+#define VBOXDISPIFESCAPE_DATA_OFFSET() ((sizeof (VBOXDISPIFESCAPE) + 7) & ~7)
+#define VBOXDISPIFESCAPE_DATA(_pHead, _t) ( (_t*)(((uint8_t*)(_pHead)) + VBOXDISPIFESCAPE_DATA_OFFSET()))
+#define VBOXDISPIFESCAPE_DATA_SIZE(_s) ( (_s) < VBOXDISPIFESCAPE_DATA_OFFSET() ? 0 : (_s) - VBOXDISPIFESCAPE_DATA_OFFSET() )
+#define VBOXDISPIFESCAPE_SIZE(_cbData) ((_cbData) ? VBOXDISPIFESCAPE_DATA_OFFSET() + (_cbData) : sizeof (VBOXDISPIFESCAPE))
 
 #define IOCTL_VIDEO_VBOX_SETVISIBLEREGION \
     CTL_CODE(FILE_DEVICE_VIDEO, 0xA01, METHOD_BUFFERED, FILE_ANY_ACCESS)
 
-
 #endif /* __VBoxDisplay_h__ */