Changeset 98357 in vbox for trunk/src

Timestamp:

Jan 31, 2023 10:53:46 AM (2 years ago)

Author:

vboxsync

Message:

FE/SDL: Fixed and simplified the host key detection ("--detecthostkey") on Windows; we need a window for that in order to receive any keypress events. Factored out that functionality into an own function, as the main function is waaaaaay too long. See @todo. ​bugref:9449

File:

• trunk/src/VBox/Frontends/VBoxSDL/VBoxSDL.cpp (modified) (7 diffs)

trunk/src/VBox/Frontends/VBoxSDL/VBoxSDL.cpp

@@ -689 +689 @@
 #endif /* VBOXSDL_WITH_X11 */
 
+/**
+ * Returns a stringified version of a keyboard modifier.
+ *
+ * @returns Stringified version of the keyboard modifier.
+ * @param   mod                 Modifier code to return a stringified version for.
+ */
+static const char *keyModToStr(unsigned mod)
+{
+    switch (mod)
+    {
+        RT_CASE_RET_STR(KMOD_NONE);
+        RT_CASE_RET_STR(KMOD_LSHIFT);
+        RT_CASE_RET_STR(KMOD_RSHIFT);
+        RT_CASE_RET_STR(KMOD_LCTRL);
+        RT_CASE_RET_STR(KMOD_RCTRL);
+        RT_CASE_RET_STR(KMOD_LALT);
+        RT_CASE_RET_STR(KMOD_RALT);
+        RT_CASE_RET_STR(KMOD_LGUI);
+        RT_CASE_RET_STR(KMOD_RGUI);
+        RT_CASE_RET_STR(KMOD_NUM);
+        RT_CASE_RET_STR(KMOD_CAPS);
+        RT_CASE_RET_STR(KMOD_MODE);
+        RT_CASE_RET_STR(KMOD_SCROLL);
+        default:
+            break;
+    }
+
+    return "<Unknown>";
+}
+
+/**
+ * Handles detecting a host key by printing its values to stdout.
+ *
+ * @returns RTEXITCODE
+ */
+static RTEXITCODE handleDetectHostKey(void)
+{
+    RTEXITCODE rcExit  = RTEXITCODE_SUCCESS;
+
+    int rc = SDL_InitSubSystem(SDL_INIT_VIDEO | SDL_INIT_TIMER);
+    if (rc == 0)
+    {
+        /* We need a window, otherwise we won't get any keypress events. */
+        SDL_Window *pWnd = SDL_CreateWindow("VBoxSDL",
+                                            SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 640, 480, SDL_WINDOW_SHOWN);
+        RTPrintf("Please hit one or two function key(s) to get the --hostkey value. ..\n");
+        RTPrintf("Press CTRL+C to quit.\n");
+        SDL_Event e1;
+        while (SDL_WaitEvent(&e1))
+        {
+            if (    e1.key.keysym.sym == SDLK_c
+                && (e1.key.keysym.mod & KMOD_CTRL) != 0)
+                break;
+            if (e1.type == SDL_QUIT)
+                break;
+            if (e1.type == SDL_KEYDOWN)
+            {
+                unsigned const mod = SDL_GetModState() & ~(KMOD_MODE | KMOD_NUM | KMOD_RESERVED);
+                RTPrintf("--hostkey %d", e1.key.keysym.sym);
+                if (mod)
+                    RTPrintf(" %d\n", mod);
+                else
+                    RTPrintf("\n");
+
+                if (mod)
+                    RTPrintf("Host key is '%s' + '%s'\n", keyModToStr(mod), SDL_GetKeyName(e1.key.keysym.sym));
+                else
+                    RTPrintf("Host key is '%s'\n", SDL_GetKeyName(e1.key.keysym.sym));
+            }
+        }
+        SDL_DestroyWindow(pWnd);
+        SDL_Quit();
+    }
+    else
+    {
+        RTPrintf("Error: SDL_InitSubSystem failed with message '%s'\n", SDL_GetError());
+        rcExit = RTEXITCODE_FAILURE;
+    }
+
+    return rcExit;
+}
 
 /** entry point */
@@ -696 +777 @@
     RT_NOREF(envp);
 #ifdef RT_OS_WINDOWS
+    /* As we run with the WINDOWS subsystem, we need to either attach to or create an own console
+     * to get any stdout / stderr output. */
+    bool fAllocConsole = IsDebuggerPresent();
+    if (!fAllocConsole)
+    {
+        if (!AttachConsole(ATTACH_PARENT_PROCESS))
+            fAllocConsole = true;
+    }
+
+    if (fAllocConsole)
+    {
+        if (!AllocConsole())
+            MessageBox(GetDesktopWindow(), L"Unable to attach to or allocate a console!", L"VBoxSDL", MB_OK | MB_ICONERROR);
+        /* Continue running. */
+    }
+
+    RTFILE hStdIn;
+    RTFileFromNative(&hStdIn,  (RTHCINTPTR)GetStdHandle(STD_INPUT_HANDLE));
+    /** @todo Closing of standard handles not support via IPRT (yet). */
+    RTStrmOpenFileHandle(hStdIn, "r", 0, &g_pStdIn);
+
+    RTFILE hStdOut;
+    RTFileFromNative(&hStdOut,  (RTHCINTPTR)GetStdHandle(STD_OUTPUT_HANDLE));
+    /** @todo Closing of standard handles not support via IPRT (yet). */
+    RTStrmOpenFileHandle(hStdOut, "wt", 0, &g_pStdOut);
+
+    RTFILE hStdErr;
+    RTFileFromNative(&hStdErr,  (RTHCINTPTR)GetStdHandle(STD_ERROR_HANDLE));
+    RTStrmOpenFileHandle(hStdErr, "wt", 0, &g_pStdErr);
+
+    if (!fAllocConsole) /* When attaching to the parent console, make sure we start on a fresh line. */
+        RTPrintf("\n");
+
     ATL::CComModule _Module; /* Required internally by ATL (constructor records instance in global variable). */
-#endif
+#endif /* RT_OS_WINDOWS */
 
 #ifdef Q_WS_X11
@@ -731 +845 @@
 #endif
 
+    RTEXITCODE rcExit = RTEXITCODE_SUCCESS;
+
     /*
      * the hostkey detection mode is unrelated to VM processing, so handle it before
@@ -738 +854 @@
                       || !strcmp(argv[1], "--detecthostkey")))
     {
-        Uint32 fInitSubSystem = SDL_INIT_VIDEO | SDL_INIT_TIMER;
-        int rc = SDL_InitSubSystem(fInitSubSystem);
-        if (rc != 0)
-        {
-            RTPrintf("Error: SDL_InitSubSystem failed with message '%s'\n", SDL_GetError());
-            return 1;
-        }
-        RTPrintf("Please hit one or two function key(s) to get the --hostkey value...\n");
-        SDL_Event event1;
-        while (SDL_WaitEvent(&event1))
-        {
-            if (event1.type == SDL_KEYDOWN)
-            {
-                SDL_Event event2;
-                unsigned  mod = SDL_GetModState() & ~(KMOD_MODE | KMOD_NUM | KMOD_RESERVED);
-                while (SDL_WaitEvent(&event2))
-                {
-                    if (event2.type == SDL_KEYDOWN || event2.type == SDL_KEYUP)
-                    {
-                        /* pressed additional host key */
-                        RTPrintf("--hostkey %d", event1.key.keysym.sym);
-                        if (event2.type == SDL_KEYDOWN)
-                        {
-                            RTPrintf(" %d", event2.key.keysym.sym);
-                            RTPrintf(" %d\n", SDL_GetModState() & ~(KMOD_MODE | KMOD_NUM | KMOD_RESERVED));
-                        }
-                        else
-                        {
-                            RTPrintf(" %d\n", mod);
-                        }
-                        /* we're done */
-                        break;
-                    }
-                }
-                /* we're down */
-                break;
-            }
-        }
-        SDL_Quit();
-        return 1;
-    }
+        rcExit = handleDetectHostKey();
+#ifdef RT_OS_WINDOWS
+        FreeConsole(); /* Detach or destroy (from) console. */
+#endif
+        return rcExit;
+    }
+
+    /** @todo r=andy This function is waaaaaay to long, uses goto's and leaks stuff. Use RTGetOpt handling. */
 
     HRESULT hrc;
@@ -1295 +1378 @@
     else if (pcszSettingsPwFile)
     {
-        int rcExit = settingsPasswordFile(pVirtualBox, pcszSettingsPwFile);
+        rcExit = settingsPasswordFile(pVirtualBox, pcszSettingsPwFile);
         if (rcExit != RTEXITCODE_SUCCESS)
             goto leave;
@@ -2782 +2865 @@
     LogFlow(("Returning from main()!\n"));
     RTLogFlush(NULL);
+
+#ifdef RT_OS_WINDOWS
+    FreeConsole(); /* Detach or destroy (from) console. */
+#endif
+
     return FAILED(hrc) ? 1 : 0;
 }
@@ -2802 +2890 @@
         return RTMsgInitFailure(rc);
 
-#ifdef RT_OS_WINDOWS
-    /* As we run with the WINDOWS subsystem, we need to either attach to or create an own console
-     * to get any stdout / stderr output. */
-    bool fAllocConsole = ::IsDebuggerPresent();
-    if (!fAllocConsole)
-    {
-        if (!AttachConsole(ATTACH_PARENT_PROCESS))
-            fAllocConsole = true;
-    }
-
-    if (fAllocConsole)
-    {
-        if (!AllocConsole())
-            MessageBox(GetDesktopWindow(), L"Unable to attach to or allocate a console!", L"VBoxSDL", MB_OK | MB_ICONERROR);
-        /* Continue running. */
-    }
-
-    RTFILE hStdOut;
-    RTFileFromNative(&hStdOut,  (RTHCINTPTR)GetStdHandle(STD_OUTPUT_HANDLE));
-    /** @todo Closing of standard handles not support via IPRT (yet). */
-    RTStrmOpenFileHandle(hStdOut, "wt", 0, &g_pStdOut);
-
-    RTFILE hStdErr;
-    RTFileFromNative(&hStdErr,  (RTHCINTPTR)GetStdHandle(STD_ERROR_HANDLE));
-    RTStrmOpenFileHandle(hStdErr, "wt", 0, &g_pStdErr);
-
-    if (!fAllocConsole) /* When attaching to the parent console, make sure we start on a fresh line. */
-        RTPrintf("\n");
-#endif /* RT_OS_WINDOWS */
-
-    int rcExit = TrustedMain(argc, argv, NULL);
-
-#ifdef RT_OS_WINDOWS
-    FreeConsole(); /* Detach or destroy (from) console. */
-#endif
-
-    return rcExit;
+    return TrustedMain(argc, argv, NULL);
 }
 #endif /* !VBOX_WITH_HARDENING */