Changeset 4557 in vbox

Timestamp:

Sep 6, 2007 6:59:25 AM (17 years ago)

Author:

vboxsync

Message:

Main: Attempted to fix the CoInitializeEx race condition (#2304).

Location:

trunk/src/VBox

Files:

• Frontends/VirtualBox/src/COMDefs.cpp (modified) (1 diff)
• Main/glue/initterm.cpp (modified) (2 diffs)

trunk/src/VBox/Frontends/VirtualBox/src/COMDefs.cpp

@@ -90 +90 @@
     LogFlowFuncEnter();
 
-    HRESULT rc = S_OK;
-
-#if !defined (VBOX_WITH_XPCOM)
-
-    /* disable this damn CoInitialize* somehow made by Qt during
-     * creation of the QApplication instance (didn't explore deeply
-     * why does it do this) */
-    CoUninitialize();
-
-#endif /* !defined (VBOX_WITH_XPCOM) */
-
-    rc = com::Initialize();
+    /* Note: On Win32, Qt somehow calls CoInitialize[Ex]() during creation of
+     * the QApplication instance (didn't explore deeply why it does so) with
+     * different flags which is incompatible with our multithreaded
+     * apartment. com::Initialize() will properly care of this situation. */
+
+    HRESULT rc = com::Initialize();
 
 #if defined (VBOX_WITH_XPCOM)

trunk/src/VBox/Main/glue/initterm.cpp

@@ -46 +46 @@
 #include "VBox/com/assert.h"
 
+#include "../include/Logging.h"
 
 namespace com
@@ -171 +172 @@
 #if !defined (VBOX_WITH_XPCOM)
 
-    rc = CoInitializeEx (NULL, COINIT_MULTITHREADED |
-                               COINIT_DISABLE_OLE1DDE |
-                               COINIT_SPEED_OVER_MEMORY);
+    DWORD flags = COINIT_MULTITHREADED |
+                  COINIT_DISABLE_OLE1DDE |
+                  COINIT_SPEED_OVER_MEMORY;
+
+    rc = CoInitializeEx (NULL, flags);
+
+    /* If we fail to set the necessary apartment model, it may mean that some
+     * DLL that was indirectly loaded by the process calling this function has
+     * already initialized COM on the given thread in an incompatible way
+     * which we can't leave with. Therefore, we try to fix this by using the
+     * brute force method: */
+
+    enum { MaxTries = 10000 };
+    int tries = MaxTries;
+    while (rc == RPC_E_CHANGED_MODE && tries --)
+    {
+        LogFlowFunc (("COM already initialized in wrong apartment mode, "
+                      "will reinitialize.\n"));
+
+        CoUninitialize();
+        rc = CoInitializeEx (NULL, flags);
+        if (rc == S_OK)
+        {
+            /* We've successfully reinitialized COM; restore the
+             * initialization reference counter */
+
+            LogFlowFunc (("Will call CoInitializeEx() %d times.\n",
+                          MaxTries - tries));
+
+            while (tries ++ < MaxTries)
+            {
+                rc = CoInitializeEx (NULL, flags);
+                Assert (rc == S_FALSE);
+            }
+        }
+    }
+
+    /* the overall result must be either S_OK or S_FALSE */
+    AssertMsg (rc == S_OK || rc == S_FALSE, ("rc=%08X\n", rc));
 
 #else /* !defined (VBOX_WITH_XPCOM) */