Changeset 71716 in vbox for trunk/src/VBox/Main

Timestamp:

Apr 6, 2018 6:16:30 PM (7 years ago)

Author:

vboxsync

Message:

VBoxSDS,VBoxSVC: Replaced the enviornment variable hack (VBOX_SERVICE_PROCESS) for preventing VBoxSDS and VBoxSVC from getting watched as clients, with a special export in the executable (Is_VirtualBox_service_process_like_VBoxSDS_And_VBoxSDS). Some cleanups here and there.

Location:

trunk/src/VBox/Main

Files:

• include/RpcChannelHook.h (modified) (3 diffs)
• src-all/win/RpcChannelHook.cpp (modified) (6 diffs)
• src-all/win/VBoxProxyStub.c (modified) (2 diffs)
• src-global/win/VBoxSDS.cpp (modified) (2 diffs)
• src-server/win/svcmain.cpp (modified) (3 diffs)

trunk/src/VBox/Main/include/RpcChannelHook.h

@@ -26 +26 @@
 
 
-typedef void* PRPC_CHANNEL_HOOK;
+typedef void *PRPC_CHANNEL_HOOK;
 
 // {CEDA3E95-A46A-4C41-AA01-EFFD856E455C}
@@ -32 +32 @@
 { 0xceda3e95, 0xa46a, 0x4c41,{ 0xaa, 0x1, 0xef, 0xfd, 0x85, 0x6e, 0x45, 0x5c } };
 
-#if defined(__cplusplus)
-extern "C"
-{
-#endif // #if defined(__cplusplus)
-
-    // C wrapper for using in proxy
-    void SetupClientRpcChannelHook(void);
-
-#if defined(__cplusplus)
-}
-#endif // #if defined(__cplusplus)
+RT_C_DECLS_BEGIN
+// C wrapper for using in proxy
+void SetupClientRpcChannelHook(void);
+RT_C_DECLS_END
 
 
@@ -78 +71 @@
 protected:
     const GUID m_ChannelHookID;
-    static volatile bool s_bChannelRegistered;
-    static volatile bool s_bVBpoxSDSCalledOnce;
+    static volatile bool s_fChannelRegistered;
+    static volatile bool s_fVBpoxSDSCalledOnce;
     static CRpcChannelHook s_RpcChannelHook;
 

trunk/src/VBox/Main/src-all/win/RpcChannelHook.cpp

@@ -31 +31 @@
 #ifdef RT_OS_WINDOWS
 
-volatile bool CRpcChannelHook::s_bChannelRegistered = false;
-volatile bool CRpcChannelHook::s_bVBpoxSDSCalledOnce = false;
+volatile bool CRpcChannelHook::s_fChannelRegistered = false;
+volatile bool CRpcChannelHook::s_fVBpoxSDSCalledOnce = false;
 CRpcChannelHook CRpcChannelHook::s_RpcChannelHook;
 
+
 /*
-*   RpcChannelHook IUnknown interface implementation.
-*/
-
+ * RpcChannelHook IUnknown interface implementation.
+ */
 
 STDMETHODIMP CRpcChannelHook::QueryInterface(REFIID riid, void **ppv)
@@ -69 +69 @@
 }
 
+
+/**
+ * This is a function that we can call from non-C++ proxy stub code.
+ *
+ * @note Warning! function is not thread safe!
+ * @todo Consider using RTONCE to serialize this (though it's likely unncessary
+ *       due to COM).
+ */
+void SetupClientRpcChannelHook(void)
+{
+    // register single hook only
+    if (!CRpcChannelHook::IsChannelHookRegistered())
+    {
+        HRESULT hr = CoRegisterChannelHook(RPC_CHANNEL_EXTENSION_GUID, &CRpcChannelHook::s_RpcChannelHook);
+        NOREF(hr);
+        Assert(SUCCEEDED(hr));
+        CRpcChannelHook::RegisterChannelHook();
+        LogFunc(("Registered RPC client channel hook \n"));
+    }
+}
+
+
 /*
-*   C wrapper functions.
-*/
-
-// Warning: functions below are not thread safe
-extern "C"
-{
-    void SetupClientRpcChannelHook(void)
-    {
-        // register single hook only
-        if (!CRpcChannelHook::IsChannelHookRegistered())
-        {
-            HRESULT hr = CoRegisterChannelHook(RPC_CHANNEL_EXTENSION_GUID,
-                &CRpcChannelHook::s_RpcChannelHook);
-            NOREF(hr);
-            Assert(SUCCEEDED(hr));
-            CRpcChannelHook::RegisterChannelHook();
-            LogFunc(("Registered RPC client channel hook \n"));
-        }
-    }
-}
-
-
-/*
-*   Internal functions.
-*/
+ * Internal methods.
+ */
 
 bool CRpcChannelHook::IsChannelHookRegistered()
 {
-    return ASMAtomicReadBool(&s_bChannelRegistered);
+    return ASMAtomicReadBool(&s_fChannelRegistered);
 }
 
@@ -104 +103 @@
 void CRpcChannelHook::RegisterChannelHook()
 {
-    ASMAtomicWriteBool(&s_bChannelRegistered, true);
+    ASMAtomicWriteBool(&s_fChannelRegistered, true);
 }
 
@@ -124 +123 @@
 }
 
-/*
-* This is callback of RPC channel hook called on COM client when COM method call
-* finished on server and response returned.
-* We use it to catch a moment when a new VirtualBox object sucessfully instantiated.
-* This callback is called in client process - VirtualBox.exe, VBoxManage or custom client
-* If it happend we register new API client in VBoxSDS.
-* Parameters:
-*   uExtent - unique ID of our RPC channel
-*   rIID - IID of called server interface (IVirtualBox in our case)
-*   pDataBuffer - NULL, such as we have nothing to transfer from server side
-*   hrFault - result of called COM server method
-*/
-STDMETHODIMP_(void) CRpcChannelHook::ClientNotify(REFGUID uExtent, REFIID riid,
-    ULONG cbDataSize, void *pDataBuffer, DWORD lDataRep, HRESULT hrFault)
+/**
+ * This is callback of RPC channel hook called on COM client when COM method call
+ * finished on server and response returned.
+ * We use it to catch a moment when a new VirtualBox object sucessfully instantiated.
+ * This callback is called in client process - VirtualBox.exe, VBoxManage or custom client
+ * If it happend we register new API client in VBoxSDS.
+ * Parameters:
+ * @param   uExtent         Unique ID of our RPC channel.
+ * @param   riid            Interface ID of called server interface (IVirtualBox
+ *                          in our case).
+ * @param   cbDataBuffer    ???????????????????
+ * @param   pDataBuffer     NULL, such as we have nothing to transfer from
+ *                          server side.
+ * @param   lDataRep        ???????????
+ * @param   hrFault         result of called COM server method.
+ */
+STDMETHODIMP_(void) CRpcChannelHook::ClientNotify(REFGUID uExtent, REFIID riid, ULONG cbDataSize, void *pDataBuffer,
+                                                  DWORD lDataRep, HRESULT hrFault)
 {
     NOREF(cbDataSize);
@@ -143 +146 @@
     NOREF(lDataRep);
 
-    // Check that it created VirtualBox and this is first method called on server
-    // (CreateInstance)
-    if (uExtent == m_ChannelHookID &&
-        riid == IID_IVirtualBox &&
-        !ASMAtomicReadBool(&s_bVBpoxSDSCalledOnce) &&
-        SUCCEEDED(hrFault) )
+    /*
+     * Check that it created VirtualBox and this is first method called on server
+     * (CreateInstance)
+     */
+    if (   riid == IID_IVirtualBox
+        && uExtent == m_ChannelHookID
+        && SUCCEEDED(hrFault)
+        && !ASMAtomicReadBool(&s_fVBpoxSDSCalledOnce) )
     {
         LogFunc(("Finished call of VirtualBox method\n"));
 
-        ASMAtomicWriteBool(&s_bVBpoxSDSCalledOnce, true);
+        ASMAtomicWriteBool(&s_fVBpoxSDSCalledOnce, true);
+
+        /*
+         * Connect to VBoxSDS.
+         * Note: VBoxSDS can handle duplicate calls
+         */
         ComPtr<IVirtualBoxClientList> ptrClientList;
-        /*
-        * Connect to VBoxSDS.
-        * Note: VBoxSDS can handle duplicate calls
-        */
         HRESULT hrc = CoCreateInstance(CLSID_VirtualBoxClientList, NULL, CLSCTX_LOCAL_SERVER, IID_IVirtualBoxClientList,
-            (void **)ptrClientList.asOutParam());
+                                       (void **)ptrClientList.asOutParam());
         if (SUCCEEDED(hrc))
         {
@@ -173 +179 @@
 }
 
-STDMETHODIMP_(void) CRpcChannelHook::ClientFillBuffer(REFGUID uExtent, REFIID riid,
-    ULONG *pDataSize, void *pDataBuffer)
-{
-    NOREF(uExtent);
-    NOREF(riid);
-    NOREF(pDataSize);
-    NOREF(pDataBuffer);
-}
-
-STDMETHODIMP_(void) CRpcChannelHook::ServerGetSize(REFGUID uExtent, REFIID riid,
-    HRESULT hrFault, ULONG *pDataSize)
+STDMETHODIMP_(void) CRpcChannelHook::ClientFillBuffer(REFGUID uExtent, REFIID riid, ULONG *pDataSize, void *pDataBuffer)
+{
+    RT_NOREF(uExtent, riid, pDataSize, pDataBuffer);
+}
+
+STDMETHODIMP_(void) CRpcChannelHook::ServerGetSize(REFGUID uExtent, REFIID riid, HRESULT hrFault, ULONG *pDataSize)
 {
     // Nothing to send to client side from server side
-    NOREF(uExtent);
-    NOREF(riid);
-    NOREF(hrFault);
+    RT_NOREF(uExtent, riid, hrFault);
     *pDataSize = 0;
 }
 
-STDMETHODIMP_(void) CRpcChannelHook::ServerNotify(REFGUID uExtent, REFIID riid,
-    ULONG cbDataSize, void *pDataBuffer, DWORD lDataRep)
+STDMETHODIMP_(void) CRpcChannelHook::ServerNotify(REFGUID uExtent, REFIID riid, ULONG cbDataSize, void *pDataBuffer,
+                                                  DWORD lDataRep)
 {
     // Nothing to do on server side
-    NOREF(uExtent);
-    NOREF(riid);
-    NOREF(cbDataSize);
-    NOREF(pDataBuffer);
-    NOREF(lDataRep);
-}
-
-STDMETHODIMP_(void) CRpcChannelHook::ServerFillBuffer(REFGUID uExtent, REFIID riid,
-    ULONG *pDataSize, void *pDataBuffer, HRESULT hrFault)
+    RT_NOREF(uExtent, riid, cbDataSize, pDataBuffer, lDataRep);
+}
+
+STDMETHODIMP_(void) CRpcChannelHook::ServerFillBuffer(REFGUID uExtent, REFIID riid, ULONG *pDataSize, void *pDataBuffer,
+                                                      HRESULT hrFault)
 {
     // Nothing to send to client side from server side
-    NOREF(uExtent);
-    NOREF(riid);
-    NOREF(pDataSize);
-    NOREF(pDataBuffer);
-    NOREF(hrFault);
+    RT_NOREF(uExtent, riid, pDataSize, pDataBuffer, hrFault);
 }
 

trunk/src/VBox/Main/src-all/win/VBoxProxyStub.c

@@ -148 +148 @@
 };
 
-BOOL IsVBoxServiceProcess(void)
-{
-    if (GetEnvironmentVariable(L"VBOX_SERVICE_PROCESS", NULL, 0) == 0)
-    {
-        int res = GetLastError();
-        if (res != ERROR_ENVVAR_NOT_FOUND)
-            LogRel(("Error: cannot get service environment variable: %Rrwa\n", res));
-        return false;
-    }
-    return true;
-}
-
 
 /**
@@ -185 +173 @@
             Log12(("VBoxProxyStub[%u]/DllMain: DLL_PROCESS_ATTACH\n", GetCurrentProcessId()));
 
-            /* Install RPC channel hook to intercept a moment just after VirtualBox object activation.
-               It's reports to VBoxSDS that a new VirtualBox API client started. */
 #ifdef VBOX_SDS_CLIENTS_WATCHER
-            if(!IsVBoxServiceProcess())
+            /*
+             * Install RPC channel hook to intercept a moment just after VirtualBox object activation.
+             * It's reports to VBoxSDS that a new VirtualBox API client started.
+             */
+            if (GetProcAddress(GetModuleHandle(NULL), "Is_VirtualBox_service_process_like_VBoxSDS_And_VBoxSDS") == NULL)
                 SetupClientRpcChannelHook();
 #endif

trunk/src/VBox/Main/src-global/win/VBoxSDS.cpp

@@ -695 +695 @@
 
 
-int SetServiceEnvFlag()
+/** Special export that make VBoxProxyStub not register this process as one that
+ * VBoxSDS should be watching.
+ */
+extern "C" DECLEXPORT(void) VBOXCALL Is_VirtualBox_service_process_like_VBoxSDS_And_VBoxSDS(void)
 {
-    int rc = VINF_SUCCESS;
-    if (!SetEnvironmentVariable(L"VBOX_SERVICE_PROCESS", L""))
-    {
-        rc = RTErrConvertFromWin32(GetLastError());
-        LogRel(("Error: cannot set service environment flag:  %Rrs\n", rc));
-    }
-    return rc;
+    /* never called, just need to be here */
 }
 
@@ -728 +725 @@
      */
     RTR3InitExe(argc, &argv, 0);
-
-    SetServiceEnvFlag();
 
     static const RTGETOPTDEF s_aOptions[] =

trunk/src/VBox/Main/src-server/win/svcmain.cpp

@@ -516 +516 @@
 void    VirtualBoxClassFactory::i_finishVBoxSvc()
 {
-    LogRelFunc(("Finish work of VBoxSVc and VBoxSDS\n"));
+    LogRelFunc(("Finish work of VBoxSVC and VBoxSDS\n"));
     if (m_ptrVirtualBoxSDS.isNotNull())
     {
@@ -787 +787 @@
 
 
-int SetServiceEnvFlag()
-{
-    int rc = VINF_SUCCESS;
-    if (!SetEnvironmentVariable(L"VBOX_SERVICE_PROCESS", L""))
-    {
-        rc = RTErrConvertFromWin32(GetLastError());
-        LogRel(("Error: cannot set service environment flag:  %Rrs\n", rc));
-    }
-    return rc;
+/** Special export that make VBoxProxyStub not register this process as one that
+ * VBoxSDS should be watching.
+ */
+extern "C" DECLEXPORT(void) VBOXCALL Is_VirtualBox_service_process_like_VBoxSDS_And_VBoxSDS(void)
+{
+    /* never called, just need to be here */
 }
 
@@ -834 +831 @@
      */
     RTR3InitExe(argc, &argv, 0);
-
-    SetServiceEnvFlag();
 
     static const RTGETOPTDEF s_aOptions[] =