Changeset 75242 in vbox for trunk/src

Timestamp:

Nov 3, 2018 2:59:35 AM (6 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

126340

Message:

DBGPlugInOS2: Early attempts and finding symbols/images. Currently requires copying OS2KRNLR.SYM into <VMDIR>\DEBUG\ as 'DOSCALLS' for the debugger to find it.

File:

• trunk/src/VBox/Debugger/DBGPlugInOS2.cpp (modified) (7 diffs)

trunk/src/VBox/Debugger/DBGPlugInOS2.cpp

@@ -89 +89 @@
     uint32_t    mte_modver;     /**< added even later. */
 } LDRMTE;
+/** @name LDRMTE::mte_flag2 values
+ * @{ */
+#define MTEFORMATMASK       UINT16_C(0x0003)
+#define MTEFORMATR1         UINT16_C(0x0000)
+#define MTEFORMATNE         UINT16_C(0x0001)
+#define MTEFORMATLX         UINT16_C(0x0002)
+#define MTEFORMATR2         UINT16_C(0x0003)
+#define MTESYSTEMDLL        UINT16_C(0x0004)
+#define MTELOADORATTACH     UINT16_C(0x0008)
+#define MTECIRCLEREF        UINT16_C(0x0010)
+#define MTEFREEFIXUPS       UINT16_C(0x0020) /* had different meaning earlier */
+#define MTEPRELOADED        UINT16_C(0x0040)
+#define MTEGETMTEDONE       UINT16_C(0x0080)
+#define MTEPACKSEGDONE      UINT16_C(0x0100)
+#define MTE20LIELIST        UINT16_C(0x0200)
+#define MTESYSPROCESSED     UINT16_C(0x0400)
+#define MTEPSDMOD           UINT16_C(0x0800)
+#define MTEDLLONEXTLST      UINT16_C(0x1000)
+#define MTEPDUMPCIRCREF     UINT16_C(0x2000)
+/** @} */
+/** @name LDRMTE::mte_flag1 values
+ * @{ */
+#define MTE1_NOAUTODS           UINT32_C(0x00000000)
+#define MTE1_SOLO               UINT32_C(0x00000001)
+#define MTE1_INSTANCEDS         UINT32_C(0x00000002)
+#define MTE1_INSTLIBINIT        UINT32_C(0x00000004)
+#define MTE1_GINISETUP          UINT32_C(0x00000008)
+#define MTE1_NOINTERNFIXUPS     UINT32_C(0x00000010)
+#define MTE1_NOEXTERNFIXUPS     UINT32_C(0x00000020)
+#define MTE1_CLASS_ALL          UINT32_C(0x00000000)
+#define MTE1_CLASS_PROGRAM      UINT32_C(0x00000040)
+#define MTE1_CLASS_GLOBAL       UINT32_C(0x00000080)
+#define MTE1_CLASS_SPECIFIC     UINT32_C(0x000000c0)
+#define MTE1_CLASS_MASK         UINT32_C(0x000000c0)
+#define MTE1_MTEPROCESSED       UINT32_C(0x00000100)
+#define MTE1_USED               UINT32_C(0x00000200)
+#define MTE1_DOSLIB             UINT32_C(0x00000400)
+#define MTE1_DOSMOD             UINT32_C(0x00000800) /**< The OS/2 kernel (DOSCALLS).*/
+#define MTE1_MEDIAFIXED         UINT32_C(0x00001000)
+#define MTE1_LDRINVALID         UINT32_C(0x00002000)
+#define MTE1_PROGRAMMOD         UINT32_C(0x00000000)
+#define MTE1_DEVDRVMOD          UINT32_C(0x00004000)
+#define MTE1_LIBRARYMOD         UINT32_C(0x00008000)
+#define MTE1_VDDMOD             UINT32_C(0x00010000)
+#define MTE1_MVDMMOD            UINT32_C(0x00020000)
+#define MTE1_INGRAPH            UINT32_C(0x00040000)
+#define MTE1_GINIDONE           UINT32_C(0x00080000)
+#define MTE1_ADDRALLOCED        UINT32_C(0x00100000)
+#define MTE1_FSDMOD             UINT32_C(0x00200000)
+#define MTE1_FSHMOD             UINT32_C(0x00400000)
+#define MTE1_LONGNAMES          UINT32_C(0x00800000)
+#define MTE1_MEDIACONTIG        UINT32_C(0x01000000)
+#define MTE1_MEDIA16M           UINT32_C(0x02000000)
+#define MTE1_SWAPONLOAD         UINT32_C(0x04000000)
+#define MTE1_PORTHOLE           UINT32_C(0x08000000)
+#define MTE1_MODPROT            UINT32_C(0x10000000)
+#define MTE1_NEWMOD             UINT32_C(0x20000000)
+#define MTE1_DLLTERM            UINT32_C(0x40000000)
+#define MTE1_SYMLOADED          UINT32_C(0x80000000)
+/** @} */
+
 
 /**
@@ -95 +156 @@
 typedef struct LDRSMTE
 {
-    uint32_t    smte_mpages;      /**< module page count. */
-    uint32_t    smte_startobj;    /**< Entrypoint segment number. */
-    uint32_t    smte_eip;         /**< Entrypoint offset value. */
-    uint32_t    smte_stackobj;    /**< Stack segment number. */
-    uint32_t    smte_esp;         /**< Stack offset value*/
-    uint32_t    smte_pageshift;   /**< Page shift value. */
-    uint32_t    smte_fixupsize;   /**< Size of the fixup section. */
-    uint32_t    smte_objtab;      /**< Pointer to LDROTE array. */
-    uint32_t    smte_objcnt;      /**< Number of segments. */
-    uint32_t    smte_objmap;      /**< Address of the object page map. */
-    uint32_t    smte_itermap;     /**< File offset of the iterated data map*/
-    uint32_t    smte_rsrctab;     /**< Pointer to resource table? */
-    uint32_t    smte_rsrccnt;     /**< Number of resource table entries. */
-    uint32_t    smte_restab;      /**< Pointer to the resident name table. */
-    uint32_t    smte_enttab;      /**< Possibly entry point table address, if not file offset. */
-    uint32_t    smte_fpagetab;    /* Offset of Fixup Page Table */
-    uint32_t    smte_frectab;     /* Offset of Fixup Record Table */
-    uint32_t    smte_impmod;
-    uint32_t    smte_impproc;
-    uint32_t    smte_datapage;
-    uint32_t    smte_nrestab;
-    uint32_t    smte_cbnrestab;
-    uint32_t    smte_autods;
-    uint32_t    smte_debuginfo;   /* Offset of the debugging info */
-    uint32_t    smte_debuglen;    /* The len of the debug info in bytes */
-    uint32_t    smte_heapsize;
-    uint32_t    smte_path;        /**< Address of full name string. */
-    uint16_t    smte_semcount;
-    uint16_t    smte_semowner;
-    uint32_t    smte_pfilecache;  /** Address of cached data if replace-module is used. */
-    uint32_t    smte_stacksize;   /**< Stack size for .exe thread 1. */
-    uint16_t    smte_alignshift;
-    uint16_t    smte_NEexpver;
-    uint16_t    smte_pathlen;     /**< Length of smte_path */
-    uint16_t    smte_NEexetype;
-    uint16_t    smte_csegpack;
-    uint8_t     smte_major_os;    /**< added later to lie about OS version */
-    uint8_t     smte_minor_os;    /**< added later to lie about OS version */
+    uint32_t    smte_mpages;      /**< 0x00: module page count. */
+    uint32_t    smte_startobj;    /**< 0x04: Entrypoint segment number. */
+    uint32_t    smte_eip;         /**< 0x08: Entrypoint offset value. */
+    uint32_t    smte_stackobj;    /**< 0x0c: Stack segment number. */
+    uint32_t    smte_esp;         /**< 0x10: Stack offset value*/
+    uint32_t    smte_pageshift;   /**< 0x14: Page shift value. */
+    uint32_t    smte_fixupsize;   /**< 0x18: Size of the fixup section. */
+    uint32_t    smte_objtab;      /**< 0x1c: Pointer to LDROTE array. */
+    uint32_t    smte_objcnt;      /**< 0x20: Number of segments. */
+    uint32_t    smte_objmap;      /**< 0x20: Address of the object page map. */
+    uint32_t    smte_itermap;     /**< 0x20: File offset of the iterated data map*/
+    uint32_t    smte_rsrctab;     /**< 0x20: Pointer to resource table? */
+    uint32_t    smte_rsrccnt;     /**< 0x30: Number of resource table entries. */
+    uint32_t    smte_restab;      /**< 0x30: Pointer to the resident name table. */
+    uint32_t    smte_enttab;      /**< 0x30: Possibly entry point table address, if not file offset. */
+    uint32_t    smte_fpagetab;    /**< 0x30 */
+    uint32_t    smte_frectab;     /**< 0x40 */
+    uint32_t    smte_impmod;      /**< 0x44 */
+    uint32_t    smte_impproc;     /**< 0x48 */
+    uint32_t    smte_datapage;    /**< 0x4c */
+    uint32_t    smte_nrestab;     /**< 0x50 */
+    uint32_t    smte_cbnrestab;   /**< 0x54 */
+    uint32_t    smte_autods;      /**< 0x58 */
+    uint32_t    smte_debuginfo;   /**< 0x5c */
+    uint32_t    smte_debuglen;    /**< 0x60 */
+    uint32_t    smte_heapsize;    /**< 0x64 */
+    uint32_t    smte_path;        /**< 0x68 Address of full name string. */
+    uint16_t    smte_semcount;    /**< 0x6c */
+    uint16_t    smte_semowner;    /**< 0x6e */
+    uint32_t    smte_pfilecache;  /**< 0x70: Address of cached data if replace-module is used. */
+    uint32_t    smte_stacksize;   /**< 0x74: Stack size for .exe thread 1. */
+    uint16_t    smte_alignshift;  /**< 0x78: */
+    uint16_t    smte_NEexpver;    /**< 0x7a: */
+    uint16_t    smte_pathlen;     /**< 0x7c: Length of smte_path */
+    uint16_t    smte_NEexetype;   /**< 0x7e: */
+    uint16_t    smte_csegpack;    /**< 0x80: */
+    uint8_t     smte_major_os;    /**< 0x82: added later to lie about OS version */
+    uint8_t     smte_minor_os;    /**< 0x83: added later to lie about OS version */
 } LDRSMTE;
+AssertCompileSize(LDRSMTE, 0x84);
 
 typedef struct LDROTE
@@ -444 +506 @@
 } DBGDIGGEROS2BUF;
 
-
-static void dbgdiggerOS2ProcessModule(PUVM pUVM, PDBGDIGGEROS2 pThis, DBGDIGGEROS2BUF *pBuf)
+/** Arguments dbgdiggerOS2ProcessModule passes to the module open callback.  */
+typedef struct
+{
+    const char     *pszModPath;
+    const char     *pszModName;
+    LDRMTE const   *pMte;
+    LDRSMTE const  *pSwapMte;
+} DBGDIGGEROS2OPEN;
+
+
+/**
+ * @callback_method_impl{FNRTDBGCFGOPEN, Debug image/image searching callback.}
+ */
+static DECLCALLBACK(int) dbgdiggerOs2OpenModule(RTDBGCFG hDbgCfg, const char *pszFilename, void *pvUser1, void *pvUser2)
+{
+    DBGDIGGEROS2OPEN *pArgs = (DBGDIGGEROS2OPEN *)pvUser1;
+
+    RTDBGMOD hDbgMod = NIL_RTDBGMOD;
+    int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pArgs->pszModName, RTLDRARCH_WHATEVER, hDbgCfg);
+    if (RT_SUCCESS(rc))
+    {
+        /** @todo Do some info matching before using it? */
+
+        *(PRTDBGMOD)pvUser2 = hDbgMod;
+        return VINF_CALLBACK_RETURN;
+    }
+    LogRel(("DbgDiggerOs2: dbgdiggerOs2OpenModule: %Rrc - %s\n", rc, pszFilename));
+    return rc;
+}
+
+
+static void dbgdiggerOS2ProcessModule(PUVM pUVM, PDBGDIGGEROS2 pThis, DBGDIGGEROS2BUF *pBuf,
+                                      const char *pszCacheSubDir, RTDBGAS hAs, RTDBGCFG hDbgCfg)
 {
     RT_NOREF(pThis);
@@ -452 +545 @@
      * Save the MTE.
      */
+    static const char * const s_apszMteFmts[4] = { "Reserved1", "NE", "LX", "Reserved2" };
     LDRMTE const Mte = pBuf->mte;
+    if ((Mte.mte_flags2 & MTEFORMATMASK) != MTEFORMATLX)
+    {
+        LogRel(("DbgDiggerOs2: MTE format not implemented: %s (%d)\n",
+                s_apszMteFmts[(Mte.mte_flags2 & MTEFORMATMASK)], Mte.mte_flags2 & MTEFORMATMASK));
+        return;
+    }
 
     /*
@@ -473 +573 @@
     }
 
-#if 0
     /*
      * Try read the path name, falling back on module name.
      */
-    rc = VERR_NOT_AVAILABLE;
+    char szModPath[260];
+    rc = VERR_READ_ERROR;
     if (SwapMte.smte_path != 0 && SwapMte.smte_pathlen > 0)
     {
-        uint32_t cbToRead = RT_MIN(SwapMte.smte_path, sizeof(*pBuf) - 1);
-        rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrFromFlat(pUVM, &Addr, SwapMte.smte_path), pBuf->ach, cbToRead);
-        pBuf->ach[cbToRead] = '\0';
+        uint32_t cbToRead = RT_MIN(SwapMte.smte_path, sizeof(szModPath) - 1);
+        rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrFromFlat(pUVM, &Addr, SwapMte.smte_path), szModPath, cbToRead);
+        szModPath[cbToRead] = '\0';
     }
     if (RT_FAILURE(rc))
     {
-        memcpy(pBuf->ach, Mte.mte_modname, sizeof(Mte.mte_modname));
-        pBuf->ach[sizeof(Mte.mte_modname)] = '\0';
-        RTStrStripR(pBuf->ach);
-    }
-#endif
-
-    /*
-     * Create a simple module.
-     */
-    memcpy(pBuf->ach, Mte.mte_modname, sizeof(Mte.mte_modname));
-    pBuf->ach[sizeof(Mte.mte_modname)] = '\0';
-    RTStrStripR(pBuf->ach);
-
-    RTDBGMOD hDbgMod;
-    rc = RTDbgModCreate(&hDbgMod, pBuf->ach, 0 /*cbSeg*/, 0 /*fFlags*/);
+        memcpy(szModPath, Mte.mte_modname, sizeof(Mte.mte_modname));
+        szModPath[sizeof(Mte.mte_modname)] = '\0';
+        RTStrStripR(szModPath);
+    }
+    LogRel(("DbgDiggerOS2: szModPath='%s'\n", szModPath));
+
+    /*
+     * Sanitize the module name.
+     */
+    char szModName[16];
+    memcpy(szModName, Mte.mte_modname, sizeof(Mte.mte_modname));
+    szModName[sizeof(Mte.mte_modname)] = '\0';
+    RTStrStripR(szModName);
+
+    /*
+     * Read the object table into the buffer.
+     */
+    rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrFromFlat(pUVM, &Addr, SwapMte.smte_objtab),
+                       &pBuf->aOtes[0], sizeof(pBuf->aOtes[0]) * SwapMte.smte_objcnt);
     if (RT_FAILURE(rc))
     {
-        LogRel(("DbgDiggerOs2: RTDbgModCreate failed: %Rrc\n", rc));
+        LogRel(("DbgDiggerOs2: Error reading object table @ %#RX32 LB %#zx: %Rrc\n",
+                SwapMte.smte_objtab, sizeof(pBuf->aOtes[0]) * SwapMte.smte_objcnt, rc));
         return;
     }
+    for (uint32_t i = 0; i < SwapMte.smte_objcnt; i++)
+    {
+        LogRel(("DbgDiggerOs2:  seg%u: %RX32 LB %#x\n", i, pBuf->aOtes[i].ote_base, pBuf->aOtes[i].ote_size));
+        /** @todo validate it. */
+    }
+
+    /* No need to continue without an address space (shouldn't happen). */
+    if (hAs == NIL_RTDBGAS)
+        return;
+
+    /*
+     * Try find a debug file for this module.
+     */
+    RTDBGMOD hDbgMod = NIL_RTDBGMOD;
+    if (hDbgCfg != NIL_RTDBGCFG)
+    {
+        DBGDIGGEROS2OPEN Args = { szModPath, szModName, &Mte, &SwapMte };
+        RTDbgCfgOpenEx(hDbgCfg, szModPath, pszCacheSubDir, NULL,
+                       RT_OPSYS_OS2 | RTDBGCFG_O_CASE_INSENSITIVE | RTDBGCFG_O_EXECUTABLE_IMAGE
+                       | RTDBGCFG_O_RECURSIVE | RTDBGCFG_O_NO_SYSTEM_PATHS,
+                       dbgdiggerOs2OpenModule, &Args, &hDbgMod);
+    }
+
+    /*
+     * Fallback is a simple module into which we insert sections.
+     */
+    uint32_t cSegments = SwapMte.smte_objcnt;
+    if (hDbgMod == NIL_RTDBGMOD)
+    {
+        rc = RTDbgModCreate(&hDbgMod, szModName, 0 /*cbSeg*/, 0 /*fFlags*/);
+        if (RT_SUCCESS(rc))
+        {
+            uint32_t uRva = 0;
+            for (uint32_t i = 0; i < SwapMte.smte_objcnt; i++)
+            {
+                char szSegNm[16];
+                RTStrPrintf(szSegNm, sizeof(szSegNm), "seg%u", i);
+                rc = RTDbgModSegmentAdd(hDbgMod, uRva, pBuf->aOtes[i].ote_size, szSegNm, 0 /*fFlags*/, NULL);
+                if (RT_FAILURE(rc))
+                {
+                    LogRel(("DbgDiggerOs2: RTDbgModSegmentAdd failed (i=%u, ote_size=%#x): %Rrc\n",
+                            i, pBuf->aOtes[i].ote_size, rc));
+                    cSegments = i;
+                    break;
+                }
+                uRva += RT_ALIGN_32(pBuf->aOtes[i].ote_size, _4K);
+            }
+        }
+        else
+        {
+            LogRel(("DbgDiggerOs2: RTDbgModCreate failed: %Rrc\n", rc));
+            return;
+        }
+    }
+
+    /*
+     * Tag the module and link its segments.
+     */
     rc = RTDbgModSetTag(hDbgMod, DIG_OS2_MOD_TAG);
     if (RT_SUCCESS(rc))
     {
-        RTDBGAS hAs = DBGFR3AsResolveAndRetain(pUVM, DBGF_AS_KERNEL);
-        if (hAs != NIL_RTDBGAS)
-        {
-            /*
-             * Read the object table and do the linking of each of them.
-             */
-            rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrFromFlat(pUVM, &Addr, SwapMte.smte_objtab),
-                               &pBuf->aOtes[0], sizeof(pBuf->aOtes[0]) * SwapMte.smte_objcnt);
-            if (RT_SUCCESS(rc))
-            {
-                uint32_t uRva = 0;
-                for (uint32_t i = 0; i < SwapMte.smte_objcnt; i++)
-                {
-                    char szSegNm[16];
-                    RTStrPrintf(szSegNm, sizeof(szSegNm), "seg%u", i);
-                    rc = RTDbgModSegmentAdd(hDbgMod, uRva, pBuf->aOtes[i].ote_size, szSegNm, 0 /*fFlags*/, NULL);
-                    if (RT_FAILURE(rc))
-                    {
-                        LogRel(("DbgDiggerOs2: RTDbgModSegmentAdd failed (i=%u, ote_size=%#x): %Rrc\n",
-                                i, pBuf->aOtes[i].ote_size, rc));
-                        break;
-                    }
-                    rc = RTDbgAsModuleLinkSeg(hAs, hDbgMod, i, pBuf->aOtes[i].ote_base, RTDBGASLINK_FLAGS_REPLACE /*fFlags*/);
-                    if (RT_FAILURE(rc))
-                        LogRel(("DbgDiggerOs2: RTDbgAsModuleLinkSeg failed (i=%u, ote_base=%#x): %Rrc\n",
-                                i, pBuf->aOtes[i].ote_base, rc));
-                    uRva += RT_ALIGN_32(pBuf->aOtes[i].ote_size, _4K);
-                }
-            }
-            else
-                LogRel(("DbgDiggerOs2: Error reading object table @ %#RX32 LB %#zx: %Rrc\n",
-                        SwapMte.smte_objtab, sizeof(pBuf->aOtes[0]) * SwapMte.smte_objcnt, rc));
+        for (uint32_t i = 0; i < SwapMte.smte_objcnt; i++)
+        {
+            rc = RTDbgAsModuleLinkSeg(hAs, hDbgMod, i, pBuf->aOtes[i].ote_base, RTDBGASLINK_FLAGS_REPLACE /*fFlags*/);
+            if (RT_FAILURE(rc))
+                LogRel(("DbgDiggerOs2: RTDbgAsModuleLinkSeg failed (i=%u, ote_base=%#x): %Rrc\n",
+                        i, pBuf->aOtes[i].ote_base, rc));
         }
-        else
-            LogRel(("DbgDiggerOs2: DBGFR3AsResolveAndRetain failed\n"));
-        RTDbgAsRelease(hAs);
     }
     else
         LogRel(("DbgDiggerOs2: RTDbgModSetTag failed: %Rrc\n", rc));
     RTDbgModRelease(hDbgMod);
-
 }
 
@@ -602 +735 @@
                 if (RT_SUCCESS(rc))
                 {
+                    uint32_t uOs2Krnl = UINT32_MAX;
+                    RTDBGCFG hDbgCfg  = DBGFR3AsGetConfig(pUVM); /* (don't release this) */
+                    RTDBGAS  hAs      = DBGFR3AsResolveAndRetain(pUVM, DBGF_AS_GLOBAL);
+
+                    char szCacheSubDir[24];
+                    RTStrPrintf(szCacheSubDir, sizeof(szCacheSubDir), "os2-%u.%u", pThis->OS2MajorVersion, pThis->OS2MinorVersion);
+
                     DBGFR3AddrFromFlat(pUVM, &Addr, uBuf.au32[0]);
                     while (Addr.FlatPtr != 0 && Addr.FlatPtr != UINT32_MAX)
@@ -610 +750 @@
                         LogRel(("DbgDiggerOs2: Module @ %#010RX32: %.8s %#x %#x\n", (uint32_t)Addr.FlatPtr,
                                 uBuf.mte.mte_modname, uBuf.mte.mte_flags1, uBuf.mte.mte_flags2));
+                        if (uBuf.mte.mte_flags1 & MTE1_DOSMOD)
+                            uOs2Krnl = (uint32_t)Addr.FlatPtr;
 
                         DBGFR3AddrFromFlat(pUVM, &Addr, uBuf.mte.mte_link);
-                        dbgdiggerOS2ProcessModule(pUVM, pThis, &uBuf);
+                        dbgdiggerOS2ProcessModule(pUVM, pThis, &uBuf, szCacheSubDir, hAs, hDbgCfg);
                     }
+
+                    /* Load the kernel again. To make sure we didn't drop any segments due
+                       to overlap/conflicts/whatever.  */
+                    if (uOs2Krnl != UINT32_MAX)
+                    {
+                        rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrFromFlat(pUVM, &Addr, uOs2Krnl),
+                                           &uBuf.mte, sizeof(uBuf.mte));
+                        if (RT_SUCCESS(rc))
+                        {
+                            LogRel(("DbgDiggerOs2: Module @ %#010RX32: %.8s %#x %#x [again]\n", (uint32_t)Addr.FlatPtr,
+                                    uBuf.mte.mte_modname, uBuf.mte.mte_flags1, uBuf.mte.mte_flags2));
+                            dbgdiggerOS2ProcessModule(pUVM, pThis, &uBuf, szCacheSubDir, hAs, hDbgCfg);
+                        }
+                    }
+
+                    RTDbgAsRelease(hAs);
                 }
             }