VirtualBox

Changeset 101523 in vbox


Ignore:
Timestamp:
Oct 20, 2023 2:46:13 PM (19 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
159600
Message:

VMM/IEM: Cleanups. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r101518 r101523  
    115115
    116116
     117/*********************************************************************************************************************************
     118*   Defined Constants And Macros                                                                                                 *
     119*********************************************************************************************************************************/
    117120/** Always count instructions for now. */
    118121#define IEMNATIVE_WITH_INSTRUCTION_COUNTING
     122
     123
     124/*********************************************************************************************************************************
     125*   Internal Functions                                                                                                           *
     126*********************************************************************************************************************************/
     127static uint32_t iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     128                                                uint8_t idxReg, IEMNATIVEGSTREG enmGstReg);
    119129
    120130
    … …  
    18331843
    18341844
     1845
     1846/*********************************************************************************************************************************
     1847*   Register Allocator                                                                                                           *
     1848*********************************************************************************************************************************/
     1849
    18351850/**
    18361851 * Register parameter indexes (indexed by argument number).
    … …  
    18861901#endif
    18871902};
    1888 
    1889 
    1890 DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
    1891                                                      IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
    1892 {
    1893     pReNative->bmHstRegs |= RT_BIT_32(idxReg);
    1894 
    1895     pReNative->aHstRegs[idxReg].enmWhat        = enmWhat;
    1896     pReNative->aHstRegs[idxReg].fGstRegShadows = 0;
    1897     pReNative->aHstRegs[idxReg].idxVar         = idxVar;
    1898     return (uint8_t)idxReg;
    1899 }
    1900 
    1901 
    1902 /**
    1903  * Locate a register, possibly freeing one up.
    1904  *
    1905  * This ASSUMES the caller has done the minimal/optimal allocation checks and
    1906  * failed.
    1907  */
    1908 static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fAllowVolatile) RT_NOEXCEPT
    1909 {
    1910     uint32_t fRegMask = fAllowVolatile
    1911                       ? IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK
    1912                       : IEMNATIVE_HST_GREG_MASK & ~(IEMNATIVE_REG_FIXED_MASK | IEMNATIVE_CALL_VOLATILE_GREG_MASK);
    1913 
    1914     /*
    1915      * Try a freed register that's shadowing a guest register
    1916      */
    1917     uint32_t fRegs = ~pReNative->bmHstRegs & fRegMask;
    1918     if (fRegs)
    1919     {
    1920         /** @todo pick better here:    */
    1921         unsigned const idxReg = ASMBitFirstSetU32(fRegs) - 1;
    1922 
    1923         Assert(pReNative->aHstRegs[idxReg].fGstRegShadows != 0);
    1924         Assert(   (pReNative->aHstRegs[idxReg].fGstRegShadows & pReNative->bmGstRegShadows)
    1925                == pReNative->aHstRegs[idxReg].fGstRegShadows);
    1926         Assert(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg));
    1927 
    1928         pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxReg].fGstRegShadows;
    1929         pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
    1930         pReNative->aHstRegs[idxReg].fGstRegShadows = 0;
    1931         return idxReg;
    1932     }
    1933 
    1934     /*
    1935      * Try free up a variable that's in a register.
    1936      *
    1937      * We do two rounds here, first evacuating variables we don't need to be
    1938      * saved on the stack, then in the second round move things to the stack.
    1939      */
    1940     for (uint32_t iLoop = 0; iLoop < 2; iLoop++)
    1941     {
    1942         uint32_t fVars = pReNative->bmVars;
    1943         while (fVars)
    1944         {
    1945             uint32_t const idxVar = ASMBitFirstSetU32(fVars) - 1;
    1946             uint8_t const  idxReg = pReNative->aVars[idxVar].idxReg;
    1947             if (   idxReg < RT_ELEMENTS(pReNative->aHstRegs)
    1948                 && (RT_BIT_32(idxReg) & fRegMask)
    1949                 && (  iLoop == 0
    1950                     ? pReNative->aVars[idxVar].enmKind != kIemNativeVarKind_Stack
    1951                     : pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack))
    1952             {
    1953                 Assert(pReNative->bmHstRegs & RT_BIT_32(idxReg));
    1954                 Assert(   (pReNative->bmGstRegShadows & pReNative->aHstRegs[idxReg].fGstRegShadows)
    1955                        == pReNative->aHstRegs[idxReg].fGstRegShadows);
    1956                 Assert(   RT_BOOL(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg))
    1957                        == RT_BOOL(pReNative->aHstRegs[idxReg].fGstRegShadows));
    1958 
    1959                 if (pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
    1960                 {
    1961                     AssertReturn(pReNative->aVars[idxVar].idxStackSlot != UINT8_MAX, UINT8_MAX);
    1962                     uint32_t off = *poff;
    1963                     *poff = off = iemNativeEmitStoreGprByBp(pReNative, off,
    1964                                                               pReNative->aVars[idxVar].idxStackSlot * sizeof(uint64_t)
    1965                                                             - IEMNATIVE_FP_OFF_STACK_VARS,
    1966                                                             idxReg);
    1967                     AssertReturn(off != UINT32_MAX, UINT8_MAX);
    1968                 }
    1969 
    1970                 pReNative->aVars[idxVar].idxReg    = UINT8_MAX;
    1971                 pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxReg].fGstRegShadows;
    1972                 pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
    1973                 pReNative->bmHstRegs              &= ~RT_BIT_32(idxReg);
    1974                 return idxReg;
    1975             }
    1976             fVars &= ~RT_BIT_32(idxVar);
    1977         }
    1978     }
    1979 
    1980     AssertFailedReturn(UINT8_MAX);
    1981 }
    1982 
    1983 
    1984 /**
    1985  * Moves a variable to a different register or spills it onto the stack.
    1986  *
    1987  * This must be a stack variable (kIemNativeVarKind_Stack) because the other
    1988  * kinds can easily be recreated if needed later.
    1989  *
    1990  * @returns The new code buffer position, UINT32_MAX on failure.
    1991  * @param   pReNative       The native recompile state.
    1992  * @param   off             The current code buffer position.
    1993  * @param   idxVar          The variable index.
    1994  * @param   fForbiddenRegs  Mask of the forbidden registers.  Defaults to
    1995  *                          call-volatile registers.
    1996  */
    1997 static uint32_t iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
    1998                                                 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK)
    1999 {
    2000     Assert(idxVar < RT_ELEMENTS(pReNative->aVars));
    2001     Assert(pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
    2002 
    2003     uint8_t const idxRegOld = pReNative->aVars[idxVar].idxReg;
    2004     Assert(idxRegOld < RT_ELEMENTS(pReNative->aHstRegs));
    2005     Assert(pReNative->bmHstRegs & RT_BIT_32(idxRegOld));
    2006     Assert(pReNative->aHstRegs[idxRegOld].enmWhat == kIemNativeWhat_Var);
    2007     Assert(   (pReNative->bmGstRegShadows & pReNative->aHstRegs[idxRegOld].fGstRegShadows)
    2008            == pReNative->aHstRegs[idxRegOld].fGstRegShadows);
    2009     Assert(   RT_BOOL(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxRegOld))
    2010            == RT_BOOL(pReNative->aHstRegs[idxRegOld].fGstRegShadows));
    2011 
    2012 
    2013     /** @todo Add statistics on this.*/
    2014     /** @todo Implement basic variable liveness analysis (python) so variables
    2015      * can be freed immediately once no longer used.  This has the potential to
    2016      * be trashing registers and stack for dead variables. */
    2017 
    2018     /*
    2019      * First try move it to a different register, as that's cheaper.
    2020      */
    2021     fForbiddenRegs |= RT_BIT_32(idxRegOld);
    2022     fForbiddenRegs |= IEMNATIVE_REG_FIXED_MASK;
    2023     uint32_t fRegs = ~pReNative->bmHstRegs & ~fForbiddenRegs;
    2024     if (fRegs)
    2025     {
    2026         /* Avoid using shadow registers, if possible. */
    2027         if (fRegs & ~pReNative->bmHstRegsWithGstShadow)
    2028             fRegs &= ~pReNative->bmHstRegsWithGstShadow;
    2029         unsigned const idxRegNew = ASMBitFirstSetU32(fRegs) - 1;
    2030 
    2031         uint64_t fGstRegShadows = pReNative->aHstRegs[idxRegOld].fGstRegShadows;
    2032         pReNative->aHstRegs[idxRegNew].fGstRegShadows = fGstRegShadows;
    2033         pReNative->aHstRegs[idxRegNew].enmWhat        = kIemNativeWhat_Var;
    2034         pReNative->aHstRegs[idxRegNew].idxVar         = idxVar;
    2035         if (fGstRegShadows)
    2036         {
    2037             pReNative->bmHstRegsWithGstShadow |= RT_BIT_32(idxRegNew);
    2038             while (fGstRegShadows)
    2039             {
    2040                 unsigned const idxGstReg = ASMBitFirstSetU64(fGstRegShadows);
    2041                 fGstRegShadows &= ~RT_BIT_64(idxGstReg);
    2042 
    2043                 Assert(pReNative->aidxGstRegShadows[idxGstReg] == idxRegOld);
    2044                 pReNative->aidxGstRegShadows[idxGstReg] = idxRegNew;
    2045             }
    2046         }
    2047 
    2048         pReNative->aVars[idxVar].idxReg = (uint8_t)idxRegNew;
    2049         pReNative->bmHstRegs           |= RT_BIT_32(idxRegNew);
    2050     }
    2051     /*
    2052      * Otherwise we must spill the register onto the stack.
    2053      */
    2054     else
    2055     {
    2056         AssertReturn(pReNative->aVars[idxVar].idxStackSlot != UINT8_MAX, UINT32_MAX);
    2057         off = iemNativeEmitStoreGprByBp(pReNative, off,
    2058                                         pReNative->aVars[idxVar].idxStackSlot * sizeof(uint64_t) - IEMNATIVE_FP_OFF_STACK_VARS,
    2059                                         idxRegOld);
    2060         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    2061 
    2062         pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
    2063         pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxRegOld].fGstRegShadows;
    2064     }
    2065 
    2066     pReNative->bmHstRegs &= ~RT_BIT_32(idxRegOld);
    2067     pReNative->aHstRegs[idxRegOld].fGstRegShadows = 0;
    2068     return off;
    2069 }
    2070 
    2071 
    2072 /**
    2073  * Allocates a temporary host general purpose register.
    2074  *
    2075  * This may emit code to save register content onto the stack in order to free
    2076  * up a register.
    2077  *
    2078  * @returns The host register number, UINT8_MAX on failure.
    2079  * @param   pReNative       The native recompile state.
    2080  * @param   poff            Pointer to the variable with the code buffer position.
    2081  *                          This will be update if we need to move a variable from
    2082  *                          register to stack in order to satisfy the request.
    2083  * @param   fPreferVolatile Wheter to prefer volatile over non-volatile
    2084  *                          registers (@c true, default) or the other way around
    2085  *                          (@c false, for iemNativeRegAllocTmpForGuestReg()).
    2086  */
    2087 DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
    2088                                          bool fPreferVolatile /*= true*/) RT_NOEXCEPT
    2089 {
    2090     /*
    2091      * Try find a completely unused register, preferably a call-volatile one.
    2092      */
    2093     uint8_t  idxReg;
    2094     uint32_t fRegs = ~pReNative->bmHstRegs
    2095                    & ~pReNative->bmHstRegsWithGstShadow
    2096                    & (~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK);
    2097     if (fRegs)
    2098     {
    2099         if (fPreferVolatile)
    2100             idxReg = (uint8_t)ASMBitFirstSetU32(  fRegs & IEMNATIVE_CALL_VOLATILE_GREG_MASK
    2101                                                 ? fRegs & IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
    2102         else
    2103             idxReg = (uint8_t)ASMBitFirstSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
    2104                                                 ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
    2105         Assert(pReNative->aHstRegs[idxReg].fGstRegShadows == 0);
    2106         Assert(!(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg)));
    2107     }
    2108     else
    2109     {
    2110         idxReg = iemNativeRegAllocFindFree(pReNative, poff, true /*fAllowVolatile*/);
    2111         AssertReturn(idxReg != UINT8_MAX, UINT8_MAX);
    2112     }
    2113     return iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Tmp);
    2114 }
    2115 
    2116 
    2117 /**
    2118  * Allocates a temporary register for loading an immediate value into.
    2119  *
    2120  * This will emit code to load the immediate, unless there happens to be an
    2121  * unused register with the value already loaded.
    2122  *
    2123  * The caller will not modify the returned register, it must be considered
    2124  * read-only.  Free using iemNativeRegFreeTmpImm.
    2125  *
    2126  * @returns The host register number, UINT8_MAX on failure.
    2127  * @param   pReNative       The native recompile state.
    2128  * @param   poff            Pointer to the variable with the code buffer position.
    2129  * @param   uImm            The immediate value that the register must hold upon
    2130  *                          return.
    2131  * @param   fPreferVolatile Wheter to prefer volatile over non-volatile
    2132  *                          registers (@c true, default) or the other way around
    2133  *                          (@c false).
    2134  *
    2135  * @note    Reusing immediate values has not been implemented yet.
    2136  */
    2137 DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
    2138                                             bool fPreferVolatile /*= true*/) RT_NOEXCEPT
    2139 {
    2140     uint8_t idxReg = iemNativeRegAllocTmp(pReNative, poff, fPreferVolatile);
    2141     if (idxReg < RT_ELEMENTS(pReNative->aHstRegs))
    2142     {
    2143         uint32_t off = *poff;
    2144         *poff = off = iemNativeEmitLoadGprImm64(pReNative, off, idxReg, uImm);
    2145         AssertReturnStmt(off != UINT32_MAX, iemNativeRegFreeTmp(pReNative, idxReg), UINT8_MAX);
    2146     }
    2147     return idxReg;
    2148 }
    2149 
    21501903
    21511904/**
    … …  
    22241977};
    22251978
    2226 /**
    2227  * Loads the guest shadow register @a enmGstReg into host reg @a idxHstReg, zero
    2228  * extending to 64-bit width.
    2229  *
    2230  * @returns New code buffer offset on success, UINT32_MAX on failure.
    2231  * @param   pReNative   .
    2232  * @param   off         The current code buffer position.
    2233  * @param   idxHstReg   The host register to load the guest register value into.
    2234  * @param   enmGstReg   The guest register to load.
    2235  *
    2236  * @note This does not mark @a idxHstReg as having a shadow copy of @a enmGstReg,
    2237  *       that is something the caller needs to do if applicable.
    2238  */
    2239 DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    2240                                                           uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg)
    2241 {
    2242     Assert((unsigned)enmGstReg < RT_ELEMENTS(g_aGstShadowInfo));
    2243     Assert(g_aGstShadowInfo[enmGstReg].cb != 0);
    2244 
    2245     switch (g_aGstShadowInfo[enmGstReg].cb)
    2246     {
    2247         case sizeof(uint64_t):
    2248             return iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
    2249         case sizeof(uint32_t):
    2250             return iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
    2251         case sizeof(uint16_t):
    2252             return iemNativeEmitLoadGprFromVCpuU16(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
    2253 #if 0 /* not present in the table. */
    2254         case sizeof(uint8_t):
    2255             return iemNativeEmitLoadGprFromVCpuU8(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
    2256 #endif
    2257         default:
    2258             AssertFailedReturn(UINT32_MAX);
    2259     }
    2260 }
    2261 
    2262 
    2263 #ifdef VBOX_STRICT
    2264 /**
    2265  * Emitting code that checks that the content of register @a idxReg is the same
    2266  * as what's in the guest register @a enmGstReg, resulting in a breakpoint
    2267  * instruction if that's not the case.
    2268  *
    2269  * @note May of course trash IEMNATIVE_REG_FIXED_TMP0.
    2270  *       Trashes EFLAGS on AMD64.
    2271  */
    2272 static uint32_t iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    2273                                                 uint8_t idxReg, IEMNATIVEGSTREG enmGstReg)
    2274 {
    2275 # ifdef RT_ARCH_AMD64
    2276     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
    2277     AssertReturn(pbCodeBuf, UINT32_MAX);
    2278 
    2279     /* cmp reg, [mem] */
    2280     if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint8_t))
    2281     {
    2282         if (idxReg >= 8)
    2283             pbCodeBuf[off++] = X86_OP_REX_R;
    2284         pbCodeBuf[off++] = 0x38;
    2285     }
     1979
     1980DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
     1981                                                     IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
     1982{
     1983    pReNative->bmHstRegs |= RT_BIT_32(idxReg);
     1984
     1985    pReNative->aHstRegs[idxReg].enmWhat        = enmWhat;
     1986    pReNative->aHstRegs[idxReg].fGstRegShadows = 0;
     1987    pReNative->aHstRegs[idxReg].idxVar         = idxVar;
     1988    return (uint8_t)idxReg;
     1989}
     1990
     1991
     1992/**
     1993 * Locate a register, possibly freeing one up.
     1994 *
     1995 * This ASSUMES the caller has done the minimal/optimal allocation checks and
     1996 * failed.
     1997 */
     1998static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fAllowVolatile) RT_NOEXCEPT
     1999{
     2000    uint32_t fRegMask = fAllowVolatile
     2001                      ? IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK
     2002                      : IEMNATIVE_HST_GREG_MASK & ~(IEMNATIVE_REG_FIXED_MASK | IEMNATIVE_CALL_VOLATILE_GREG_MASK);
     2003
     2004    /*
     2005     * Try a freed register that's shadowing a guest register
     2006     */
     2007    uint32_t fRegs = ~pReNative->bmHstRegs & fRegMask;
     2008    if (fRegs)
     2009    {
     2010        /** @todo pick better here:    */
     2011        unsigned const idxReg = ASMBitFirstSetU32(fRegs) - 1;
     2012
     2013        Assert(pReNative->aHstRegs[idxReg].fGstRegShadows != 0);
     2014        Assert(   (pReNative->aHstRegs[idxReg].fGstRegShadows & pReNative->bmGstRegShadows)
     2015               == pReNative->aHstRegs[idxReg].fGstRegShadows);
     2016        Assert(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg));
     2017
     2018        pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxReg].fGstRegShadows;
     2019        pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
     2020        pReNative->aHstRegs[idxReg].fGstRegShadows = 0;
     2021        return idxReg;
     2022    }
     2023
     2024    /*
     2025     * Try free up a variable that's in a register.
     2026     *
     2027     * We do two rounds here, first evacuating variables we don't need to be
     2028     * saved on the stack, then in the second round move things to the stack.
     2029     */
     2030    for (uint32_t iLoop = 0; iLoop < 2; iLoop++)
     2031    {
     2032        uint32_t fVars = pReNative->bmVars;
     2033        while (fVars)
     2034        {
     2035            uint32_t const idxVar = ASMBitFirstSetU32(fVars) - 1;
     2036            uint8_t const  idxReg = pReNative->aVars[idxVar].idxReg;
     2037            if (   idxReg < RT_ELEMENTS(pReNative->aHstRegs)
     2038                && (RT_BIT_32(idxReg) & fRegMask)
     2039                && (  iLoop == 0
     2040                    ? pReNative->aVars[idxVar].enmKind != kIemNativeVarKind_Stack
     2041                    : pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack))
     2042            {
     2043                Assert(pReNative->bmHstRegs & RT_BIT_32(idxReg));
     2044                Assert(   (pReNative->bmGstRegShadows & pReNative->aHstRegs[idxReg].fGstRegShadows)
     2045                       == pReNative->aHstRegs[idxReg].fGstRegShadows);
     2046                Assert(   RT_BOOL(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg))
     2047                       == RT_BOOL(pReNative->aHstRegs[idxReg].fGstRegShadows));
     2048
     2049                if (pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     2050                {
     2051                    AssertReturn(pReNative->aVars[idxVar].idxStackSlot != UINT8_MAX, UINT8_MAX);
     2052                    uint32_t off = *poff;
     2053                    *poff = off = iemNativeEmitStoreGprByBp(pReNative, off,
     2054                                                              pReNative->aVars[idxVar].idxStackSlot * sizeof(uint64_t)
     2055                                                            - IEMNATIVE_FP_OFF_STACK_VARS,
     2056                                                            idxReg);
     2057                    AssertReturn(off != UINT32_MAX, UINT8_MAX);
     2058                }
     2059
     2060                pReNative->aVars[idxVar].idxReg    = UINT8_MAX;
     2061                pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxReg].fGstRegShadows;
     2062                pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
     2063                pReNative->bmHstRegs              &= ~RT_BIT_32(idxReg);
     2064                return idxReg;
     2065            }
     2066            fVars &= ~RT_BIT_32(idxVar);
     2067        }
     2068    }
     2069
     2070    AssertFailedReturn(UINT8_MAX);
     2071}
     2072
     2073
     2074/**
     2075 * Moves a variable to a different register or spills it onto the stack.
     2076 *
     2077 * This must be a stack variable (kIemNativeVarKind_Stack) because the other
     2078 * kinds can easily be recreated if needed later.
     2079 *
     2080 * @returns The new code buffer position, UINT32_MAX on failure.
     2081 * @param   pReNative       The native recompile state.
     2082 * @param   off             The current code buffer position.
     2083 * @param   idxVar          The variable index.
     2084 * @param   fForbiddenRegs  Mask of the forbidden registers.  Defaults to
     2085 *                          call-volatile registers.
     2086 */
     2087static uint32_t iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
     2088                                                uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK)
     2089{
     2090    Assert(idxVar < RT_ELEMENTS(pReNative->aVars));
     2091    Assert(pReNative->aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
     2092
     2093    uint8_t const idxRegOld = pReNative->aVars[idxVar].idxReg;
     2094    Assert(idxRegOld < RT_ELEMENTS(pReNative->aHstRegs));
     2095    Assert(pReNative->bmHstRegs & RT_BIT_32(idxRegOld));
     2096    Assert(pReNative->aHstRegs[idxRegOld].enmWhat == kIemNativeWhat_Var);
     2097    Assert(   (pReNative->bmGstRegShadows & pReNative->aHstRegs[idxRegOld].fGstRegShadows)
     2098           == pReNative->aHstRegs[idxRegOld].fGstRegShadows);
     2099    Assert(   RT_BOOL(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxRegOld))
     2100           == RT_BOOL(pReNative->aHstRegs[idxRegOld].fGstRegShadows));
     2101
     2102
     2103    /** @todo Add statistics on this.*/
     2104    /** @todo Implement basic variable liveness analysis (python) so variables
     2105     * can be freed immediately once no longer used.  This has the potential to
     2106     * be trashing registers and stack for dead variables. */
     2107
     2108    /*
     2109     * First try move it to a different register, as that's cheaper.
     2110     */
     2111    fForbiddenRegs |= RT_BIT_32(idxRegOld);
     2112    fForbiddenRegs |= IEMNATIVE_REG_FIXED_MASK;
     2113    uint32_t fRegs = ~pReNative->bmHstRegs & ~fForbiddenRegs;
     2114    if (fRegs)
     2115    {
     2116        /* Avoid using shadow registers, if possible. */
     2117        if (fRegs & ~pReNative->bmHstRegsWithGstShadow)
     2118            fRegs &= ~pReNative->bmHstRegsWithGstShadow;
     2119        unsigned const idxRegNew = ASMBitFirstSetU32(fRegs) - 1;
     2120
     2121        uint64_t fGstRegShadows = pReNative->aHstRegs[idxRegOld].fGstRegShadows;
     2122        pReNative->aHstRegs[idxRegNew].fGstRegShadows = fGstRegShadows;
     2123        pReNative->aHstRegs[idxRegNew].enmWhat        = kIemNativeWhat_Var;
     2124        pReNative->aHstRegs[idxRegNew].idxVar         = idxVar;
     2125        if (fGstRegShadows)
     2126        {
     2127            pReNative->bmHstRegsWithGstShadow |= RT_BIT_32(idxRegNew);
     2128            while (fGstRegShadows)
     2129            {
     2130                unsigned const idxGstReg = ASMBitFirstSetU64(fGstRegShadows);
     2131                fGstRegShadows &= ~RT_BIT_64(idxGstReg);
     2132
     2133                Assert(pReNative->aidxGstRegShadows[idxGstReg] == idxRegOld);
     2134                pReNative->aidxGstRegShadows[idxGstReg] = idxRegNew;
     2135            }
     2136        }
     2137
     2138        pReNative->aVars[idxVar].idxReg = (uint8_t)idxRegNew;
     2139        pReNative->bmHstRegs           |= RT_BIT_32(idxRegNew);
     2140    }
     2141    /*
     2142     * Otherwise we must spill the register onto the stack.
     2143     */
    22862144    else
    22872145    {
    2288         if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint64_t))
    2289             pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_R);
     2146        AssertReturn(pReNative->aVars[idxVar].idxStackSlot != UINT8_MAX, UINT32_MAX);
     2147        off = iemNativeEmitStoreGprByBp(pReNative, off,
     2148                                        pReNative->aVars[idxVar].idxStackSlot * sizeof(uint64_t) - IEMNATIVE_FP_OFF_STACK_VARS,
     2149                                        idxRegOld);
     2150        AssertReturn(off != UINT32_MAX, UINT32_MAX);
     2151
     2152        pReNative->bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
     2153        pReNative->bmGstRegShadows        &= ~pReNative->aHstRegs[idxRegOld].fGstRegShadows;
     2154    }
     2155
     2156    pReNative->bmHstRegs &= ~RT_BIT_32(idxRegOld);
     2157    pReNative->aHstRegs[idxRegOld].fGstRegShadows = 0;
     2158    return off;
     2159}
     2160
     2161
     2162/**
     2163 * Allocates a temporary host general purpose register.
     2164 *
     2165 * This may emit code to save register content onto the stack in order to free
     2166 * up a register.
     2167 *
     2168 * @returns The host register number, UINT8_MAX on failure.
     2169 * @param   pReNative       The native recompile state.
     2170 * @param   poff            Pointer to the variable with the code buffer position.
     2171 *                          This will be update if we need to move a variable from
     2172 *                          register to stack in order to satisfy the request.
     2173 * @param   fPreferVolatile Wheter to prefer volatile over non-volatile
     2174 *                          registers (@c true, default) or the other way around
     2175 *                          (@c false, for iemNativeRegAllocTmpForGuestReg()).
     2176 */
     2177DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
     2178                                         bool fPreferVolatile /*= true*/) RT_NOEXCEPT
     2179{
     2180    /*
     2181     * Try find a completely unused register, preferably a call-volatile one.
     2182     */
     2183    uint8_t  idxReg;
     2184    uint32_t fRegs = ~pReNative->bmHstRegs
     2185                   & ~pReNative->bmHstRegsWithGstShadow
     2186                   & (~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK);
     2187    if (fRegs)
     2188    {
     2189        if (fPreferVolatile)
     2190            idxReg = (uint8_t)ASMBitFirstSetU32(  fRegs & IEMNATIVE_CALL_VOLATILE_GREG_MASK
     2191                                                ? fRegs & IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
    22902192        else
    2291         {
    2292             if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint16_t))
    2293                 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
    2294             else
    2295                 AssertReturn(g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t), UINT32_MAX);
    2296             if (idxReg >= 8)
    2297                 pbCodeBuf[off++] = X86_OP_REX_R;
    2298         }
    2299         pbCodeBuf[off++] = 0x39;
    2300     }
    2301     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg, g_aGstShadowInfo[enmGstReg].off);
    2302 
    2303     /* je/jz +1 */
    2304     pbCodeBuf[off++] = 0x74;
    2305     pbCodeBuf[off++] = 0x01;
    2306 
    2307     /* int3 */
    2308     pbCodeBuf[off++] = 0xcc;
    2309 
    2310     /* For values smaller than the register size, we must check that the rest
    2311        of the register is all zeros. */
    2312     if (g_aGstShadowInfo[enmGstReg].cb < sizeof(uint32_t))
    2313     {
    2314         /* test reg64, imm32 */
    2315         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
    2316         pbCodeBuf[off++] = 0xf7;
    2317         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
    2318         pbCodeBuf[off++] = 0;
    2319         pbCodeBuf[off++] = g_aGstShadowInfo[enmGstReg].cb > sizeof(uint8_t) ? 0 : 0xff;
    2320         pbCodeBuf[off++] = 0xff;
    2321         pbCodeBuf[off++] = 0xff;
    2322 
    2323         /* je/jz +1 */
    2324         pbCodeBuf[off++] = 0x74;
    2325         pbCodeBuf[off++] = 0x01;
    2326 
    2327         /* int3 */
    2328         pbCodeBuf[off++] = 0xcc;
    2329     }
    2330     else if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t))
    2331     {
    2332         /* rol reg64, 32 */
    2333         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
    2334         pbCodeBuf[off++] = 0xc1;
    2335         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
    2336         pbCodeBuf[off++] = 32;
    2337 
    2338         /* test reg32, ffffffffh */
    2339         if (idxReg >= 8)
    2340             pbCodeBuf[off++] = X86_OP_REX_B;
    2341         pbCodeBuf[off++] = 0xf7;
    2342         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
    2343         pbCodeBuf[off++] = 0xff;
    2344         pbCodeBuf[off++] = 0xff;
    2345         pbCodeBuf[off++] = 0xff;
    2346         pbCodeBuf[off++] = 0xff;
    2347 
    2348         /* je/jz +1 */
    2349         pbCodeBuf[off++] = 0x74;
    2350         pbCodeBuf[off++] = 0x01;
    2351 
    2352         /* int3 */
    2353         pbCodeBuf[off++] = 0xcc;
    2354 
    2355         /* rol reg64, 32 */
    2356         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
    2357         pbCodeBuf[off++] = 0xc1;
    2358         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
    2359         pbCodeBuf[off++] = 32;
    2360     }
    2361 
    2362 # elif defined(RT_ARCH_ARM64)
    2363     /* mov TMP0, [gstreg] */
    2364     off = iemNativeEmitLoadGprWithGstShadowReg(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, enmGstReg);
    2365 
    2366     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    2367     AssertReturn(pu32CodeBuf, UINT32_MAX);
    2368     /* sub tmp0, tmp0, idxReg */
    2369     pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, IEMNATIVE_REG_FIXED_TMP0, IEMNATIVE_REG_FIXED_TMP0, idxReg);
    2370     /* cbz tmp0, +1 */
    2371     pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 1, IEMNATIVE_REG_FIXED_TMP0);
    2372     /* brk #0x1000+enmGstReg */
    2373     pu32CodeBuf[off++] = Armv8A64MkInstrBrk((uint32_t)enmGstReg | UINT32_C(0x1000));
    2374 
    2375 # else
    2376 #  error "Port me!"
    2377 # endif
    2378     return off;
    2379 }
    2380 #endif /* VBOX_STRICT */
     2193            idxReg = (uint8_t)ASMBitFirstSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
     2194                                                ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
     2195        Assert(pReNative->aHstRegs[idxReg].fGstRegShadows == 0);
     2196        Assert(!(pReNative->bmHstRegsWithGstShadow & RT_BIT_32(idxReg)));
     2197    }
     2198    else
     2199    {
     2200        idxReg = iemNativeRegAllocFindFree(pReNative, poff, true /*fAllowVolatile*/);
     2201        AssertReturn(idxReg != UINT8_MAX, UINT8_MAX);
     2202    }
     2203    return iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Tmp);
     2204}
     2205
     2206
     2207/**
     2208 * Allocates a temporary register for loading an immediate value into.
     2209 *
     2210 * This will emit code to load the immediate, unless there happens to be an
     2211 * unused register with the value already loaded.
     2212 *
     2213 * The caller will not modify the returned register, it must be considered
     2214 * read-only.  Free using iemNativeRegFreeTmpImm.
     2215 *
     2216 * @returns The host register number, UINT8_MAX on failure.
     2217 * @param   pReNative       The native recompile state.
     2218 * @param   poff            Pointer to the variable with the code buffer position.
     2219 * @param   uImm            The immediate value that the register must hold upon
     2220 *                          return.
     2221 * @param   fPreferVolatile Wheter to prefer volatile over non-volatile
     2222 *                          registers (@c true, default) or the other way around
     2223 *                          (@c false).
     2224 *
     2225 * @note    Reusing immediate values has not been implemented yet.
     2226 */
     2227DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
     2228                                            bool fPreferVolatile /*= true*/) RT_NOEXCEPT
     2229{
     2230    uint8_t idxReg = iemNativeRegAllocTmp(pReNative, poff, fPreferVolatile);
     2231    if (idxReg < RT_ELEMENTS(pReNative->aHstRegs))
     2232    {
     2233        uint32_t off = *poff;
     2234        *poff = off = iemNativeEmitLoadGprImm64(pReNative, off, idxReg, uImm);
     2235        AssertReturnStmt(off != UINT32_MAX, iemNativeRegFreeTmp(pReNative, idxReg), UINT8_MAX);
     2236    }
     2237    return idxReg;
     2238}
    23812239
    23822240
    … …  
    29462804
    29472805
     2806/*********************************************************************************************************************************
     2807*   Code Emitters (larger snippets)                                                                                              *
     2808*********************************************************************************************************************************/
     2809
     2810/**
     2811 * Loads the guest shadow register @a enmGstReg into host reg @a idxHstReg, zero
     2812 * extending to 64-bit width.
     2813 *
     2814 * @returns New code buffer offset on success, UINT32_MAX on failure.
     2815 * @param   pReNative   .
     2816 * @param   off         The current code buffer position.
     2817 * @param   idxHstReg   The host register to load the guest register value into.
     2818 * @param   enmGstReg   The guest register to load.
     2819 *
     2820 * @note This does not mark @a idxHstReg as having a shadow copy of @a enmGstReg,
     2821 *       that is something the caller needs to do if applicable.
     2822 */
     2823DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     2824                                                          uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT
     2825{
     2826    Assert((unsigned)enmGstReg < RT_ELEMENTS(g_aGstShadowInfo));
     2827    Assert(g_aGstShadowInfo[enmGstReg].cb != 0);
     2828
     2829    switch (g_aGstShadowInfo[enmGstReg].cb)
     2830    {
     2831        case sizeof(uint64_t):
     2832            return iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
     2833        case sizeof(uint32_t):
     2834            return iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
     2835        case sizeof(uint16_t):
     2836            return iemNativeEmitLoadGprFromVCpuU16(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
     2837#if 0 /* not present in the table. */
     2838        case sizeof(uint8_t):
     2839            return iemNativeEmitLoadGprFromVCpuU8(pReNative, off, idxHstReg, g_aGstShadowInfo[enmGstReg].off);
     2840#endif
     2841        default:
     2842            AssertFailedReturn(UINT32_MAX);
     2843    }
     2844}
     2845
     2846
     2847#ifdef VBOX_STRICT
     2848/**
     2849 * Emitting code that checks that the content of register @a idxReg is the same
     2850 * as what's in the guest register @a enmGstReg, resulting in a breakpoint
     2851 * instruction if that's not the case.
     2852 *
     2853 * @note May of course trash IEMNATIVE_REG_FIXED_TMP0.
     2854 *       Trashes EFLAGS on AMD64.
     2855 */
     2856static uint32_t iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     2857                                                uint8_t idxReg, IEMNATIVEGSTREG enmGstReg)
     2858{
     2859# ifdef RT_ARCH_AMD64
     2860    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
     2861    AssertReturn(pbCodeBuf, UINT32_MAX);
     2862
     2863    /* cmp reg, [mem] */
     2864    if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint8_t))
     2865    {
     2866        if (idxReg >= 8)
     2867            pbCodeBuf[off++] = X86_OP_REX_R;
     2868        pbCodeBuf[off++] = 0x38;
     2869    }
     2870    else
     2871    {
     2872        if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint64_t))
     2873            pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_R);
     2874        else
     2875        {
     2876            if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint16_t))
     2877                pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     2878            else
     2879                AssertReturn(g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t), UINT32_MAX);
     2880            if (idxReg >= 8)
     2881                pbCodeBuf[off++] = X86_OP_REX_R;
     2882        }
     2883        pbCodeBuf[off++] = 0x39;
     2884    }
     2885    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg, g_aGstShadowInfo[enmGstReg].off);
     2886
     2887    /* je/jz +1 */
     2888    pbCodeBuf[off++] = 0x74;
     2889    pbCodeBuf[off++] = 0x01;
     2890
     2891    /* int3 */
     2892    pbCodeBuf[off++] = 0xcc;
     2893
     2894    /* For values smaller than the register size, we must check that the rest
     2895       of the register is all zeros. */
     2896    if (g_aGstShadowInfo[enmGstReg].cb < sizeof(uint32_t))
     2897    {
     2898        /* test reg64, imm32 */
     2899        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
     2900        pbCodeBuf[off++] = 0xf7;
     2901        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
     2902        pbCodeBuf[off++] = 0;
     2903        pbCodeBuf[off++] = g_aGstShadowInfo[enmGstReg].cb > sizeof(uint8_t) ? 0 : 0xff;
     2904        pbCodeBuf[off++] = 0xff;
     2905        pbCodeBuf[off++] = 0xff;
     2906
     2907        /* je/jz +1 */
     2908        pbCodeBuf[off++] = 0x74;
     2909        pbCodeBuf[off++] = 0x01;
     2910
     2911        /* int3 */
     2912        pbCodeBuf[off++] = 0xcc;
     2913    }
     2914    else if (g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t))
     2915    {
     2916        /* rol reg64, 32 */
     2917        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
     2918        pbCodeBuf[off++] = 0xc1;
     2919        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
     2920        pbCodeBuf[off++] = 32;
     2921
     2922        /* test reg32, ffffffffh */
     2923        if (idxReg >= 8)
     2924            pbCodeBuf[off++] = X86_OP_REX_B;
     2925        pbCodeBuf[off++] = 0xf7;
     2926        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
     2927        pbCodeBuf[off++] = 0xff;
     2928        pbCodeBuf[off++] = 0xff;
     2929        pbCodeBuf[off++] = 0xff;
     2930        pbCodeBuf[off++] = 0xff;
     2931
     2932        /* je/jz +1 */
     2933        pbCodeBuf[off++] = 0x74;
     2934        pbCodeBuf[off++] = 0x01;
     2935
     2936        /* int3 */
     2937        pbCodeBuf[off++] = 0xcc;
     2938
     2939        /* rol reg64, 32 */
     2940        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg < 8 ? 0 : X86_OP_REX_B);
     2941        pbCodeBuf[off++] = 0xc1;
     2942        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg & 7);
     2943        pbCodeBuf[off++] = 32;
     2944    }
     2945
     2946# elif defined(RT_ARCH_ARM64)
     2947    /* mov TMP0, [gstreg] */
     2948    off = iemNativeEmitLoadGprWithGstShadowReg(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, enmGstReg);
     2949
     2950    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
     2951    AssertReturn(pu32CodeBuf, UINT32_MAX);
     2952    /* sub tmp0, tmp0, idxReg */
     2953    pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, IEMNATIVE_REG_FIXED_TMP0, IEMNATIVE_REG_FIXED_TMP0, idxReg);
     2954    /* cbz tmp0, +1 */
     2955    pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 1, IEMNATIVE_REG_FIXED_TMP0);
     2956    /* brk #0x1000+enmGstReg */
     2957    pu32CodeBuf[off++] = Armv8A64MkInstrBrk((uint32_t)enmGstReg | UINT32_C(0x1000));
     2958
     2959# else
     2960#  error "Port me!"
     2961# endif
     2962    return off;
     2963}
     2964#endif /* VBOX_STRICT */
     2965
     2966
     2967
    29482968/**
    29492969 * Emits a code for checking the return code of a call and rcPassUp, returning
    … …  
    29602980    /* edx = rcPassUp */
    29612981    off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, X86_GREG_xDX, RT_UOFFSETOF(VMCPUCC, iem.s.rcPassUp));
    2962     AssertReturn(off != UINT32_MAX, UINT32_MAX);
    2963 
    2964     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
     2982# ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     2983    off = iemNativeEmitLoadGpr8Imm(pReNative, off, X86_GREG_xCX, idxInstr);
     2984# endif
     2985
     2986    /* edx = eax | rcPassUp */
     2987    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
    29652988    AssertReturn(pbCodeBuf, UINT32_MAX);
    2966 
    2967     /* edx = eax | rcPassUp */
    29682989    pbCodeBuf[off++] = 0x0b;                    /* or edx, eax */
    29692990    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, X86_GREG_xDX, X86_GREG_xAX);
    29702991
    2971     /* Jump to non-zero status return path, loading cl with the instruction number. */
    2972     pbCodeBuf[off++] = 0xb0 + X86_GREG_xCX;     /* mov cl, imm8 (pCallEntry->idxInstr) */
    2973     pbCodeBuf[off++] = idxInstr;
    2974 
    2975     pbCodeBuf[off++] = 0x0f;                    /* jnz rel32 */
    2976     pbCodeBuf[off++] = 0x85;
    2977     uint32_t const idxLabel = iemNativeMakeLabel(pReNative, kIemNativeLabelType_NonZeroRetOrPassUp);
    2978     AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
    2979     AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
    2980     pbCodeBuf[off++] = 0x00;
    2981     pbCodeBuf[off++] = 0x00;
    2982     pbCodeBuf[off++] = 0x00;
    2983     pbCodeBuf[off++] = 0x00;
     2992    /* Jump to non-zero status return path. */
     2993    off = iemNativeEmitJnzToNewLabel(pReNative, off, kIemNativeLabelType_NonZeroRetOrPassUp);
    29842994
    29852995    /* done. */
    … …  
    32613271        off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xCX, pCallEntry->auParams[2]);
    32623272# endif
    3263     off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, (uintptr_t)g_apfnIemThreadedFunctions[pCallEntry->enmFunction]);
    3264 
    3265     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
    3266     AssertReturn(pbCodeBuf, UINT32_MAX);
    3267     pbCodeBuf[off++] = 0xff;                    /* call rax */
    3268     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
     3273
     3274    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)g_apfnIemThreadedFunctions[pCallEntry->enmFunction]);
    32693275
    32703276# if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS)
    … …  
    32833289    if (cParams > 2)
    32843290        off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_ARG3_GREG, pCallEntry->auParams[2]);
    3285     off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0,
    3286                                     (uintptr_t)g_apfnIemThreadedFunctions[pCallEntry->enmFunction]);
    3287 
    3288     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    3289     AssertReturn(pu32CodeBuf, UINT32_MAX);
    3290 
    3291     pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
     3291
     3292    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)g_apfnIemThreadedFunctions[pCallEntry->enmFunction]);
    32923293
    32933294#else
    … …  
    33213322        off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, 0);
    33223323#endif
    3323 #ifdef RT_ARCH_AMD64
    3324         off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, (uintptr_t)iemNativeHlpExecRaiseGp0);
    3325         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    3326 
    3327         /* call rax */
    3328         uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
    3329         AssertReturn(pbCodeBuf, UINT32_MAX);
    3330         pbCodeBuf[off++] = 0xff;
    3331         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
    3332 
    3333 #elif defined(RT_ARCH_ARM64)
    3334         off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uintptr_t)iemNativeHlpExecRaiseGp0);
    3335         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    3336         pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
    3337 #else
    3338 # error "Port me"
    3339 #endif
     3324        off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseGp0);
    33403325
    33413326        /* jump back to the return sequence. */
    33423327        off = iemNativeEmitJmpToLabel(pReNative, off, iemNativeFindLabel(pReNative, kIemNativeLabelType_Return));
    3343 
    3344 #ifdef RT_ARCH_AMD64
    3345         /* int3 poison */
    3346         pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    3347         AssertReturn(pbCodeBuf, UINT32_MAX);
    3348         pbCodeBuf[off++] = 0xcc;
    3349 #endif
    33503328    }
    33513329    return off;
    … …  
    33773355        /* Call helper and jump to return point. */
    33783356# ifdef RT_OS_WINDOWS
     3357#  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    33793358        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_x8,  X86_GREG_xCX); /* cl = instruction number */
    3380         AssertReturn(off != UINT32_MAX, UINT32_MAX);
     3359#  endif
    33813360        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, IEMNATIVE_REG_FIXED_PVMCPU);
    3382         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    33833361        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xAX);
    3384         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    33853362# else
    33863363        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, IEMNATIVE_REG_FIXED_PVMCPU);
    3387         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    33883364        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xSI, X86_GREG_xAX);
    3389         AssertReturn(off != UINT32_MAX, UINT32_MAX);
     3365#  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    33903366        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xCX); /* cl = instruction number */
    3391         AssertReturn(off != UINT32_MAX, UINT32_MAX);
     3367#  endif
    33923368# endif
    3393         off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, (uintptr_t)iemNativeHlpExecStatusCodeFiddling);
    3394         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    3395 
    3396         pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
    3397         AssertReturn(pbCodeBuf, UINT32_MAX);
    3398         pbCodeBuf[off++] = 0xff;                    /* call rax */
    3399         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
    3400 
    3401         /* Jump to common return point. */
    3402         uint32_t offRel = pReNative->paLabels[idxReturnLabel].off - (off + 2);
    3403         if (-(int32_t)offRel <= 127)
    3404         {
    3405             pbCodeBuf[off++] = 0xeb;                /* jmp rel8 */
    3406             pbCodeBuf[off++] = (uint8_t)offRel;
    3407             off++;
    3408         }
    3409         else
    3410         {
    3411             offRel -= 3;
    3412             pbCodeBuf[off++] = 0xe9;                /* jmp rel32 */
    3413             pbCodeBuf[off++] = RT_BYTE1(offRel);
    3414             pbCodeBuf[off++] = RT_BYTE2(offRel);
    3415             pbCodeBuf[off++] = RT_BYTE3(offRel);
    3416             pbCodeBuf[off++] = RT_BYTE4(offRel);
    3417         }
    3418         pbCodeBuf[off++] = 0xcc;                    /* int3 poison */
     3369# ifndef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     3370        off = iemNativeEmitLoadGpr8Imm(pReNative, off, X86_GREG_xCX, 0);
     3371# endif
    34193372
    34203373#elif defined(RT_ARCH_ARM64)
    … …  
    34233376         */
    34243377        off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_CALL_RET_GREG);
    3425         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    34263378        off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    3427         AssertReturn(off != UINT32_MAX, UINT32_MAX);
    34283379        /* IEMNATIVE_CALL_ARG2_GREG is already set. */
    34293380        off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uintptr_t)iemNativeHlpExecStatusCodeFiddling);
    34303381        AssertReturn(off != UINT32_MAX, UINT32_MAX);
    3431 
    3432         uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
    3433         AssertReturn(pu32CodeBuf, UINT32_MAX);
    3434         pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
    3435 
    3436         /* Jump back to the common return point. */
    3437         int32_t const offRel = pReNative->paLabels[idxReturnLabel].off - off;
    3438         pu32CodeBuf[off++] = Armv8A64MkInstrB(offRel);
    34393382#else
    34403383# error "port me"
    34413384#endif
     3385
     3386        off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecStatusCodeFiddling);
     3387        off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    34423388    }
    34433389    return off;

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r101518 r101523  
    574574DECLHIDDEN(uint32_t)        iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
    575575
     576DECLHIDDEN(uint32_t)        iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     577                                                                 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
    576578DECLHIDDEN(uint32_t)        iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    577579                                                               uint8_t idxInstr) RT_NOEXCEPT;
    … …  
    16451647
    16461648/**
    1647  * Emits a JMP rel32 / B imm19 to the given label (ASSUMED requiring fixup).
     1649 * Emits a JMP rel32 / B imm19 to the given label.
    16481650 */
    16491651DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
    16501652{
    1651 #ifdef RT_ARCH_AMD64
    1652     /* jnz rel32 */
    1653     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
    1654     AssertReturn(pbCodeBuf, UINT32_MAX);
    1655     pbCodeBuf[off++] = 0xe9;
    1656     AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
    1657     pbCodeBuf[off++] = 0xfe;
    1658     pbCodeBuf[off++] = 0xff;
    1659     pbCodeBuf[off++] = 0xff;
    1660     pbCodeBuf[off++] = 0xff;
     1653    Assert(idxLabel < pReNative->cLabels);
     1654
     1655#ifdef RT_ARCH_AMD64
     1656    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
     1657    AssertReturn(pbCodeBuf, UINT32_MAX);
     1658    if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
     1659    {
     1660        uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
     1661        if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
     1662        {
     1663            pbCodeBuf[off++] = 0xeb;                /* jmp rel8 */
     1664            pbCodeBuf[off++] = (uint8_t)offRel;
     1665            off++;
     1666        }
     1667        else
     1668        {
     1669            offRel -= 3;
     1670            pbCodeBuf[off++] = 0xe9;                /* jmp rel32 */
     1671            pbCodeBuf[off++] = RT_BYTE1(offRel);
     1672            pbCodeBuf[off++] = RT_BYTE2(offRel);
     1673            pbCodeBuf[off++] = RT_BYTE3(offRel);
     1674            pbCodeBuf[off++] = RT_BYTE4(offRel);
     1675        }
     1676    }
     1677    else
     1678    {
     1679        pbCodeBuf[off++] = 0xe9;                    /* jmp rel32 */
     1680        AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
     1681        pbCodeBuf[off++] = 0xfe;
     1682        pbCodeBuf[off++] = 0xff;
     1683        pbCodeBuf[off++] = 0xff;
     1684        pbCodeBuf[off++] = 0xff;
     1685    }
     1686    pbCodeBuf[off++] = 0xcc;                        /* int3 poison */
    16611687
    16621688#elif defined(RT_ARCH_ARM64)
    16631689    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    16641690    AssertReturn(pu32CodeBuf, UINT32_MAX);
    1665     AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
    1666     pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
     1691    if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
     1692        pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
     1693    else
     1694    {
     1695        AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
     1696        pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
     1697    }
    16671698
    16681699#else
    … …  
    17001731                                             uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
    17011732{
     1733    Assert(idxLabel < pReNative->cLabels);
     1734
    17021735#ifdef RT_ARCH_AMD64
    17031736    /* jcc rel32 */
    … …  
    19371970
    19381971
     1972/**
     1973 * Emits a call to a 64-bit address.
     1974 */
     1975DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
     1976{
     1977#ifdef RT_ARCH_AMD64
     1978    off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
     1979
     1980    /* call rax */
     1981    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
     1982    AssertReturn(pbCodeBuf, UINT32_MAX);
     1983    pbCodeBuf[off++] = 0xff;
     1984    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
     1985
     1986#elif defined(RT_ARCH_ARM64)
     1987    off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
     1988
     1989    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1990    AssertReturn(pu32CodeBuf, UINT32_MAX);
     1991    pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
     1992#else
     1993# error "port me"
     1994#endif
     1995    return off;
     1996}
     1997
     1998
    19391999
    19402000/** @} */
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette