VirtualBox

Changeset 101844 in vbox for trunk/src


Ignore:
Timestamp:
Nov 6, 2023 2:05:35 AM (16 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
159940
Message:

VMM/IEM: Native translation of IEM_MC_CALL_CIMPL_1_THREADED and IEM_MC_FETCH_GREG_U16, enabling recompilation of 16-bit indirect calls and ltr/lldt reg16 instructions. This also includes basic argument and variable handling. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py

    r101742 r101844  
    27982798    'IEM_MC_FETCH_FCW':                                          (McBlock.parseMcGeneric,           False, False, ),
    27992799    'IEM_MC_FETCH_FSW':                                          (McBlock.parseMcGeneric,           False, False, ),
    2800     'IEM_MC_FETCH_GREG_U16':                                     (McBlock.parseMcGeneric,           False, False, ),
     2800    'IEM_MC_FETCH_GREG_U16':                                     (McBlock.parseMcGeneric,           False, True, ),
    28012801    'IEM_MC_FETCH_GREG_U16_SX_U32':                              (McBlock.parseMcGeneric,           False, False, ),
    28022802    'IEM_MC_FETCH_GREG_U16_SX_U64':                              (McBlock.parseMcGeneric,           False, False, ),

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py

    r101722 r101844  
    6969    'IEM_MC_CALC_RM_EFF_ADDR_THREADED_64':                       (None, False, False, ),
    7070
    71     'IEM_MC_CALL_CIMPL_1_THREADED':                              (None, True,  False, ),
     71    'IEM_MC_CALL_CIMPL_1_THREADED':                              (None, True,  True, ),
    7272    'IEM_MC_CALL_CIMPL_2_THREADED':                              (None, True,  False, ),
    7373    'IEM_MC_CALL_CIMPL_3_THREADED':                              (None, True,  False, ),

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r101706 r101844  
    16041604    pReNative->Core.bmGstRegShadows        = 0;
    16051605    pReNative->Core.bmVars                 = 0;
     1606    pReNative->Core.bmStack                = 0;
     1607    AssertCompile(sizeof(pReNative->Core.bmStack) * 8 == IEMNATIVE_FRAME_VAR_SLOTS); /* Must set reserved slots to 1 otherwise. */
    16061608    pReNative->Core.u64ArgVars             = UINT64_MAX;
    16071609
    … …  
    20922094#endif
    20932095};
     2096
     2097#ifdef IEMNATIVE_FP_OFF_STACK_ARG0
     2098/**
     2099 * BP offset of the stack argument slots.
     2100 *
     2101 * This array is indexed by \#argument - IEMNATIVE_CALL_ARG_GREG_COUNT and has
     2102 * IEMNATIVE_FRAME_STACK_ARG_COUNT entries.
     2103 */
     2104DECL_HIDDEN_CONST(int32_t) const g_aoffIemNativeCallStackArgBpDisp[] =
     2105{
     2106    IEMNATIVE_FP_OFF_STACK_ARG0,
     2107# ifdef IEMNATIVE_FP_OFF_STACK_ARG1
     2108    IEMNATIVE_FP_OFF_STACK_ARG1,
     2109# endif
     2110# ifdef IEMNATIVE_FP_OFF_STACK_ARG2
     2111    IEMNATIVE_FP_OFF_STACK_ARG2,
     2112# endif
     2113# ifdef IEMNATIVE_FP_OFF_STACK_ARG3
     2114    IEMNATIVE_FP_OFF_STACK_ARG3,
     2115# endif
     2116};
     2117AssertCompile(RT_ELEMENTS(g_aoffIemNativeCallStackArgBpDisp) == IEMNATIVE_FRAME_STACK_ARG_COUNT);
     2118#endif /* IEMNATIVE_FP_OFF_STACK_ARG0 */
    20942119
    20952120/**
    … …  
    21822207
    21832208/**
    2184  * Locate a register, possibly freeing one up.
     2209 * Tries to locate a suitable register in the given register mask.
    21852210 *
    21862211 * This ASSUMES the caller has done the minimal/optimal allocation checks and
    21872212 * failed.
    21882213 *
    2189  * @returns Host register number on success; throws VBox status code on failure, so no
    2190  *          need to check the return value.
    2191  */
    2192 static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fAllowVolatile)
    2193 {
    2194     uint32_t fRegMask = fAllowVolatile
    2195                       ? IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK
    2196                       : IEMNATIVE_HST_GREG_MASK & ~(IEMNATIVE_REG_FIXED_MASK | IEMNATIVE_CALL_VOLATILE_GREG_MASK);
    2197 
    2198     /*
    2199      * Try a freed register that's shadowing a guest register
    2200      */
     2214 * @returns Host register number on success, returns UINT8_MAX on failure.
     2215 */
     2216static uint8_t iemNativeRegTryAllocFree(PIEMRECOMPILERSTATE pReNative, uint32_t fRegMask)
     2217{
     2218    Assert(!(fRegMask & ~IEMNATIVE_HST_GREG_MASK));
    22012219    uint32_t fRegs = ~pReNative->Core.bmHstRegs & fRegMask;
    22022220    if (fRegs)
    … …  
    22042222        /** @todo pick better here:    */
    22052223        unsigned const idxReg = ASMBitFirstSetU32(fRegs) - 1;
     2224
     2225        Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows != 0);
     2226        Assert(   (pReNative->Core.aHstRegs[idxReg].fGstRegShadows & pReNative->Core.bmGstRegShadows)
     2227               == pReNative->Core.aHstRegs[idxReg].fGstRegShadows);
     2228        Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg));
     2229
     2230        pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
     2231        pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
     2232        pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
     2233        return idxReg;
     2234    }
     2235    return UINT8_MAX;
     2236}
     2237
     2238
     2239/**
     2240 * Locate a register, possibly freeing one up.
     2241 *
     2242 * This ASSUMES the caller has done the minimal/optimal allocation checks and
     2243 * failed.
     2244 *
     2245 * @returns Host register number on success. Returns UINT8_MAX if no registers
     2246 *          found, the caller is supposed to deal with this and raise a
     2247 *          allocation type specific status code (if desired).
     2248 *
     2249 * @throws  VBox status code if we're run into trouble spilling a variable of
     2250 *          recording debug info.  Does NOT throw anything if we're out of
     2251 *          registers, though.
     2252 */
     2253static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile,
     2254                                         uint32_t fRegMask = IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK)
     2255{
     2256    Assert(!(fRegMask & ~IEMNATIVE_HST_GREG_MASK));
     2257    Assert(!(fRegMask & ~IEMNATIVE_REG_FIXED_MASK));
     2258
     2259    /*
     2260     * Try a freed register that's shadowing a guest register
     2261     */
     2262    uint32_t fRegs = ~pReNative->Core.bmHstRegs & fRegMask;
     2263    if (fRegs)
     2264    {
     2265        unsigned const idxReg = (fPreferVolatile
     2266                                 ? ASMBitFirstSetU32(fRegs)
     2267                                 : ASMBitLastSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
     2268                                                    ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK: fRegs))
     2269                              - 1;
    22062270
    22072271        Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows != 0);
    … …  
    22612325    }
    22622326
    2263     AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_OUT_OF_REGISTERS));
     2327    return UINT8_MAX;
    22642328}
    22652329
    … …  
    23912455    else
    23922456    {
    2393         idxReg = iemNativeRegAllocFindFree(pReNative, poff, true /*fAllowVolatile*/);
     2457        idxReg = iemNativeRegAllocFindFree(pReNative, poff, fPreferVolatile);
    23942458        AssertStmt(idxReg != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_ALLOCATOR_NO_FREE_TMP));
    23952459    }
    … …  
    28742938 *                          been allocated as such already and won't need moving,
    28752939 *                          just freeing.
    2876  * @param   fFreeArgVars    Whether to free argument variables for the call.
    28772940 */
    28782941DECL_HIDDEN_THROW(uint32_t)
    2879 iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, bool fFreeArgVars)
    2880 {
    2881     /*
    2882      * Free argument variables first (simplified).
    2883      */
    2884     AssertStmt(cArgs <= RT_ELEMENTS(pReNative->Core.aidxArgVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6));
    2885     if (fFreeArgVars && cArgs > 0)
    2886     {
    2887         for (uint32_t i = 0; i < cArgs; i++)
    2888         {
    2889             uint8_t idxVar = pReNative->Core.aidxArgVars[i];
    2890             if (idxVar < RT_ELEMENTS(pReNative->Core.aVars))
    2891             {
    2892                 pReNative->Core.aidxArgVars[i] = UINT8_MAX;
    2893                 pReNative->Core.bmVars        &= ~RT_BIT_32(idxVar);
    2894                 Assert(   pReNative->Core.aVars[idxVar].idxReg
    2895                        == (i < RT_ELEMENTS(g_aidxIemNativeCallRegs) ? g_aidxIemNativeCallRegs[i] : UINT8_MAX));
    2896             }
    2897         }
    2898         Assert(pReNative->Core.u64ArgVars == UINT64_MAX);
    2899     }
     2942iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs)
     2943{
     2944    Assert(cArgs <= IEMNATIVE_CALL_MAX_ARG_COUNT);
    29002945
    29012946    /*
    … …  
    31293174            else
    31303175                AssertStmt(g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t),
    3131                            IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_7));
     3176                           IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6));
    31323177            if (idxReg >= 8)
    31333178                pbCodeBuf[off++] = X86_OP_REX_R;
    … …  
    33613406 * idxSegReg, raising a \#GP(0) if it isn't.
    33623407 *
    3363  * @returns New code buffer offset, UINT32_MAX on failure.
     3408 * @returns New code buffer offset; throws VBox status code on error.
    33643409 * @param   pReNative       The native recompile state.
    33653410 * @param   off             The code buffer offset.
    … …  
    34133458{
    34143459    iemNativeRegFlushGuestShadows(pReNative, UINT64_MAX); /** @todo optimize this */
    3415     off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4, false /*fFreeArgVars*/);
     3460    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4);
    34163461
    34173462    /*
    … …  
    34683513{
    34693514    iemNativeRegFlushGuestShadows(pReNative, UINT64_MAX); /** @todo optimize this */
    3470     off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4, false /*fFreeArgVars*/);
     3515    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4);
    34713516    uint8_t const cParams = g_acIemThreadedFunctionUsedArgs[pCallEntry->enmFunction];
    34723517
    … …  
    38403885
    38413886
     3887
    38423888/*********************************************************************************************************************************
    3843 *   Emitters for IEM_MC_XXXX and the associated IEM_MC_XXXX recompiler definitions                                               *
     3889*   Emitters for IEM_MC_BEGIN and IEM_MC_END.                                                                                    *
    38443890*********************************************************************************************************************************/
    38453891
    38463892#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) \
    3847     {
     3893    { \
     3894        pReNative->fMc    = (a_fMcFlags); \
     3895        pReNative->fCImpl = (a_fCImplFlags); \
     3896        pReNative->cArgs  = ((a_cArgs) + iemNativeArgGetHiddenArgCount(pReNative))
    38483897
    38493898/** We have to get to the end in recompilation mode, as otherwise we won't
    … …  
    38533902
    38543903
    3855 /*
    3856  * Standalone CImpl deferals.
    3857  */
     3904
     3905/*********************************************************************************************************************************
     3906*   Emitters for standalone C-implementation deferals (IEM_MC_DEFER_TO_CIMPL_XXXX)                                               *
     3907*********************************************************************************************************************************/
    38583908
    38593909#define IEM_MC_DEFER_TO_CIMPL_0_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl) \
     3910    pReNative->fMc    = 0; \
     3911    pReNative->fCImpl = (a_fFlags); \
    38603912    return iemNativeEmitCImplCall0(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr) /** @todo not used ... */
    38613913
    38623914
    38633915#define IEM_MC_DEFER_TO_CIMPL_1_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0) \
     3916    pReNative->fMc    = 0; \
     3917    pReNative->fCImpl = (a_fFlags); \
    38643918    return iemNativeEmitCImplCall1(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0)
    38653919
    … …  
    38723926
    38733927#define IEM_MC_DEFER_TO_CIMPL_2_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1) \
     3928    pReNative->fMc    = 0; \
     3929    pReNative->fCImpl = (a_fFlags); \
    38743930    return iemNativeEmitCImplCall2(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1)
    38753931
    … …  
    38823938
    38833939#define IEM_MC_DEFER_TO_CIMPL_3_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2) \
     3940    pReNative->fMc    = 0; \
     3941    pReNative->fCImpl = (a_fFlags); \
    38843942    return iemNativeEmitCImplCall3(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1, a2)
    38853943
    … …  
    38923950
    38933951
    3894 /*
    3895  * Advancing PC/RIP/EIP/IP.
    3896  */
     3952
     3953/*********************************************************************************************************************************
     3954*   Emitters for advancing PC/RIP/EIP/IP (IEM_MC_ADVANCE_RIP_AND_FINISH_XXX)                                                     *
     3955*********************************************************************************************************************************/
    38973956
    38983957/** Emits the flags check for IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC64_WITH_FLAGS
    … …  
    40074066
    40084067
    4009 /*
    4010  * Changing PC/RIP/EIP/IP with a relative jump.
    4011  */
     4068
     4069/*********************************************************************************************************************************
     4070*   Emitters for changing PC/RIP/EIP/IP with a relative jump (IEM_MC_REL_JMP_XXX_AND_FINISH_XXX).                                *
     4071*********************************************************************************************************************************/
    40124072
    40134073#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC64(a_i8, a_cbInstr, a_enmEffOpSize) \
    … …  
    41764236
    41774237
    4178 /*
    4179  * Conditionals.
    4180  */
     4238
     4239/*********************************************************************************************************************************
     4240*   Emitters for conditionals (IEM_MC_IF_XXX, IEM_MC_ELSE, IEM_MC_ENDIF)                                                         *
     4241*********************************************************************************************************************************/
    41814242
    41824243/**
    … …  
    47914852
    47924853
    4793 /*
    4794  * General purpose register stores.
    4795  */
     4854
     4855/*********************************************************************************************************************************
     4856*   Emitters for IEM_MC_ARG_XXX, IEM_MC_LOCAL, IEM_MC_LOCAL_CONST, ++                                                            *
     4857*********************************************************************************************************************************/
     4858/** Number of hidden arguments for CIMPL calls.
     4859 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
     4860#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
     4861# define IEM_CIMPL_HIDDEN_ARGS 3
     4862#else
     4863# define IEM_CIMPL_HIDDEN_ARGS 2
     4864#endif
     4865
     4866#define IEM_MC_ARG(a_Type, a_Name, a_iArg) \
     4867    uint8_t const a_Name = iemNativeArgAlloc(pReNative, (a_iArg), sizeof(a_Type))
     4868
     4869#define IEM_MC_ARG_CONST(a_Type, a_Name, a_iArg) \
     4870    uint8_t const a_Name = iemNativeArgAllocConst(pReNative, (a_iArg), sizeof(a_Type), (a_Value))
     4871
     4872#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_iArg) \
     4873    uint8_t const a_Name = iemNativeArgAllocLocalRef(pReNative, (a_iArg), (a_Local))
     4874
     4875#define IEM_MC_LOCAL(a_Type, a_Name) \
     4876    uint8_t const a_Name = iemNativeVarAlloc(pReNative, sizeof(a_Type))
     4877
     4878#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) \
     4879    uint8_t const a_Name = iemNativeVarAllocConst(pReNative, sizeof(a_Type), (a_Value))
     4880
     4881
     4882/**
     4883 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
     4884 */
     4885DECLINLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
     4886{
     4887    if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
     4888        return IEM_CIMPL_HIDDEN_ARGS;
     4889    if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
     4890        return 1;
     4891    return 0;
     4892}
     4893
     4894
     4895/**
     4896 * Internal work that allocates a variable with kind set to
     4897 * kIemNativeVarKind_Invalid and no current stack allocation.
     4898 *
     4899 * The kind will either be set by the caller or later when the variable is first
     4900 * assigned a value.
     4901 */
     4902static uint8_t iemNativeVarAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
     4903{
     4904    Assert(cbType > 0 && cbType <= 64);
     4905    unsigned const idxVar = ASMBitFirstSetU32(~pReNative->Core.bmVars);
     4906    AssertStmt(idxVar < RT_ELEMENTS(pReNative->Core.aVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_EXHAUSTED));
     4907    pReNative->Core.bmVars |= RT_BIT_32(idxVar);
     4908    pReNative->Core.aVars[idxVar].enmKind        = kIemNativeVarKind_Invalid;
     4909    pReNative->Core.aVars[idxVar].cbVar          = cbType;
     4910    pReNative->Core.aVars[idxVar].idxStackSlot   = UINT8_MAX;
     4911    pReNative->Core.aVars[idxVar].idxReg         = UINT8_MAX;
     4912    pReNative->Core.aVars[idxVar].uArgNo         = UINT8_MAX;
     4913    pReNative->Core.aVars[idxVar].idxReferrerVar = UINT8_MAX;
     4914    pReNative->Core.aVars[idxVar].enmGstReg      = kIemNativeGstReg_End;
     4915    pReNative->Core.aVars[idxVar].u.uValue       = 0;
     4916    return idxVar;
     4917}
     4918
     4919
     4920/**
     4921 * Internal work that allocates an argument variable w/o setting enmKind.
     4922 */
     4923static uint8_t iemNativeArgAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
     4924{
     4925    iArgNo += iemNativeArgGetHiddenArgCount(pReNative);
     4926    AssertStmt(iArgNo < RT_ELEMENTS(pReNative->Core.aidxArgVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_1));
     4927    AssertStmt(pReNative->Core.aidxArgVars[iArgNo] == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_DUP_ARG_NO));
     4928
     4929    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
     4930    pReNative->Core.aidxArgVars[iArgNo]  = idxVar;
     4931    pReNative->Core.aVars[idxVar].uArgNo = iArgNo;
     4932    return idxVar;
     4933}
     4934
     4935
     4936/**
     4937 * Changes the variable to a stack variable.
     4938 *
     4939 * Currently this is s only possible to do the first time the variable is used,
     4940 * switching later is can be implemented but not done.
     4941 *
     4942 * @param   pReNative   The recompiler state.
     4943 * @param   idxVar      The variable.
     4944 * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
     4945 */
     4946static void iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
     4947{
     4948    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
     4949    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
     4950    {
     4951        /* We could in theory transition from immediate to stack as well, but it
     4952           would involve the caller doing work storing the value on the stack. So,
     4953           till that's required we only allow transition from invalid. */
     4954        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
     4955                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
     4956        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Stack;
     4957
     4958        if (pReNative->Core.aVars[idxVar].idxStackSlot == UINT8_MAX)
     4959        {
     4960            if (pReNative->Core.aVars[idxVar].cbVar <= sizeof(uint64_t))
     4961            {
     4962                unsigned const iSlot = ASMBitFirstSetU32(~pReNative->Core.bmStack);
     4963                AssertStmt(iSlot < IEMNATIVE_FRAME_VAR_SLOTS, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
     4964                pReNative->Core.bmStack |= RT_BIT_32(iSlot);
     4965                pReNative->Core.aVars[idxVar].idxStackSlot = iSlot;
     4966                return;
     4967            }
     4968            /* cbVar -> fBitAlignMask: 16 -> 1; 32 -> 3; 64 -> 7;*/
     4969            AssertCompile(RT_IS_POWER_OF_TWO(IEMNATIVE_FRAME_VAR_SLOTS)); /* If not we have to add an overflow check. */
     4970            Assert(pReNative->Core.aVars[idxVar].cbVar <= 64);
     4971            uint32_t const fBitAlignMask = RT_BIT_32(ASMBitLastSetU32(pReNative->Core.aVars[idxVar].cbVar) - 4) - 1;
     4972            uint32_t       fBitAllocMask = RT_BIT_32((pReNative->Core.aVars[idxVar].cbVar + 7) >> 3) - 1;
     4973            uint32_t       bmStack       = ~pReNative->Core.bmStack;
     4974            while (bmStack != UINT32_MAX)
     4975            {
     4976                unsigned const iSlot = ASMBitFirstSetU32(bmStack) - 1;
     4977                AssertStmt(iSlot < IEMNATIVE_FRAME_VAR_SLOTS, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
     4978                if (!(iSlot & fBitAlignMask))
     4979                {
     4980                    if ((bmStack & (fBitAllocMask << iSlot)) == (fBitAllocMask << iSlot))
     4981                    {
     4982                        pReNative->Core.bmStack |= (fBitAllocMask << iSlot);
     4983                        pReNative->Core.aVars[idxVar].idxStackSlot = iSlot;
     4984                        return;
     4985                    }
     4986                }
     4987                bmStack |= fBitAlignMask << (iSlot & ~fBitAlignMask);
     4988            }
     4989            AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
     4990        }
     4991    }
     4992}
     4993
     4994
     4995/**
     4996 * Changes it to a variable with a constant value.
     4997 *
     4998 * This does not require stack storage as we know the value and can always
     4999 * reload it, unless of course it's referenced.
     5000 *
     5001 * @param   pReNative   The recompiler state.
     5002 * @param   idxVar      The variable.
     5003 * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
     5004 */
     5005static void iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue)
     5006{
     5007    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
     5008    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Immediate)
     5009    {
     5010        /* Only simple trasnsitions for now. */
     5011        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
     5012                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
     5013        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Immediate;
     5014    }
     5015    pReNative->Core.aVars[idxVar].u.uValue = uValue;
     5016}
     5017
     5018
     5019/**
     5020 * Changes the variable to a reference (pointer) to @a idxOtherVar.
     5021 *
     5022 * @param   pReNative   The recompiler state.
     5023 * @param   idxVar      The variable.
     5024 * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
     5025 */
     5026static void iemNativeVarSetKindToLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint8_t idxOtherVar)
     5027{
     5028    Assert(idxVar      < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
     5029    Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxOtherVar)));
     5030
     5031    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_VarRef)
     5032    {
     5033        /* Only simple trasnsitions for now. */
     5034        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
     5035                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
     5036        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Immediate;
     5037    }
     5038    pReNative->Core.aVars[idxVar].u.idxRefVar = idxOtherVar;
     5039
     5040    /* Update the other variable, ensure it's a stack variable. */
     5041    /** @todo handle variables with const values... that's go boom now. */
     5042    pReNative->Core.aVars[idxOtherVar].idxReferrerVar = idxVar;
     5043    iemNativeVarSetKindToStack(pReNative, idxOtherVar);
     5044}
     5045
     5046
     5047DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
     5048{
     5049    return iemNativeArgAllocInt(pReNative, iArgNo, cbType);
     5050}
     5051
     5052
     5053DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue)
     5054{
     5055    uint8_t const idxVar = iemNativeArgAllocInt(pReNative, iArgNo, cbType);
     5056    iemNativeVarSetKindToConst(pReNative, idxVar, uValue);
     5057    return idxVar;
     5058}
     5059
     5060
     5061DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar)
     5062{
     5063    AssertStmt(   idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars)
     5064               && (pReNative->Core.bmVars & RT_BIT_32(idxOtherVar))
     5065               && pReNative->Core.aVars[idxOtherVar].uArgNo == UINT8_MAX,
     5066               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_1));
     5067
     5068    uint8_t const idxArgVar = iemNativeArgAlloc(pReNative, iArgNo, sizeof(uintptr_t));
     5069    iemNativeVarSetKindToLocalRef(pReNative, idxArgVar, idxOtherVar);
     5070    return idxArgVar;
     5071}
     5072
     5073
     5074DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
     5075{
     5076    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
     5077    iemNativeVarSetKindToStack(pReNative, idxVar);
     5078    return idxVar;
     5079}
     5080
     5081
     5082DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue)
     5083{
     5084    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
     5085    iemNativeVarSetKindToConst(pReNative, idxVar, uValue);
     5086    return idxVar;
     5087}
     5088
     5089
     5090/**
     5091 * Makes sure variable @a idxVar has a register assigned to it.
     5092 *
     5093 * @returns The host register number.
     5094 * @param   pReNative   The recompiler state.
     5095 * @param   idxVar      The variable.
     5096 * @param   poff        Pointer to the instruction buffer offset.
     5097 *                      In case a register needs to be freed up.
     5098 */
     5099DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocRegister(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff)
     5100{
     5101    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
     5102
     5103    uint8_t idxReg = pReNative->Core.aVars[idxVar].idxReg;
     5104    if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
     5105        return idxReg;
     5106
     5107    /*
     5108     * We have to allocate a register for the variable, even if its a stack one
     5109     * as we don't know if there are modification being made to it before its
     5110     * finalized (todo: analyze and insert hints about that?).
     5111     *
     5112     * If we can, we try get the correct register for argument variables. This
     5113     * is assuming that most argument variables are fetched as close as possible
     5114     * to the actual call, so that there aren't any interfering hidden calls
     5115     * (memory accesses, etc) inbetween.
     5116     *
     5117     * If we cannot or it's a variable, we make sure no argument registers
     5118     * that will be used by this MC block will be allocated here, and we always
     5119     * prefer non-volatile registers to avoid needing to spill stuff for internal
     5120     * call.
     5121     */
     5122    /** @todo Detect too early argument value fetches and warn about hidden
     5123     * calls causing less optimal code to be generated in the python script. */
     5124
     5125    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
     5126    if (   uArgNo < RT_ELEMENTS(g_aidxIemNativeCallRegs)
     5127        && !(pReNative->Core.bmHstRegs & RT_BIT_32(g_aidxIemNativeCallRegs[uArgNo])))
     5128        idxReg = g_aidxIemNativeCallRegs[uArgNo];
     5129    else
     5130    {
     5131        uint32_t const fNotArgsMask = ~g_afIemNativeCallRegs[RT_MIN(pReNative->cArgs, IEMNATIVE_CALL_ARG_GREG_COUNT)];
     5132        uint32_t const fRegs        = ~pReNative->Core.bmHstRegs
     5133                                    & ~pReNative->Core.bmHstRegsWithGstShadow
     5134                                    & (~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
     5135                                    & fNotArgsMask;
     5136        if (fRegs)
     5137        {
     5138            /* Pick from the top as that both arm64 and amd64 have a block of non-volatile registers there. */
     5139            idxReg = (uint8_t)ASMBitLastSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
     5140                                               ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
     5141            Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows == 0);
     5142            Assert(!(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg)));
     5143        }
     5144        else
     5145        {
     5146            idxReg = iemNativeRegAllocFindFree(pReNative, poff, false /*fPreferVolatile*/,
     5147                                               IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & fNotArgsMask);
     5148            AssertStmt(idxReg != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_ALLOCATOR_NO_FREE_VAR));
     5149        }
     5150    }
     5151    iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
     5152    pReNative->Core.aVars[idxVar].idxReg = idxReg;
     5153    return idxReg;
     5154}
     5155
     5156
     5157
     5158/*********************************************************************************************************************************
     5159*   Emitters for IEM_MC_CALL_CIMPL_XXX                                                                                           *
     5160*********************************************************************************************************************************/
     5161
     5162/**
     5163 * Emits code to load a reference to the given guest register into @a idxGprDst.
     5164  */
     5165DECL_INLINE_THROW(uint32_t)
     5166iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
     5167                               IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass)
     5168{
     5169    /*
     5170     * Get the offset relative to the CPUMCTX structure.
     5171     */
     5172    uint32_t offCpumCtx;
     5173    switch (enmClass)
     5174    {
     5175        case kIemNativeGstRegRef_Gpr:
     5176            Assert(idxRegInClass < 16);
     5177            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, aGRegs[idxRegInClass]);
     5178            break;
     5179
     5180        case kIemNativeGstRegRef_GprHighByte:    /**< AH, CH, DH, BH*/
     5181            Assert(idxRegInClass < 4);
     5182            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, aGRegs[0].bHi) + idxRegInClass * sizeof(CPUMCTXGREG);
     5183            break;
     5184
     5185        case kIemNativeGstRegRef_EFlags:
     5186            Assert(idxRegInClass == 0);
     5187            offCpumCtx = RT_UOFFSETOF(CPUMCTX, eflags);
     5188            break;
     5189
     5190        case kIemNativeGstRegRef_MxCsr:
     5191            Assert(idxRegInClass == 0);
     5192            offCpumCtx = RT_UOFFSETOF(CPUMCTX, XState.x87.MXCSR);
     5193            break;
     5194
     5195        case kIemNativeGstRegRef_FpuReg:
     5196            Assert(idxRegInClass < 8);
     5197            AssertFailed(); /** @todo what kind of indexing? */
     5198            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aRegs[idxRegInClass]);
     5199            break;
     5200
     5201        case kIemNativeGstRegRef_MReg:
     5202            Assert(idxRegInClass < 8);
     5203            AssertFailed(); /** @todo what kind of indexing? */
     5204            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aRegs[idxRegInClass]);
     5205            break;
     5206
     5207        case kIemNativeGstRegRef_XReg:
     5208            Assert(idxRegInClass < 16);
     5209            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aXMM[idxRegInClass]);
     5210            break;
     5211
     5212        default:
     5213            AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_5));
     5214    }
     5215
     5216    /*
     5217     * Load the value into the destination register.
     5218     */
     5219#ifdef RT_ARCH_AMD64
     5220    off = iemNativeEmitLeaGprByVCpu(pReNative, off, idxGprDst, offCpumCtx + RT_UOFFSETOF(VMCPUCC, cpum.GstCtx));
     5221
     5222#elif defined(RT_ARCH_ARM64)
     5223    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
     5224    Assert(offCpumCtx < 4096);
     5225    pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, idxGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, offCpumCtx);
     5226
     5227#else
     5228# error "Port me!"
     5229#endif
     5230
     5231    return off;
     5232}
     5233
     5234
     5235/**
     5236 * Common code for CIMPL and AIMPL calls.
     5237 *
     5238 * These are calls that uses argument variables and such.  They should not be
     5239 * confused with internal calls required to implement an MC operation,
     5240 * like a TLB load and similar.
     5241 *
     5242 * Upon return all that is left to do is to load any hidden arguments and
     5243 * perform the call. All argument variables are freed.
     5244 *
     5245 * @returns New code buffer offset; throws VBox status code on error.
     5246 * @param   pReNative       The native recompile state.
     5247 * @param   off             The code buffer offset.
     5248 * @param   cArgs           The total nubmer of arguments (includes hidden
     5249 *                          count).
     5250 * @param   cHiddenArgs     The number of hidden arguments.  The hidden
     5251 *                          arguments must not have any variable declared for
     5252 *                          them, whereas all the regular arguments must
     5253 *                          (tstIEMCheckMc ensures this).
     5254 */
     5255DECL_HIDDEN_THROW(uint32_t)
     5256iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs)
     5257{
     5258#ifdef VBOX_STRICT
     5259    /*
     5260     * Assert sanity.
     5261     */
     5262    Assert(cArgs <= IEMNATIVE_CALL_MAX_ARG_COUNT);
     5263    Assert(cHiddenArgs < IEMNATIVE_CALL_ARG_GREG_COUNT);
     5264    for (unsigned i = 0; i < cHiddenArgs; i++)
     5265        Assert(pReNative->Core.aidxArgVars[i] == UINT8_MAX);
     5266    for (unsigned i = cHiddenArgs; i < cArgs; i++)
     5267    {
     5268        Assert(pReNative->Core.aidxArgVars[i] != UINT8_MAX); /* checked by tstIEMCheckMc.cpp */
     5269        Assert(pReNative->Core.bmVars & RT_BIT_32(pReNative->Core.aidxArgVars[i]));
     5270    }
     5271#endif
     5272
     5273    uint8_t const cRegArgs = RT_MIN(cArgs, RT_ELEMENTS(g_aidxIemNativeCallRegs));
     5274
     5275    /*
     5276     * First, go over the host registers that will be used for arguments and make
     5277     * sure they either hold the desired argument or are free.
     5278     */
     5279    if (pReNative->Core.bmHstRegs & g_afIemNativeCallRegs[cRegArgs])
     5280        for (uint32_t i = 0; i < cRegArgs; i++)
     5281        {
     5282            uint8_t const idxArgReg = g_aidxIemNativeCallRegs[i];
     5283            if (pReNative->Core.bmHstRegs & RT_BIT_32(idxArgReg))
     5284            {
     5285                if (pReNative->Core.aHstRegs[idxArgReg].enmWhat == kIemNativeWhat_Var)
     5286                {
     5287                    uint8_t const idxVar = pReNative->Core.aHstRegs[idxArgReg].idxVar;
     5288                    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
     5289                    Assert(pReNative->Core.aVars[idxVar].idxReg == idxArgReg);
     5290                    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
     5291                    if (uArgNo == i)
     5292                    { /* prefect */ }
     5293                    else
     5294                    {
     5295                        /* The variable allocator logic should make sure this is impossible. */
     5296                        AssertStmt(uArgNo == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_10));
     5297
     5298                        if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     5299                            off = iemNativeRegMoveOrSpillStackVar(pReNative, off, idxVar);
     5300                        else
     5301                        {
     5302                            /* just free it, can be reloaded if used again */
     5303                            pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
     5304                            pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxArgReg);
     5305                            iemNativeRegClearGstRegShadowing(pReNative, idxArgReg, off);
     5306                        }
     5307                    }
     5308                }
     5309                else
     5310                    AssertStmt(pReNative->Core.aHstRegs[idxArgReg].enmWhat == kIemNativeWhat_Arg,
     5311                               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_8));
     5312            }
     5313        }
     5314
     5315    Assert(!(pReNative->Core.bmHstRegs & g_afIemNativeCallRegs[cHiddenArgs])); /* No variables for hidden arguments. */
     5316
     5317    /*
     5318     * Make sure the argument variables are loaded into their respective registers.
     5319     *
     5320     * We can optimize this by ASSUMING that any register allocations are for
     5321     * registeres that have already been loaded and are ready.  The previous step
     5322     * saw to that.
     5323     */
     5324    if (~pReNative->Core.bmHstRegs & (g_afIemNativeCallRegs[cRegArgs] & ~g_afIemNativeCallRegs[cHiddenArgs]))
     5325        for (unsigned i = cHiddenArgs; i < cRegArgs; i++)
     5326        {
     5327            uint8_t const idxArgReg = g_aidxIemNativeCallRegs[i];
     5328            if (pReNative->Core.bmHstRegs & RT_BIT_32(idxArgReg))
     5329                Assert(   pReNative->Core.aHstRegs[idxArgReg].idxVar == pReNative->Core.aidxArgVars[i]
     5330                       && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].uArgNo == i
     5331                       && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].idxReg == idxArgReg);
     5332            else
     5333            {
     5334                uint8_t const idxVar = pReNative->Core.aidxArgVars[i];
     5335                if (pReNative->Core.aVars[idxVar].uArgNo < RT_ELEMENTS(pReNative->Core.aHstRegs))
     5336                {
     5337                    Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
     5338                    off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].idxReg);
     5339                }
     5340                else
     5341                {
     5342                    /* Use ARG0 as temp for stuff we need registers for. */
     5343                    switch (pReNative->Core.aVars[idxVar].enmKind)
     5344                    {
     5345                        case kIemNativeVarKind_Stack:
     5346                            AssertStmt(pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX,
     5347                                       IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
     5348                            off = iemNativeEmitLoadGprByBp(pReNative, off, idxArgReg,
     5349                                                             IEMNATIVE_FP_OFF_STACK_VARS
     5350                                                           + pReNative->Core.aVars[idxVar].idxStackSlot * sizeof(uint64_t));
     5351                            continue;
     5352
     5353                        case kIemNativeVarKind_Immediate:
     5354                            off = iemNativeEmitLoadGprImm64(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].u.uValue);
     5355                            continue;
     5356
     5357                        case kIemNativeVarKind_VarRef:
     5358                        {
     5359                            uint8_t const idxOtherVar = pReNative->Core.aVars[idxVar].u.idxRefVar;
     5360                            Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
     5361                            AssertStmt(pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX,
     5362                                       IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_4));
     5363                            off = iemNativeEmitLeaGprByBp(pReNative, off, idxArgReg,
     5364                                                            IEMNATIVE_FP_OFF_STACK_VARS
     5365                                                          + pReNative->Core.aVars[idxOtherVar].idxStackSlot * sizeof(uint64_t));
     5366                            continue;
     5367                        }
     5368
     5369                        case kIemNativeVarKind_GstRegRef:
     5370                            off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, idxArgReg,
     5371                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
     5372                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
     5373                            continue;
     5374
     5375                        case kIemNativeVarKind_Invalid:
     5376                        case kIemNativeVarKind_End:
     5377                            break;
     5378                    }
     5379                    AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
     5380                }
     5381            }
     5382        }
     5383#ifdef VBOX_STRICT
     5384    else
     5385        for (unsigned i = cHiddenArgs; i < cRegArgs; i++)
     5386        {
     5387            Assert(pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].uArgNo == i);
     5388            Assert(pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].idxReg == g_aidxIemNativeCallRegs[i]);
     5389        }
     5390#endif
     5391
     5392#ifdef IEMNATIVE_FP_OFF_STACK_ARG0
     5393    /*
     5394     * If there are any stack arguments, make sure they are in their place as well.
     5395     *
     5396     * We can use IEMNATIVE_CALL_ARG0_GREG as temporary register since it the
     5397     * caller will load it later and it must be free (see first loop).
     5398     */
     5399    if (cArgs > IEMNATIVE_CALL_ARG_GREG_COUNT)
     5400        for (unsigned i = IEMNATIVE_CALL_ARG_GREG_COUNT; i < cArgs; i++)
     5401        {
     5402            uint8_t const idxVar    = pReNative->Core.aidxArgVars[i];
     5403            int32_t const offBpDisp = g_aoffIemNativeCallStackArgBpDisp[i - IEMNATIVE_CALL_ARG_GREG_COUNT];
     5404            if (pReNative->Core.aVars[idxVar].uArgNo < RT_ELEMENTS(pReNative->Core.aHstRegs))
     5405            {
     5406                Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack); /* Imm as well? */
     5407                off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].uArgNo);
     5408            }
     5409            else
     5410            {
     5411                /* Use ARG0 as temp for stuff we need registers for. */
     5412                switch (pReNative->Core.aVars[idxVar].enmKind)
     5413                {
     5414                    case kIemNativeVarKind_Stack:
     5415                        AssertStmt(pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX,
     5416                                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
     5417                        off = iemNativeEmitLoadGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG /* is free */,
     5418                                                         IEMNATIVE_FP_OFF_STACK_VARS
     5419                                                       + pReNative->Core.aVars[idxVar].idxStackSlot * sizeof(uint64_t));
     5420                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
     5421                        continue;
     5422
     5423                    case kIemNativeVarKind_Immediate:
     5424                        off = iemNativeEmitStoreImm64ByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].u.uValue);
     5425                        continue;
     5426
     5427                    case kIemNativeVarKind_VarRef:
     5428                    {
     5429                        uint8_t const idxOtherVar = pReNative->Core.aVars[idxVar].u.idxRefVar;
     5430                        Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
     5431                        AssertStmt(pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX,
     5432                                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_4));
     5433                        off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG,
     5434                                                        IEMNATIVE_FP_OFF_STACK_VARS
     5435                                                      + pReNative->Core.aVars[idxOtherVar].idxStackSlot * sizeof(uint64_t));
     5436                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
     5437                        continue;
     5438                    }
     5439
     5440                    case kIemNativeVarKind_GstRegRef:
     5441                        off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, IEMNATIVE_CALL_ARG0_GREG,
     5442                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
     5443                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
     5444                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
     5445                        continue;
     5446
     5447                    case kIemNativeVarKind_Invalid:
     5448                    case kIemNativeVarKind_End:
     5449                        break;
     5450                }
     5451                AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
     5452            }
     5453        }
     5454#else
     5455    AssertCompile(IEMNATIVE_CALL_MAX_ARG_COUNT <= IEMNATIVE_CALL_ARG_GREG_COUNT);
     5456#endif
     5457
     5458    /*
     5459     * Free all argument variables (simplified).
     5460     * Their lifetime always expires with the call they are for.
     5461     */
     5462    /** @todo Unfortunately we have some high 32-bit register clears using
     5463     *        register references held by argument vars. Almost all the
     5464     *        IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF uses are stupid!  However we need to
     5465     *        add checks to the python scripts to ensure argument vars aren't used
     5466     *        after a call. That's a bit of work. */
     5467    for (uint32_t i = cHiddenArgs; i < cArgs; i++)
     5468    {
     5469        uint8_t idxVar = pReNative->Core.aidxArgVars[i];
     5470        Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
     5471        pReNative->Core.aidxArgVars[i] = UINT8_MAX;
     5472        pReNative->Core.bmVars        &= ~RT_BIT_32(idxVar);
     5473        Assert(   pReNative->Core.aVars[idxVar].idxReg
     5474               == (i < RT_ELEMENTS(g_aidxIemNativeCallRegs) ? g_aidxIemNativeCallRegs[i] : UINT8_MAX));
     5475    }
     5476    Assert(pReNative->Core.u64ArgVars == UINT64_MAX);
     5477
     5478    /*
     5479     * Flush volatile registers as we make the call.
     5480     */
     5481    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, cRegArgs);
     5482
     5483    return off;
     5484}
     5485
     5486
     5487/** Common emit function for IEM_MC_CALL_CIMPL_XXXX. */
     5488DECL_HIDDEN_THROW(uint32_t)
     5489iemNativeEmitCallCImplCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5490                             uintptr_t pfnCImpl, uint8_t cArgs)
     5491
     5492{
     5493//pReNative->pInstrBuf[off++] = 0xcc;
     5494    /*
     5495     * Do all the call setup and cleanup.
     5496     */
     5497    off = iemNativeEmitCallCommon(pReNative, off, cArgs + IEM_CIMPL_HIDDEN_ARGS, IEM_CIMPL_HIDDEN_ARGS);
     5498
     5499    /*
     5500     * Load the two hidden arguments.
     5501     */
     5502#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
     5503    off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict */
     5504    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     5505    off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, cbInstr);
     5506#else
     5507    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     5508    off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, cbInstr);
     5509#endif
     5510
     5511    /*
     5512     * Make the call and check the return code.
     5513     */
     5514    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)pfnCImpl);
     5515#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
     5516    off = iemNativeEmitLoadGprByBpU32(pReNative, off, X86_GREG_xAX, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict (see above) */
     5517#endif
     5518    return iemNativeEmitCheckCallRetAndPassUp(pReNative, off, idxInstr);
     5519}
     5520
     5521
     5522#define IEM_MC_CALL_CIMPL_1_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0) \
     5523    off = iemNativeEmitCallCImpl1(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0)
     5524
     5525/** Emits code for IEM_MC_CALL_CIMPL_1. */
     5526DECL_INLINE_THROW(uint32_t)
     5527iemNativeEmitCallCImpl1(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5528                        uintptr_t pfnCImpl, uint8_t idxArg0)
     5529{
     5530    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
     5531    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
     5532    RT_NOREF_PV(idxArg0);
     5533
     5534    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 1);
     5535}
     5536
     5537
     5538#define IEM_MC_CALL_CIMPL_2_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1) \
     5539    off = iemNativeEmitCallCImpl2(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1)
     5540
     5541/** Emits code for IEM_MC_CALL_CIMPL_2. */
     5542DECL_INLINE_THROW(uint32_t)
     5543iemNativeEmitCallCImpl2(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5544                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1)
     5545{
     5546    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
     5547    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
     5548    RT_NOREF_PV(idxArg0);
     5549
     5550    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
     5551    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
     5552    RT_NOREF_PV(idxArg1);
     5553
     5554    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 2);
     5555}
     5556
     5557
     5558#define IEM_MC_CALL_CIMPL_3_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2) \
     5559    off = iemNativeEmitCallCImpl3(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2)
     5560
     5561/** Emits code for IEM_MC_CALL_CIMPL_3. */
     5562DECL_INLINE_THROW(uint32_t)
     5563iemNativeEmitCallCImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5564                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
     5565{
     5566    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
     5567    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
     5568    RT_NOREF_PV(idxArg0);
     5569
     5570    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
     5571    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
     5572    RT_NOREF_PV(idxArg1);
     5573
     5574    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
     5575    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
     5576    RT_NOREF_PV(idxArg2);
     5577
     5578    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 3);
     5579}
     5580
     5581
     5582#define IEM_MC_CALL_CIMPL_4_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
     5583    off = iemNativeEmitCallCImpl4(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2, a3)
     5584
     5585/** Emits code for IEM_MC_CALL_CIMPL_4. */
     5586DECL_INLINE_THROW(uint32_t)
     5587iemNativeEmitCallCImpl4(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5588                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3)
     5589{
     5590    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
     5591    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
     5592    RT_NOREF_PV(idxArg0);
     5593
     5594    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
     5595    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
     5596    RT_NOREF_PV(idxArg1);
     5597
     5598    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
     5599    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
     5600    RT_NOREF_PV(idxArg2);
     5601
     5602    Assert(idxArg3 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg3)));
     5603    Assert(pReNative->Core.aVars[idxArg3].uArgNo == 3 + IEM_CIMPL_HIDDEN_ARGS);
     5604    RT_NOREF_PV(idxArg3);
     5605
     5606    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 4);
     5607}
     5608
     5609
     5610#define IEM_MC_CALL_CIMPL_5_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
     5611    off = iemNativeEmitCallCImpl5(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2, a3, a4)
     5612
     5613/** Emits code for IEM_MC_CALL_CIMPL_4. */
     5614DECL_INLINE_THROW(uint32_t)
     5615iemNativeEmitCallCImpl5(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
     5616                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3, uint8_t idxArg4)
     5617{
     5618    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
     5619    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
     5620    RT_NOREF_PV(idxArg0);
     5621
     5622    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
     5623    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
     5624    RT_NOREF_PV(idxArg1);
     5625
     5626    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
     5627    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
     5628    RT_NOREF_PV(idxArg2);
     5629
     5630    Assert(idxArg3 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg3)));
     5631    Assert(pReNative->Core.aVars[idxArg3].uArgNo == 3 + IEM_CIMPL_HIDDEN_ARGS);
     5632    RT_NOREF_PV(idxArg3);
     5633
     5634    Assert(idxArg4 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg4)));
     5635    Assert(pReNative->Core.aVars[idxArg4].uArgNo == 4 + IEM_CIMPL_HIDDEN_ARGS);
     5636    RT_NOREF_PV(idxArg4);
     5637
     5638    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 5);
     5639}
     5640
     5641
     5642
     5643/*********************************************************************************************************************************
     5644*   Emitters for general purpose register fetches (IEM_MC_FETCH_GREG_XXX).                                                       *
     5645*********************************************************************************************************************************/
     5646
     5647#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) \
     5648    off = iemNativeEmitFetchGregU16(pReNative, off, a_u16Dst, a_iGReg)
     5649
     5650/** Emits code for IEM_MC_FETCH_GREG_U16. */
     5651DECL_INLINE_THROW(uint32_t)
     5652iemNativeEmitFetchGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg)
     5653{
     5654    Assert(idxDstVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxDstVar)));
     5655    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint16_t));
     5656
     5657    /*
     5658     * We can either just load the low 16-bit of the GPR into a host register
     5659     * for the variable, or we can do so via a shadow copy host register. The
     5660     * latter will avoid having to reload it if it's being stored later, but
     5661     * will waste a host register if it isn't touched again.  Since we don't
     5662     * know what going to happen, we choose the latter for now.
     5663     */
     5664    uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,
     5665                                                                  (IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + iGReg),
     5666                                                                  kIemNativeGstRegUse_ReadOnly);
     5667
     5668    iemNativeVarSetKindToStack(pReNative, idxDstVar);
     5669    uint8_t const idxVarReg = iemNativeVarAllocRegister(pReNative, idxDstVar, &off);
     5670    off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxVarReg, idxGstFullReg);
     5671
     5672    iemNativeRegFreeTmp(pReNative, idxGstFullReg);
     5673    return off;
     5674}
     5675
     5676
     5677
     5678/*********************************************************************************************************************************
     5679*   Emitters for general purpose register stores (IEM_MC_STORE_GREG_XXX).                                                        *
     5680*********************************************************************************************************************************/
    47965681
    47975682#define IEM_MC_STORE_GREG_U8_CONST_THREADED(a_iGRegEx, a_u8Value) \
    … …  
    47995684
    48005685/** Emits code for IEM_MC_STORE_GREG_U8_CONST_THREADED. */
    4801 DECLINLINE(uint32_t) iemNativeEmitStoreGregU8Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t u8Value)
     5686DECL_INLINE_THROW(uint32_t)
     5687iemNativeEmitStoreGregU8Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t u8Value)
    48025688{
    48035689    uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,
    … …  
    48785764
    48795765/** Emits code for IEM_MC_SUB_GREG_U16. */
    4880 DECLINLINE(uint32_t) iemNativeEmitSubGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend)
     5766DECL_INLINE_THROW(uint32_t)
     5767iemNativeEmitSubGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend)
    48815768{
    48825769    uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r101705 r101844  
    4949 * @{  */
    5050/** The size of the area for stack variables and spills and stuff.
    51  * @note This limit is duplicated in the python script(s). */
    52 #define IEMNATIVE_FRAME_VAR_SIZE            0xc0
    53 #ifdef RT_ARCH_AMD64
    54 /** Number of stack arguments slots for calls made from the frame. */
    55 # define IEMNATIVE_FRAME_STACK_ARG_COUNT    4
     51 * @note This limit is duplicated in the python script(s).  We add 0x40 for
     52 *       alignment padding. */
     53#define IEMNATIVE_FRAME_VAR_SIZE            (0xc0 + 0x40)
     54/** Number of 64-bit variable slots (0x100 / 8 = 32. */
     55#define IEMNATIVE_FRAME_VAR_SLOTS           (IEMNATIVE_FRAME_VAR_SIZE / 8)
     56AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
     57
     58#ifdef RT_ARCH_AMD64
    5659/** An stack alignment adjustment (between non-volatile register pushes and
    5760 *  the stack variable area, so the latter better aligned). */
    5861# define IEMNATIVE_FRAME_ALIGN_SIZE         8
     62
     63/** Number of stack arguments slots for calls made from the frame. */
     64# ifdef RT_OS_WINDOWS
     65#  define IEMNATIVE_FRAME_STACK_ARG_COUNT   4
     66# else
     67#  define IEMNATIVE_FRAME_STACK_ARG_COUNT   2
     68# endif
    5969/** Number of any shadow arguments (spill area) for calls we make. */
    6070# ifdef RT_OS_WINDOWS
    … …  
    7787/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
    7888# define IEMNATIVE_FP_OFF_STACK_ARG1        (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
     89# ifdef RT_OS_WINDOWS
    7990/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
    80 # define IEMNATIVE_FP_OFF_STACK_ARG2        (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
     91#  define IEMNATIVE_FP_OFF_STACK_ARG2       (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
    8192/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
    82 # define IEMNATIVE_FP_OFF_STACK_ARG3        (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
     93#  define IEMNATIVE_FP_OFF_STACK_ARG3       (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
     94# endif
    8395
    8496# ifdef RT_OS_WINDOWS
    … …  
    94106
    95107#elif RT_ARCH_ARM64
    96 /** No stack argument slots, enough got 8 registers for arguments.  */
     108/** No alignment padding needed for arm64. */
     109# define IEMNATIVE_FRAME_ALIGN_SIZE         0
     110/** No stack argument slots, got 8 registers for arguments will suffice. */
    97111# define IEMNATIVE_FRAME_STACK_ARG_COUNT    0
    98112/** There are no argument spill area. */
    … …  
    239253#endif
    240254
     255/** This is the maximum argument count we'll ever be needing. */
     256#define IEMNATIVE_CALL_MAX_ARG_COUNT        7
    241257/** @} */
    242258
    … …  
    374390    kIemNativeGstRegRef_MReg,
    375391    kIemNativeGstRegRef_XReg,
    376     kIemNativeGstRegRef_YReg,
     392    //kIemNativeGstRegRef_YReg, - doesn't work.
    377393    kIemNativeGstRegRef_End
    378394} IEMNATIVEGSTREGREF;
    … …  
    523539    };
    524540
     541    /** Allocation bitmap for the stack. */
     542    uint32_t                    bmStack;
    525543    /** Allocation bitmap for aVars. */
    526544    uint32_t                    bmVars;
    … …  
    615633    /** Check IRQ seqeunce number (for generating unique lables). */
    616634    uint16_t                    uCheckIrqSeqNo;
    617     uint16_t                    uPadding3;
     635    uint8_t                     bPadding3;
     636
     637    /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
     638    uint8_t                     cArgs;
     639    /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
     640    uint32_t                    fCImpl;
     641    /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
     642    uint32_t                    fMc;
    618643
    619644    /** Core state requiring care with branches. */
    … …  
    12521277
    12531278/**
     1279 * Emits a load effective address to a GRP of a VCpu field.
     1280 */
     1281DECL_INLINE_THROW(uint32_t)
     1282iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
     1283{
     1284#ifdef RT_ARCH_AMD64
     1285    /* lea gprdst, [rbx + offDisp] */
     1286    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
     1287    if (iGprDst < 8)
     1288        pbCodeBuf[off++] = X86_OP_REX_W;
     1289    else
     1290        pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     1291    pbCodeBuf[off++] = 0x8d;
     1292    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
     1293
     1294#elif defined(RT_ARCH_ARM64)
     1295    if (offVCpu < (unsigned)_4K)
     1296    {
     1297        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1298        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
     1299    }
     1300    else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
     1301    {
     1302        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1303        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
     1304                                                         offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
     1305    }
     1306    else
     1307    {
     1308        Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
     1309        off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
     1310        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1311        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
     1312    }
     1313
     1314#else
     1315# error "port me"
     1316#endif
     1317    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1318    return off;
     1319}
     1320
     1321
     1322/**
    12541323 * Emits a gprdst = gprsrc load.
    12551324 */
    … …  
    12711340#elif RT_ARCH_ARM64
    12721341    /* mov dst, src;   alias for: orr dst, xzr, src */
    1273     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    1274     pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
     1342    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1343    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
    12751344
    12761345#else
    … …  
    12801349    return off;
    12811350}
     1351
     1352
     1353/**
     1354 * Emits a gprdst = gprsrc[31:0] load.
     1355 * @note Bits 63 thru 32 are cleared.
     1356 */
     1357DECL_INLINE_THROW(uint32_t)
     1358iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     1359{
     1360#ifdef RT_ARCH_AMD64
     1361    /* mov gprdst, gprsrc */
     1362    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
     1363    if ((iGprDst | iGprSrc) >= 8)
     1364        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     1365                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     1366                         :                X86_OP_REX_R;
     1367    pbCodeBuf[off++] = 0x8b;
     1368    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     1369
     1370#elif RT_ARCH_ARM64
     1371    /* mov dst32, src32;   alias for: orr dst32, wzr, src32 */
     1372    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1373    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);
     1374
     1375#else
     1376# error "port me"
     1377#endif
     1378    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1379    return off;
     1380}
     1381
     1382
     1383/**
     1384 * Emits a gprdst = gprsrc[15:0] load.
     1385 * @note Bits 63 thru 15 are cleared.
     1386 */
     1387DECL_INLINE_THROW(uint32_t)
     1388iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     1389{
     1390#ifdef RT_ARCH_AMD64
     1391    /* movzx Gv,Ew */
     1392    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
     1393    if ((iGprDst | iGprSrc) >= 8)
     1394        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     1395                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     1396                         :                X86_OP_REX_R;
     1397    pbCodeBuf[off++] = 0x0f;
     1398    pbCodeBuf[off++] = 0xb7;
     1399    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     1400
     1401#elif RT_ARCH_ARM64
     1402    /* and gprdst, gprsrc, #0xffff */
     1403    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1404# if 1
     1405    Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
     1406    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /*f64Bit*/);
     1407# else
     1408    Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
     1409    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
     1410# endif
     1411
     1412#else
     1413# error "port me"
     1414#endif
     1415    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1416    return off;
     1417}
     1418
     1419
     1420/**
     1421 * Emits a gprdst = gprsrc[7:0] load.
     1422 * @note Bits 63 thru 8 are cleared.
     1423 */
     1424DECL_INLINE_THROW(uint32_t)
     1425iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     1426{
     1427#ifdef RT_ARCH_AMD64
     1428    /* movzx Gv,Eb */
     1429    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
     1430    if ((iGprDst | iGprSrc) >= 8)
     1431        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     1432                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     1433                         :                X86_OP_REX_R;
     1434    pbCodeBuf[off++] = 0x0f;
     1435    pbCodeBuf[off++] = 0xb6;
     1436    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     1437
     1438#elif RT_ARCH_ARM64
     1439    /* and gprdst, gprsrc, #0xff */
     1440    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1441# if 1
     1442    Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
     1443    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);
     1444# else
     1445    Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
     1446    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
     1447# endif
     1448
     1449#else
     1450# error "port me"
     1451#endif
     1452    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1453    return off;
     1454}
     1455
     1456#if 0 /** @todo */
     1457/**
     1458 * Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
     1459 * @note Bits 63 thru 8 are cleared.
     1460 */
     1461DECL_INLINE_THROW(uint32_t)
     1462iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     1463{
     1464#ifdef RT_ARCH_AMD64
     1465    /* movzx Gv,Eb */
     1466    /** @todo */
     1467    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
     1468    if ((iGprDst | iGprSrc) >= 8)
     1469        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     1470                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     1471                         :                X86_OP_REX_R;
     1472    pbCodeBuf[off++] = 0x0f;
     1473    pbCodeBuf[off++] = 0xb6;
     1474    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     1475
     1476#elif RT_ARCH_ARM64
     1477    /* ubfx gprdst, gprsrc, #8, #8 */
     1478    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1479    Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
     1480    pu32CodeBuf[off++] = /** @todo ubfx */;
     1481
     1482#else
     1483# error "port me"
     1484#endif
     1485    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1486    return off;
     1487}
     1488#endif
    12821489
    12831490#ifdef RT_ARCH_AMD64
    … …  
    13431550
    13441551
    1345 #ifdef RT_ARCH_AMD64
    13461552/**
    13471553 * Emits a load effective address to a GRP with an BP relative source address.
    … …  
    13501556iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
    13511557{
     1558#ifdef RT_ARCH_AMD64
    13521559    /* lea gprdst, [rbp + offDisp] */
    13531560    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    … …  
    13571564        pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
    13581565    pbCodeBuf[off++] = 0x8d;
    1359     return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
    1360 }
    1361 #endif
     1566    off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
     1567
     1568#elif defined(RT_ARCH_ARM64)
     1569    if ((uint32_t)offDisp < (unsigned)_4K)
     1570    {
     1571        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1572        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);
     1573    }
     1574    else if ((uint32_t)-offDisp < (unsigned)_4K)
     1575    {
     1576        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1577        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);
     1578    }
     1579    else
     1580    {
     1581        Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
     1582        off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
     1583        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     1584        if (offDisp >= 0)
     1585            pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
     1586        else
     1587            pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
     1588    }
     1589
     1590#else
     1591# error "port me"
     1592#endif
     1593
     1594    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1595    return off;
     1596}
    13621597
    13631598
    … …  
    18442079{
    18452080#if defined(RT_ARCH_AMD64)
    1846     /* movzx reg32, reg16 */
     2081    /* movzx Gv,Ew */
    18472082    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    18482083    if (iGprDst >= 8)
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette