VirtualBox

Changeset 102790 in vbox for trunk


Ignore:
Timestamp:
Jan 9, 2024 1:41:28 AM (13 months ago)
Author:
vboxsync
Message:

VMM/IEM: Emit TLB lookup for POP GPR instructions. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllMemRWTmpl.cpp.h

    r102785 r102790  
    535535
    536536/**
     537 * Safe/fallback stack fetch function that longjmps on error.
     538 */
     539TMPL_MEM_TYPE RT_CONCAT3(iemMemFetchStack,TMPL_MEM_FN_SUFF,SafeJmp)(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
     540{
     541# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
     542    pVCpu->iem.s.DataTlb.cTlbSafeReadPath++;
     543# endif
     544
     545    /* Read the data. */
     546    uint8_t              bUnmapInfo;
     547    TMPL_MEM_TYPE const *puSrc = (TMPL_MEM_TYPE const *)iemMemMapJmp(pVCpu, &bUnmapInfo, sizeof(TMPL_MEM_TYPE), X86_SREG_SS,
     548                                                                     GCPtrMem, IEM_ACCESS_STACK_R, TMPL_MEM_TYPE_ALIGN);
     549    TMPL_MEM_TYPE const  uValue = *puSrc;
     550    iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
     551
     552    /* Commit the register and RSP values. */
     553    Log10(("IEM RD " TMPL_MEM_FMT_DESC " SS|%RGv: " TMPL_MEM_FMT_TYPE "\n", GCPtrMem, uValue));
     554    return uValue;
     555}
     556
     557
     558/**
    537559 * Safe/fallback stack push function that longjmps on error.
    538560 */

  • trunk/src/VBox/VMM/VMMAll/IEMAllMemRWTmplInline.cpp.h

    r102785 r102790  
    731731RT_CONCAT3(iemMemStoreStack,TMPL_MEM_FN_SUFF,Jmp)(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, TMPL_MEM_TYPE uValue) IEM_NOEXCEPT_MAY_LONGJMP
    732732{
    733 #  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     733#   if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
    734734    /*
    735735     * Apply segmentation and check that the item doesn't cross a page boundrary.
    736736     */
    737737    RTGCPTR const GCPtrEff = iemMemApplySegmentToWriteJmp(pVCpu, X86_SREG_SS, sizeof(TMPL_MEM_TYPE), GCPtrMem);
    738 #  if TMPL_MEM_TYPE_SIZE > 1
     738#    if TMPL_MEM_TYPE_SIZE > 1
    739739    if (RT_LIKELY(TMPL_MEM_ALIGN_CHECK(GCPtrEff)))
    740 #  endif
     740#    endif
    741741    {
    742742        /*
    … …  
    774774       outdated page pointer, or other troubles.  (This will do a TLB load.) */
    775775    Log12Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrEff));
    776 #  endif
     776#   endif
    777777    RT_CONCAT3(iemMemStoreStack,TMPL_MEM_FN_SUFF,SafeJmp)(pVCpu, GCPtrMem, uValue);
    778778}
    … …  
    790790                                                      TMPL_MEM_TYPE uValue) IEM_NOEXCEPT_MAY_LONGJMP
    791791{
    792 #  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
    793     /*
    794      * Decrement the stack pointer (prep), apply segmentation and check that
    795      * the item doesn't cross a page boundrary.
     792#    if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     793    /*
     794     * Apply segmentation to the address and check that the item doesn't cross
     795     * a page boundrary.
    796796     */
    797797    RTGCPTR const GCPtrEff = iemMemApplySegmentToWriteJmp(pVCpu, X86_SREG_SS, sizeof(TMPL_MEM_TYPE), GCPtrMem);
    798 #  if TMPL_MEM_TYPE_SIZE > 1
     798#     if TMPL_MEM_TYPE_SIZE > 1
    799799    if (RT_LIKELY(   !(GCPtrEff & (sizeof(uint16_t) - 1U))
    800800                  || TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(pVCpu, GCPtrEff, uint16_t) ))
    801 #  endif
     801#     endif
    802802    {
    803803        /*
    … …  
    835835       outdated page pointer, or other troubles.  (This will do a TLB load.) */
    836836    Log12Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrEff));
    837 #  endif
     837#     endif
    838838    RT_CONCAT3(iemMemStoreStack,TMPL_MEM_FN_SUFF,SRegSafeJmp)(pVCpu, GCPtrMem, uValue);
    839839}
    … …  
    854854                && pVCpu->cpum.GstCtx.ss.u64Base == 0));
    855855
    856 #  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     856#   if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
    857857    /*
    858858     * Check that the item doesn't cross a page boundrary.
    859859     */
    860 #  if TMPL_MEM_TYPE_SIZE > 1
     860#    if TMPL_MEM_TYPE_SIZE > 1
    861861    if (RT_LIKELY(TMPL_MEM_ALIGN_CHECK(GCPtrMem)))
    862 #  endif
     862#    endif
    863863    {
    864864        /*
    … …  
    897897       outdated page pointer, or other troubles.  (This will do a TLB load.) */
    898898    Log12Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrMem));
    899 #  endif
     899#   endif
    900900    RT_CONCAT3(iemMemStoreStack,TMPL_MEM_FN_SUFF,SafeJmp)(pVCpu, GCPtrMem, uValue);
    901901}
    … …  
    912912                                                          TMPL_MEM_TYPE uValue) IEM_NOEXCEPT_MAY_LONGJMP
    913913{
    914 #  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
    915     /*
    916      * Calculate the new stack pointer and check that the item doesn't cross a page boundrary.
     914#    if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     915    /*
     916     * Check that the item doesn't cross a page boundrary.
    917917     */
    918918    if (RT_LIKELY(   !(GCPtrMem & (sizeof(uint16_t) - 1))
    … …  
    954954       outdated page pointer, or other troubles.  (This will do a TLB load.) */
    955955    Log12Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrMem));
    956 #  endif
     956#    endif
    957957    RT_CONCAT3(iemMemStoreStack,TMPL_MEM_FN_SUFF,SRegSafeJmp)(pVCpu, GCPtrMem, uValue);
    958958}
    959959#   endif /* TMPL_WITH_PUSH_SREG */
    960960
     961
     962/**
     963 * Stack fetch function that longjmps on error.
     964 */
     965DECL_INLINE_THROW(TMPL_MEM_TYPE)
     966RT_CONCAT3(iemMemFetchStack,TMPL_MEM_FN_SUFF,Jmp)(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
     967{
     968#   if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     969    /*
     970     * Apply segmentation to the address and check that the item doesn't cross
     971     * a page boundrary.
     972     */
     973    RTGCPTR const GCPtrEff = iemMemApplySegmentToWriteJmp(pVCpu, X86_SREG_SS, sizeof(TMPL_MEM_TYPE), GCPtrMem);
     974#    if TMPL_MEM_TYPE_SIZE > 1
     975    if (RT_LIKELY(TMPL_MEM_ALIGN_CHECK(GCPtrEff)))
     976#    endif
     977    {
     978        /*
     979         * TLB lookup.
     980         */
     981        uint64_t const uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrEff);
     982        PIEMTLBENTRY   pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
     983        if (RT_LIKELY(pTlbe->uTag == uTag))
     984        {
     985            /*
     986             * Check TLB page table level access flags.
     987             */
     988            AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
     989            uint64_t const fNoUser = (IEM_GET_CPL(pVCpu) + 1) & IEMTLBE_F_PT_NO_USER;
     990            if (RT_LIKELY(   (pTlbe->fFlagsAndPhysRev & (  IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
     991                                                         | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
     992                                                         | IEMTLBE_F_PT_NO_ACCESSED | fNoUser))
     993                          == pVCpu->iem.s.DataTlb.uTlbPhysRev))
     994            {
     995                /*
     996                 * Do the pop.
     997                 */
     998                STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
     999                Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
     1000                Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
     1001                TMPL_MEM_TYPE const uValue = *(TMPL_MEM_TYPE const *)&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
     1002                Log9Ex(LOG_GROUP_IEM_MEM,("IEM RD " TMPL_MEM_FMT_DESC " SS|%RGv: " TMPL_MEM_FMT_TYPE "\n", GCPtrEff, uValue));
     1003                return uValue;
     1004            }
     1005        }
     1006    }
     1007
     1008    /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
     1009       outdated page pointer, or other troubles.  (This will do a TLB load.) */
     1010    Log10Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrEff));
     1011#   endif
     1012    return RT_CONCAT3(iemMemFetchStack,TMPL_MEM_FN_SUFF,SafeJmp)(pVCpu, GCPtrMem);
     1013}
     1014
     1015
     1016/**
     1017 * Flat stack fetch function that longjmps on error.
     1018 */
     1019DECL_INLINE_THROW(TMPL_MEM_TYPE)
     1020RT_CONCAT3(iemMemFlatFetchStack,TMPL_MEM_FN_SUFF,Jmp)(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
     1021{
     1022#   if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
     1023    /*
     1024     * Check that the item doesn't cross a page boundrary.
     1025     */
     1026#    if TMPL_MEM_TYPE_SIZE > 1
     1027    if (RT_LIKELY(TMPL_MEM_ALIGN_CHECK(GCPtrMem)))
     1028#    endif
     1029    {
     1030        /*
     1031         * TLB lookup.
     1032         */
     1033        uint64_t const uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrMem);
     1034        PIEMTLBENTRY   pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
     1035        if (RT_LIKELY(pTlbe->uTag == uTag))
     1036        {
     1037            /*
     1038             * Check TLB page table level access flags.
     1039             */
     1040            AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
     1041            uint64_t const fNoUser = (IEM_GET_CPL(pVCpu) + 1) & IEMTLBE_F_PT_NO_USER;
     1042            if (RT_LIKELY(   (pTlbe->fFlagsAndPhysRev & (  IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
     1043                                                         | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
     1044                                                         | IEMTLBE_F_PT_NO_ACCESSED | fNoUser))
     1045                          == pVCpu->iem.s.DataTlb.uTlbPhysRev))
     1046            {
     1047                /*
     1048                 * Do the pop.
     1049                 */
     1050                STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
     1051                Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
     1052                Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
     1053                TMPL_MEM_TYPE const uValue = *(TMPL_MEM_TYPE const *)&pTlbe->pbMappingR3[GCPtrMem & GUEST_PAGE_OFFSET_MASK];
     1054                Log9Ex(LOG_GROUP_IEM_MEM,("IEM RD " TMPL_MEM_FMT_DESC " SS|%RGv: " TMPL_MEM_FMT_TYPE "\n", GCPtrMem, uValue));
     1055                return uValue;
     1056            }
     1057        }
     1058    }
     1059
     1060    /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
     1061       outdated page pointer, or other troubles.  (This will do a TLB load.) */
     1062    Log10Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrMem));
     1063#   endif
     1064    return RT_CONCAT3(iemMemFetchStack,TMPL_MEM_FN_SUFF,SafeJmp)(pVCpu, GCPtrMem);
     1065}
    9611066
    9621067

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102785 r102790  
    111111#ifdef IEMNATIVE_WITH_TLB_LOOKUP
    112112# define IEMNATIVE_WITH_TLB_LOOKUP_PUSH
     113#endif
     114#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     115# define IEMNATIVE_WITH_TLB_LOOKUP_POP
    113116#endif
    114117
    … …  
    17541757 * Used by TB code to store unsigned 8-bit data w/ segmentation.
    17551758 */
    1756 IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    17571759IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value))
    17581760{
    … …  
    18441846
    18451847/**
    1846  * Used by TB code to pop a 16-bit general purpose register off a generic stack.
    1847  */
    1848 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackPopGRegU16,(PVMCPUCC pVCpu, uint8_t iGReg))
    1849 {
    1850     iemMemStackPopGRegU16Jmp(pVCpu, iGReg); /** @todo iemMemStackPopGRegU16SafeJmp */
    1851 }
    1852 
    1853 
    1854 /**
    1855  * Used by TB code to pop a 32-bit general purpose register off a generic stack.
    1856  */
    1857 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackPopGRegU32,(PVMCPUCC pVCpu, uint8_t iGReg))
    1858 {
    1859     iemMemStackPopGRegU32Jmp(pVCpu, iGReg); /** @todo iemMemStackPopGRegU32SafeJmp */
    1860 }
    1861 
    1862 
    1863 /**
    1864  * Used by TB code to pop a 64-bit general purpose register off a generic stack.
    1865  */
    1866 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackPopGRegU64,(PVMCPUCC pVCpu, uint8_t iGReg))
    1867 {
    1868     iemMemStackPopGRegU64Jmp(pVCpu, iGReg); /** @todo iemMemStackPopGRegU64SafeJmp */
     1848 * Used by TB code to fetch an unsigned 16-bit item off a generic stack.
     1849 */
     1850IEM_DECL_NATIVE_HLP_DEF(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     1851{
     1852#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     1853    return iemMemFetchStackU16SafeJmp(pVCpu, GCPtrMem);
     1854#else
     1855    return iemMemFetchStackU16Jmp(pVCpu, GCPtrMem);
     1856#endif
     1857}
     1858
     1859
     1860/**
     1861 * Used by TB code to fetch an unsigned 32-bit item off a generic stack.
     1862 */
     1863IEM_DECL_NATIVE_HLP_DEF(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     1864{
     1865#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     1866    return iemMemFetchStackU32SafeJmp(pVCpu, GCPtrMem);
     1867#else
     1868    return iemMemFetchStackU32Jmp(pVCpu, GCPtrMem);
     1869#endif
     1870}
     1871
     1872
     1873/**
     1874 * Used by TB code to fetch an unsigned 64-bit item off a generic stack.
     1875 */
     1876IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     1877{
     1878#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     1879    return iemMemFetchStackU64SafeJmp(pVCpu, GCPtrMem);
     1880#else
     1881    return iemMemFetchStackU64Jmp(pVCpu, GCPtrMem);
     1882#endif
    18691883}
    18701884
    … …  
    20712085
    20722086/**
    2073  * Used by TB code to pop a 16-bit general purpose register off a flat 32-bit stack.
    2074  */
    2075 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlat32PopGRegU16,(PVMCPUCC pVCpu, uint8_t iGReg))
    2076 {
    2077     iemMemFlat32StackPopGRegU16Jmp(pVCpu, iGReg); /** @todo iemMemFlat32StackPopGRegU16SafeJmp */
    2078 }
    2079 
    2080 
    2081 /**
    2082  * Used by TB code to pop a 64-bit general purpose register off a flat 32-bit stack.
    2083  */
    2084 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlat32PopGRegU32,(PVMCPUCC pVCpu, uint8_t iGReg))
    2085 {
    2086     iemMemFlat32StackPopGRegU32Jmp(pVCpu, iGReg); /** @todo iemMemFlat32StackPopGRegU32SafeJmp */
    2087 }
    2088 
    2089 
    2090 /**
    2091  * Used by TB code to pop a 16-bit general purpose register off a flat 64-bit stack.
    2092  */
    2093 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlat64PopGRegU16,(PVMCPUCC pVCpu, uint8_t iGReg))
    2094 {
    2095     iemMemFlat64StackPopGRegU16Jmp(pVCpu, iGReg); /** @todo iemMemFlat64StackPopGRegU16SafeJmp */
    2096 }
    2097 
    2098 
    2099 /**
    2100  * Used by TB code to pop a 64-bit general purpose register off a flat 64-bit stack.
    2101  */
    2102 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlat64PopGRegU64,(PVMCPUCC pVCpu, uint8_t iGReg))
    2103 {
    2104     iemMemFlat64StackPopGRegU64Jmp(pVCpu, iGReg); /** @todo iemMemFlat64StackPopGRegU64SafeJmp */
     2087 * Used by TB code to fetch an unsigned 16-bit item off a generic stack.
     2088 */
     2089IEM_DECL_NATIVE_HLP_DEF(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     2090{
     2091#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     2092    return iemMemFetchStackU16SafeJmp(pVCpu, GCPtrMem);
     2093#else
     2094    return iemMemFlatFetchStackU16Jmp(pVCpu, GCPtrMem);
     2095#endif
     2096}
     2097
     2098
     2099/**
     2100 * Used by TB code to fetch an unsigned 32-bit item off a generic stack.
     2101 */
     2102IEM_DECL_NATIVE_HLP_DEF(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     2103{
     2104#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     2105    return iemMemFetchStackU32SafeJmp(pVCpu, GCPtrMem);
     2106#else
     2107    return iemMemFlatFetchStackU32Jmp(pVCpu, GCPtrMem);
     2108#endif
     2109}
     2110
     2111
     2112/**
     2113 * Used by TB code to fetch an unsigned 64-bit item off a generic stack.
     2114 */
     2115IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
     2116{
     2117#ifdef IEMNATIVE_WITH_TLB_LOOKUP_POP
     2118    return iemMemFetchStackU64SafeJmp(pVCpu, GCPtrMem);
     2119#else
     2120    return iemMemFlatFetchStackU64Jmp(pVCpu, GCPtrMem);
     2121#endif
    21052122}
    21062123
    … …  
    1157911596     * First we calculate the new RSP and the effective stack pointer value.
    1158011597     * For 64-bit mode and flat 32-bit these two are the same.
     11598     * (Code structure is very similar to that of PUSH)
    1158111599     */
    1158211600    uint8_t const cbMem       = RT_BYTE1(cBitsVarAndFlat) / 8;
    … …  
    1174511763         * Emit code to do the actual storing / fetching.
    1174611764         */
    11747         PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     11765        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
    1174811766        if (idxRegValue != UINT8_MAX)
    1174911767        {
    … …  
    1184911867#define IEM_MC_POP_GREG_U16(a_iGReg) \
    1185011868    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16,  0, 0, 0), \
    11851                                     (uintptr_t)iemNativeHlpStackPopGRegU16, pCallEntry->idxInstr)
     11869                                    (uintptr_t)iemNativeHlpStackFetchU16, pCallEntry->idxInstr)
    1185211870#define IEM_MC_POP_GREG_U32(a_iGReg) \
    1185311871    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(32,  0, 0, 0), \
    11854                                     (uintptr_t)iemNativeHlpStackPopGRegU32, pCallEntry->idxInstr)
     11872                                    (uintptr_t)iemNativeHlpStackFetchU32, pCallEntry->idxInstr)
    1185511873#define IEM_MC_POP_GREG_U64(a_iGReg) \
    1185611874    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(64,  0, 0, 0), \
    11857                                     (uintptr_t)iemNativeHlpStackPopGRegU64, pCallEntry->idxInstr)
     11875                                    (uintptr_t)iemNativeHlpStackFetchU64, pCallEntry->idxInstr)
    1185811876
    1185911877#define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) \
    1186011878    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16, 32, 0, 0), \
    11861                                     (uintptr_t)iemNativeHlpStackFlat32PopGRegU16, pCallEntry->idxInstr)
     11879                                    (uintptr_t)iemNativeHlpStackFlatFetchU16, pCallEntry->idxInstr)
    1186211880#define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) \
    1186311881    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(32, 32, 0, 0), \
    11864                                     (uintptr_t)iemNativeHlpStackFlat32PopGRegU32, pCallEntry->idxInstr)
     11882                                    (uintptr_t)iemNativeHlpStackFlatFetchU32, pCallEntry->idxInstr)
    1186511883
    1186611884#define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) \
    1186711885    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16, 64, 0, 0), \
    11868                                     (uintptr_t)iemNativeHlpStackFlat64PopGRegU16, pCallEntry->idxInstr)
     11886                                    (uintptr_t)iemNativeHlpStackFlatFetchU16, pCallEntry->idxInstr)
    1186911887#define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) \
    1187011888    off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(64, 64, 0, 0), \
    11871                                     (uintptr_t)iemNativeHlpStackFlat64PopGRegU64, pCallEntry->idxInstr)
     11889                                    (uintptr_t)iemNativeHlpStackFlatFetchU64, pCallEntry->idxInstr)
     11890
     11891
     11892DECL_FORCE_INLINE_THROW(uint32_t)
     11893iemNativeEmitStackPopUse16Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem)
     11894{
     11895    /* Use16BitSp: */
     11896#ifdef RT_ARCH_AMD64
     11897    off = iemNativeEmitLoadGprFromGpr16Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
     11898    off = iemNativeEmitAddGpr16ImmEx(pCodeBuf, off, idxRegRsp, cbMem); /* ASSUMES this does NOT modify bits [63:16]! */
     11899#else
     11900    /* bfi regrsp, regeff, #0, #16 - moves bits 15:0 from idxVarReg to idxGstTmpReg bits 15:0. */
     11901    pCodeBuf[off++] = Armv8A64MkInstrBfi(idxRegRsp, idxRegEffSp, 0, 16, false /*f64Bit*/);
     11902    /* add regeff, regrsp, #cbMem */
     11903    pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegEffSp, idxRegRsp, cbMem, false /*f64Bit*/);
     11904    /* and regeff, regeff, #0xffff */
     11905    Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
     11906    pCodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegEffSp, idxRegEffSp, 15, 0,  false /*f64Bit*/);
     11907#endif
     11908    return off;
     11909}
     11910
     11911
     11912DECL_FORCE_INLINE(uint32_t)
     11913iemNativeEmitStackPopUse32Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem)
     11914{
     11915    /* Use32BitSp: */
     11916    off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
     11917    off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegRsp, cbMem);
     11918    return off;
     11919}
     11920
    1187211921
    1187311922/** IEM_MC[|_FLAT32|_FLAT64]_POP_GREG_U16/32/64 */
    … …  
    1188711936               || (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT);
    1188811937        Assert(   pfnFunction
    11889                == (  cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlat32PopGRegU16
    11890                    : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlat32PopGRegU32
    11891                    : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlat64PopGRegU16
    11892                    : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlat64PopGRegU64
     11938               == (  cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU16
     11939                   : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU32
     11940                   : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU16
     11941                   : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU64
    1189311942                   : UINT64_C(0xc000b000a0009000) ));
    1189411943    }
    1189511944    else
    1189611945        Assert(   pfnFunction
    11897                == (  cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackPopGRegU16
    11898                    : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackPopGRegU32
    11899                    : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackPopGRegU64
     11946               == (  cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU16
     11947                   : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU32
     11948                   : cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU64
    1190011949                   : UINT64_C(0xc000b000a0009000) ));
    1190111950#endif
    … …  
    1191211961     * may end up making calls.
    1191311962     */
    11914     /** @todo we could postpone this till we make the call and reload the
    11915      * registers after returning from the call. Not sure if that's sensible or
    11916      * not, though. */
    1191711963    off = iemNativeRegFlushPendingWrites(pReNative, off);
    1191811964
    1191911965    /*
    11920      * Move/spill/flush stuff out of call-volatile registers.
    11921      * This is the easy way out. We could contain this to the tlb-miss branch
    11922      * by saving and restoring active stuff here.
    11923      */
    11924     /** @todo save+restore active registers and maybe guest shadows in tlb-miss.  */
    11925     off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /* vacate all non-volatile regs */);
    11926 
    11927     /* For now, flush the any shadow copy of the guest register that is about
    11928        to be popped and the xSP register. */
    11929     iemNativeRegFlushGuestShadows(pReNative,
    11930                                   RT_BIT_64(IEMNATIVEGSTREG_GPR(idxGReg)) | RT_BIT_64(IEMNATIVEGSTREG_GPR(X86_GREG_xSP)));
    11931 
    11932     /*
    11933      * Define labels and allocate the result register (trying for the return
    11934      * register if we can).
    11935      */
    11936     uint16_t const uTlbSeqNo        = pReNative->uTlbSeqNo++;
    11937     uint32_t const idxLabelTlbMiss  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
    11938     uint32_t const idxLabelTlbDone  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    11939 
    11940     /*
    11941      * First we try to go via the TLB.
    11942      */
    11943 //pReNative->pInstrBuf[off++] = 0xcc;
    11944     /** @todo later. */
    11945     RT_NOREF(cBitsVarAndFlat);
    11946 
    11947     /*
    11948      * Call helper to do the popping.
     11966     * Determine the effective stack pointer, for non-FLAT modes we also update RSP.
     11967     * For FLAT modes we'll do this in TlbDone as we'll be using the incoming RSP
     11968     * directly as the effective stack pointer.
     11969     * (Code structure is very similar to that of PUSH)
     11970     */
     11971    uint8_t const cbMem       = RT_BYTE1(cBitsVarAndFlat) / 8;
     11972    uint8_t const cBitsFlat   = RT_BYTE2(cBitsVarAndFlat);      RT_NOREF(cBitsFlat);
     11973    uint8_t const idxRegRsp   = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xSP),
     11974                                                                kIemNativeGstRegUse_ForUpdate, true /*fNoVolatileRegs*/);
     11975    uint8_t const idxRegEffSp = cBitsFlat != 0 ? idxRegRsp : iemNativeRegAllocTmp(pReNative, &off);
     11976    uint32_t      offFixupJumpToUseOtherBitSp = UINT32_MAX;
     11977    if (cBitsFlat != 0)
     11978    {
     11979        Assert(idxRegEffSp == idxRegRsp);
     11980        Assert(cBitsFlat == 32 || cBitsFlat == 64);
     11981        Assert(IEM_F_MODE_X86_IS_FLAT(pReNative->fExec));
     11982    }
     11983    else /** @todo We can skip the test if we're targeting pre-386 CPUs. */
     11984    {
     11985        Assert(idxRegEffSp != idxRegRsp);
     11986        uint8_t const idxRegSsAttr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(X86_SREG_SS),
     11987                                                                     kIemNativeGstRegUse_ReadOnly);
     11988#ifdef RT_ARCH_AMD64
     11989        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
     11990#else
     11991        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
     11992#endif
     11993        off = iemNativeEmitTestAnyBitsInGpr32Ex(pCodeBuf, off, idxRegSsAttr, X86DESCATTR_D);
     11994        iemNativeRegFreeTmp(pReNative, idxRegSsAttr);
     11995        offFixupJumpToUseOtherBitSp = off;
     11996        if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
     11997        {
     11998/** @todo can skip idxRegRsp updating when popping ESP.   */
     11999            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /*8-bit suffices*/, kIemNativeInstrCond_e); /* jump if zero */
     12000            off = iemNativeEmitStackPopUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
     12001        }
     12002        else
     12003        {
     12004            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /*8-bit suffices*/, kIemNativeInstrCond_ne); /* jump if not zero */
     12005            off = iemNativeEmitStackPopUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
     12006        }
     12007        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     12008    }
     12009    /* SpUpdateEnd: */
     12010    uint32_t const offLabelSpUpdateEnd = off;
     12011
     12012    /*
     12013     * Okay, now prepare for TLB lookup and jump to code (or the TlbMiss if
     12014     * we're skipping lookup).
     12015     */
     12016    uint8_t const  iSegReg           = cBitsFlat != 0 ? UINT8_MAX : X86_SREG_SS;
     12017    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, idxRegEffSp, &off, iSegReg, cbMem);
     12018    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
     12019    uint32_t const idxLabelTlbMiss   = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
     12020    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
     12021                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
     12022                                     : UINT32_MAX;
     12023    /** @todo can do a better job picking the register here. For cbMem >= 4 this
     12024     *        will be the resulting register value. */
     12025    uint8_t const  idxRegMemResult   = iemNativeRegAllocTmp(pReNative, &off); /* pointer then value */
     12026
     12027    if (!TlbState.fSkip)
     12028        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
     12029    else
     12030        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbMiss); /** @todo short jump */
     12031
     12032    /*
     12033     * Use16BitSp:
     12034     */
     12035    if (cBitsFlat == 0)
     12036    {
     12037#ifdef RT_ARCH_AMD64
     12038        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
     12039#else
     12040        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
     12041#endif
     12042        iemNativeFixupFixedJump(pReNative, offFixupJumpToUseOtherBitSp, off);
     12043        if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
     12044            off = iemNativeEmitStackPopUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
     12045        else
     12046            off = iemNativeEmitStackPopUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
     12047        off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, offLabelSpUpdateEnd);
     12048        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     12049    }
     12050
     12051    /*
     12052     * TlbMiss:
     12053     *
     12054     * Call helper to do the pushing.
    1194912055     */
    1195012056    iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
    … …  
    1195612062#endif
    1195712063
    11958     /* IEMNATIVE_CALL_ARG1_GREG = iGReg */
    11959     off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxGReg);
     12064    uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
     12065                                     | (idxRegMemResult < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegMemResult) : 0)
     12066                                     | (idxRegEffSp != idxRegRsp ? RT_BIT_32(idxRegEffSp) : 0);
     12067    off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
     12068
     12069
     12070    /* IEMNATIVE_CALL_ARG1_GREG = EffSp/RSP */
     12071    if (idxRegEffSp != IEMNATIVE_CALL_ARG1_GREG)
     12072        off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxRegEffSp);
    1196012073
    1196112074    /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
    … …  
    1196512078    off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
    1196612079
    11967     iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     12080    /* Move the return register content to idxRegMemResult. */
     12081    if (idxRegMemResult != IEMNATIVE_CALL_RET_GREG)
     12082        off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
     12083
     12084    /* Restore variables and guest shadow registers to volatile registers. */
     12085    off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
     12086    off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
     12087
     12088#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     12089    if (!TlbState.fSkip)
     12090    {
     12091        /* end of TlbMiss - Jump to the done label. */
     12092        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
     12093        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
     12094
     12095        /*
     12096         * TlbLookup:
     12097         */
     12098        off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, cbMem - 1, IEM_ACCESS_TYPE_READ,
     12099                                     idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
     12100
     12101        /*
     12102         * Emit code to load the value (from idxRegMemResult into idxRegMemResult).
     12103         */
     12104        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
     12105        switch (cbMem)
     12106        {
     12107            case 2:
     12108                off = iemNativeEmitLoadGprByGprU16Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
     12109                break;
     12110            case 4:
     12111                off = iemNativeEmitLoadGprByGprU32Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
     12112                break;
     12113            case 8:
     12114                off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
     12115                break;
     12116            default:
     12117                AssertFailed();
     12118        }
     12119
     12120        TlbState.freeRegsAndReleaseVars(pReNative);
     12121
     12122        /*
     12123         * TlbDone:
     12124         *
     12125         * Set the new RSP value (FLAT accesses needs to calculate it first) and
     12126         * commit the popped register value.
     12127         */
     12128        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     12129    }
     12130#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
     12131
     12132    if (idxGReg != X86_GREG_xSP)
     12133    {
     12134        /* Set the register. */
     12135        if (cbMem >= sizeof(uint32_t))
     12136        {
     12137            iemNativeRegClearAndMarkAsGstRegShadow(pReNative, idxRegMemResult,  IEMNATIVEGSTREG_GPR(idxGReg), off);
     12138            off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegMemResult,
     12139                                                 RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[idxGReg]));
     12140        }
     12141        else
     12142        {
     12143            Assert(cbMem == sizeof(uint16_t));
     12144            uint8_t const idxRegDst = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGReg),
     12145                                                                      kIemNativeGstRegUse_ForUpdate);
     12146            off = iemNativeEmitGprMergeInGpr16(pReNative, off, idxRegDst, idxRegMemResult);
     12147            off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegDst, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[idxGReg]));
     12148            iemNativeRegFreeTmp(pReNative, idxRegDst);
     12149        }
     12150
     12151        /* Complete RSP calculation for FLAT mode. */
     12152        if (idxRegEffSp == idxRegRsp)
     12153        {
     12154            if (cBitsFlat == 64)
     12155                off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRsp, sizeof(uint64_t));
     12156            else
     12157                off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxRegRsp, sizeof(uint32_t));
     12158        }
     12159    }
     12160    else
     12161    {
     12162        /* We're popping RSP, ESP or SP. Only the is a bit extra work, of course. */
     12163        if (cbMem == sizeof(uint64_t))
     12164            off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegRsp, idxRegMemResult);
     12165        else if (cbMem == sizeof(uint32_t))
     12166            off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRsp, idxRegMemResult);
     12167        else
     12168        {
     12169            if (idxRegEffSp == idxRegRsp)
     12170            {
     12171                if (cBitsFlat == 64)
     12172                    off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRsp, sizeof(uint64_t));
     12173                else
     12174                    off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxRegRsp, sizeof(uint32_t));
     12175            }
     12176            off = iemNativeEmitGprMergeInGpr16(pReNative, off, idxRegRsp, idxRegMemResult);
     12177        }
     12178    }
     12179    off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegRsp, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rsp));
     12180
     12181    iemNativeRegFreeTmp(pReNative, idxRegRsp);
     12182    if (idxRegEffSp != idxRegRsp)
     12183        iemNativeRegFreeTmp(pReNative, idxRegEffSp);
     12184    iemNativeRegFreeTmp(pReNative, idxRegMemResult);
    1196812185
    1196912186    return off;

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r102766 r102790  
    52625262void            iemMemStoreStackU64SafeJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t uValue) IEM_NOEXCEPT_MAY_LONGJMP;
    52635263
     5264uint16_t        iemMemFetchStackU16SafeJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
     5265uint32_t        iemMemFetchStackU32SafeJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
     5266uint64_t        iemMemFetchStackU64SafeJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
    52645267
    52655268#endif

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102785 r102790  
    14271427
    14281428
     1429/**
     1430 * Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
     1431 * destination.
     1432 */
     1433DECL_FORCE_INLINE(uint32_t)
     1434iemNativeEmitGprMergeInGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
     1435{
     1436#ifdef RT_ARCH_AMD64
     1437    /* mov reg16, r/m16 */
     1438    pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     1439    if (idxDst >= 8 || idxSrc >= 8)
     1440        pCodeBuf[off++] = (idxDst < 8 ? 0 : X86_OP_REX_R) | (idxSrc < 8 ? 0 : X86_OP_REX_B);
     1441    pCodeBuf[off++] = 0x8b;
     1442    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxDst & 7, idxSrc & 7);
     1443
     1444#elif defined(RT_ARCH_ARM64)
     1445    /* bfi w1, w2, 0, 16 - moves bits 15:0 from idxSrc to idxDst bits 15:0. */
     1446    pCodeBuf[off++] = Armv8A64MkInstrBfi(idxDst, idxSrc, 0, 16);
     1447
     1448#else
     1449# error "Port me!"
     1450#endif
     1451    return off;
     1452}
     1453
     1454
     1455/**
     1456 * Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
     1457 * destination.
     1458 */
     1459DECL_INLINE_THROW(uint32_t)
     1460iemNativeEmitGprMergeInGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
     1461{
     1462#ifdef RT_ARCH_AMD64
     1463    off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, idxDst, idxSrc);
     1464#elif defined(RT_ARCH_ARM64)
     1465    off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, idxDst, idxSrc);
     1466#else
     1467# error "Port me!"
     1468#endif
     1469    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1470    return off;
     1471}
     1472
    14291473
    14301474#ifdef RT_ARCH_AMD64
    … …  
    28142858 * so not suitable as a base for conditional jumps.
    28152859 *
     2860 * @note AMD64: Will only update the lower 16 bits of the register.
    28162861 * @note ARM64: Will update the entire register.
    2817  * @note AMD64: May perhaps only update the lower 16 bits of the register.
    28182862 * @note ARM64: Larger constants will require a temporary register.  Failing to
    28192863 *       specify one when needed will trigger fatal assertion / throw.
    … …  
    33263370    return off;
    33273371}
     3372
     3373
     3374/**
     3375 * Emits a 16-bit GPR add with a signed immediate addend.
     3376 *
     3377 * This will optimize using INC/DEC/whatever and ARM64 will not set flags,
     3378 * so not suitable as a base for conditional jumps.
     3379 *
     3380 * @note AMD64: Will only update the lower 16 bits of the register.
     3381 * @note ARM64: Will update the entire register.
     3382 * @note ARM64: Larger constants will require a temporary register.  Failing to
     3383 *       specify one when needed will trigger fatal assertion / throw.
     3384 * @sa   iemNativeEmitSubGpr16ImmEx
     3385 */
     3386DECL_FORCE_INLINE_THROW(uint32_t)
     3387iemNativeEmitAddGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iAddend,
     3388                           uint8_t iGprTmp = UINT8_MAX)
     3389{
     3390#ifdef RT_ARCH_AMD64
     3391    pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     3392    if (iGprDst >= 8)
     3393        pCodeBuf[off++] = X86_OP_REX_B;
     3394    if (iAddend == 1)
     3395    {
     3396        /* inc r/m16 */
     3397        pCodeBuf[off++] = 0xff;
     3398        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     3399    }
     3400    else if (iAddend == -1)
     3401    {
     3402        /* dec r/m16 */
     3403        pCodeBuf[off++] = 0xff;
     3404        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
     3405    }
     3406    else if ((int8_t)iAddend == iAddend)
     3407    {
     3408        /* add r/m16, imm8 */
     3409        pCodeBuf[off++] = 0x83;
     3410        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     3411        pCodeBuf[off++] = (uint8_t)iAddend;
     3412    }
     3413    else
     3414    {
     3415        /* add r/m16, imm16 */
     3416        pCodeBuf[off++] = 0x81;
     3417        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     3418        pCodeBuf[off++] = RT_BYTE1((uint16_t)iAddend);
     3419        pCodeBuf[off++] = RT_BYTE2((uint16_t)iAddend);
     3420    }
     3421    RT_NOREF(iGprTmp);
     3422
     3423#elif defined(RT_ARCH_ARM64)
     3424    uint32_t uAbsAddend = RT_ABS(iAddend);
     3425    if (uAbsAddend < 4096)
     3426    {
     3427        if (iAddend >= 0)
     3428            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend, false /*f64Bit*/);
     3429        else
     3430            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend, false /*f64Bit*/);
     3431    }
     3432    else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
     3433    {
     3434        if (iAddend >= 0)
     3435            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
     3436                                                       false /*f64Bit*/, false /*fSetFlags*/, true /*fShift*/);
     3437        else
     3438            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
     3439                                                       false /*f64Bit*/, false /*fSetFlags*/, true /*fShift*/);
     3440    }
     3441    else if (iGprTmp != UINT8_MAX)
     3442    {
     3443        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iAddend);
     3444        pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp, false /*f64Bit*/);
     3445    }
     3446    else
     3447# ifdef IEM_WITH_THROW_CATCH
     3448        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     3449# else
     3450        AssertReleaseFailedStmt(off = UINT32_MAX);
     3451# endif
     3452    pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /*f64Bit*/);
     3453
     3454#else
     3455# error "Port me"
     3456#endif
     3457    return off;
     3458}
     3459
    33283460
    33293461
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette