VirtualBox

Changeset 103860 in vbox for trunk/src/VBox/VMM/VMMAll


Ignore:
Timestamp:
Mar 14, 2024 11:47:09 PM (9 months ago)
Author:
vboxsync
Message:

VMM/IEM: Reworked iemNativeEmitMaybeRaiseSseRelatedXcpt to avoid relative 14-bit jumps on arm64, optimizing it while at it. Also redid much of the tail label emitting (all but a few call a helper taking only pVCpu as arg). Made the fixup code assert+barf upon short 14-bit jumps to tail TB code as those will blow up for large TBs. bugref:10614 bugref:10370

Location:
trunk/src/VBox/VMM/VMMAll
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

    r103855 r103860  
    818818         * Make sure we don't have any outstanding guest register writes as we may
    819819         * raise an #NM and all guest register must be up to date in CPUMCTX.
    820          *
    821          * @todo r=aeichner Can we postpone this to the RaiseNm path?
    822820         */
     821        /** @todo r=aeichner Can we postpone this to the RaiseNm path? */
    823822        off = iemNativeRegFlushPendingWrites(pReNative, off);
    824823
    … …  
    871870     * Make sure we don't have any outstanding guest register writes as we may
    872871     * raise an #MF and all guest register must be up to date in CPUMCTX.
    873      *
    874      * @todo r=aeichner Can we postpone this to the RaiseMf path?
    875872     */
     873    /** @todo r=aeichner Can we postpone this to the RaiseMf path? */
    876874    off = iemNativeRegFlushPendingWrites(pReNative, off);
    877875
    … …  
    923921         * Make sure we don't have any outstanding guest register writes as we may
    924922         * raise an \#UD or \#NM and all guest register must be up to date in CPUMCTX.
    925          *
    926          * @todo r=aeichner Can we postpone this to the RaiseNm/RaiseUd path?
    927923         */
     924        /** @todo r=aeichner Can we postpone this to the RaiseNm/RaiseUd path? */
    928925        off = iemNativeRegFlushPendingWrites(pReNative, off, false /*fFlushShadows*/);
    929926
    … …  
    935932
    936933        /* Allocate a temporary CR0 and CR4 register. */
    937         uint8_t const idxCr0Reg       = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
    938         uint8_t const idxCr4Reg       = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4, kIemNativeGstRegUse_ReadOnly);
    939         uint8_t const idxLabelRaiseNm = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseNm);
    940         uint8_t const idxLabelRaiseUd = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseUd);
    941 
    942         /** @todo r=aeichner Optimize this more later to have less compares and branches,
    943          *                   (see IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() in IEMMc.h but check that it has some
    944          *                   actual performance benefit first). */
     934        uint8_t const idxLabelRaiseSseRelated = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseSseRelated);
     935        uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
     936        uint8_t const idxCr4Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4, kIemNativeGstRegUse_ReadOnly);
     937        uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
     938
     939        AssertCompile(!((X86_CR0_EM | X86_CR0_TS) & X86_CR4_OSFXSR));
     940#ifdef RT_ARCH_AMD64
    945941        /*
    946          * if (cr0 & X86_CR0_EM)
    947          *     return raisexcpt();
     942         * We do a modified test here:
     943         *  if (!(((cr4 & X86_CR4_OSFXSR) | cr0) ^ X86_CR4_OSFXSR)) { likely }
     944         *  else                                                    { goto RaiseSseRelated; }
     945         * This ASSUMES that CR0[bit 9] is always zero.  This is the case on
     946         * all targets except the 386, which doesn't support SSE, this should
     947         * be a safe assumption.
    948948         */
    949         off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxCr0Reg, X86_CR0_EM_BIT, idxLabelRaiseUd);
     949        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+6+3+3+7+7+6);
     950        pCodeBuf[off++] = 0xcc;
     951        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off,    idxTmpReg, X86_CR4_OSFXSR); /* Isolate CR4.OSFXSR as CR4.TSD and */
     952        off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, idxTmpReg, idxCr4Reg);      /* CR4.DE would overlap the CR0 bits. */
     953        off = iemNativeEmitOrGpr32ByGprEx(pCodeBuf, off,    idxTmpReg, idxCr0Reg);
     954        off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR0_EM | X86_CR0_TS | X86_CR4_OSFXSR);
     955        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR4_OSFXSR);
     956        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelRaiseSseRelated, kIemNativeInstrCond_ne);
     957
     958#elif defined(RT_ARCH_ARM64)
    950959        /*
    951          * if (!(cr4 & X86_CR4_OSFXSR))
    952          *     return raisexcpt();
     960         * We do a modified test here:
     961         *  if (!((cr0 & (X86_CR0_EM | X86_CR0_TS)) | (((cr4 >> X86_CR4_OSFXSR_BIT) & 1) ^ 1))) { likely }
     962         *  else                                                                                { goto RaiseSseRelated; }
    953963         */
    954         off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxCr4Reg, X86_CR4_OSFXSR_BIT, idxLabelRaiseUd);
    955         /*
    956          * if (cr0 & X86_CR0_TS)
    957          *     return raisexcpt();
    958          */
    959         off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxCr0Reg, X86_CR0_TS_BIT, idxLabelRaiseNm);
    960 
    961         /* Free but don't flush the CR0 and CR4 register. */
     964        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+5);
     965        //pCodeBuf[off++] = Armv8A64MkInstrBrk(0x1111);
     966        Assert(Armv8A64ConvertImmRImmS2Mask32(1, 32 - X86_CR0_EM_BIT) == (X86_CR0_EM | X86_CR0_TS));
     967        pCodeBuf[off++] = Armv8A64MkInstrAndImm(idxTmpReg, idxCr0Reg, 1, 32 - X86_CR0_EM_BIT, false /*f64Bit*/);
     968        pCodeBuf[off++] = Armv8A64MkInstrBfxil(idxTmpReg, idxCr4Reg, X86_CR4_OSFXSR_BIT, 1, false /*f64Bit*/);
     969        /* -> idxTmpReg[0]=OSFXSR;  idxTmpReg[2]=EM; idxTmpReg[3]=TS; (the rest is zero) */
     970        Assert(Armv8A64ConvertImmRImmS2Mask32(0, 0) == 1);
     971        pCodeBuf[off++] = Armv8A64MkInstrEorImm(idxTmpReg, idxTmpReg, 0, 0, false /*f64Bit*/);              /* -> bit 0 = ~OSFXSR */
     972        /* -> idxTmpReg[0]=~OSFXSR; idxTmpReg[2]=EM; idxTmpReg[3]=TS; (the rest is zero) */
     973        off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
     974                                                             idxLabelRaiseSseRelated);
     975#endif
     976
     977        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     978        iemNativeRegFreeTmp(pReNative, idxTmpReg);
    962979        iemNativeRegFreeTmp(pReNative, idxCr0Reg);
    963980        iemNativeRegFreeTmp(pReNative, idxCr4Reg);
     981
    964982#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    965983        pReNative->fSimdRaiseXcptChecksEmitted |= IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE;

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r103847 r103860  
    15801580
    15811581/**
     1582 * Used by TB code when it wants to raise a \#DE.
     1583 */
     1584IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu))
     1585{
     1586    iemRaiseDivideErrorJmp(pVCpu);
     1587#ifndef _MSC_VER
     1588    return VINF_IEM_RAISED_XCPT; /* not reached */
     1589#endif
     1590}
     1591
     1592
     1593/**
     1594 * Used by TB code when it wants to raise a \#UD.
     1595 */
     1596IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu))
     1597{
     1598    iemRaiseUndefinedOpcodeJmp(pVCpu);
     1599#ifndef _MSC_VER
     1600    return VINF_IEM_RAISED_XCPT; /* not reached */
     1601#endif
     1602}
     1603
     1604
     1605/**
     1606 * Used by TB code when it wants to raise an SSE related \#UD or \#NM.
     1607 *
     1608 * See IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT.
     1609 */
     1610IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu))
     1611{
     1612    if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
     1613        || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
     1614        iemRaiseUndefinedOpcodeJmp(pVCpu);
     1615    else
     1616        iemRaiseDeviceNotAvailableJmp(pVCpu);
     1617#ifndef _MSC_VER
     1618    return VINF_IEM_RAISED_XCPT; /* not reached */
     1619#endif
     1620}
     1621
     1622
     1623/**
     1624 * Used by TB code when it wants to raise an AVX related \#UD or \#NM.
     1625 *
     1626 * See IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT.
     1627 */
     1628IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu))
     1629{
     1630    if (   (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE)
     1631        || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
     1632        iemRaiseUndefinedOpcodeJmp(pVCpu);
     1633    else
     1634        iemRaiseDeviceNotAvailableJmp(pVCpu);
     1635#ifndef _MSC_VER
     1636    return VINF_IEM_RAISED_XCPT; /* not reached */
     1637#endif
     1638}
     1639
     1640
     1641/**
     1642 * Used by TB code when it wants to raise a \#NM.
     1643 */
     1644IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu))
     1645{
     1646    iemRaiseDeviceNotAvailableJmp(pVCpu);
     1647#ifndef _MSC_VER
     1648    return VINF_IEM_RAISED_XCPT; /* not reached */
     1649#endif
     1650}
     1651
     1652
     1653/**
    15821654 * Used by TB code when it wants to raise a \#GP(0).
    15831655 */
    … …  
    15921664
    15931665/**
    1594  * Used by TB code when it wants to raise a \#NM.
    1595  */
    1596 IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu))
    1597 {
    1598     iemRaiseDeviceNotAvailableJmp(pVCpu);
    1599 #ifndef _MSC_VER
    1600     return VINF_IEM_RAISED_XCPT; /* not reached */
    1601 #endif
    1602 }
    1603 
    1604 
    1605 /**
    1606  * Used by TB code when it wants to raise a \#UD.
    1607  */
    1608 IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu))
    1609 {
    1610     iemRaiseUndefinedOpcodeJmp(pVCpu);
    1611 #ifndef _MSC_VER
    1612     return VINF_IEM_RAISED_XCPT; /* not reached */
    1613 #endif
    1614 }
    1615 
    1616 
    1617 /**
    16181666 * Used by TB code when it wants to raise a \#MF.
    16191667 */
    … …  
    16331681{
    16341682    iemRaiseSimdFpExceptionJmp(pVCpu);
    1635 #ifndef _MSC_VER
    1636     return VINF_IEM_RAISED_XCPT; /* not reached */
    1637 #endif
    1638 }
    1639 
    1640 
    1641 /**
    1642  * Used by TB code when it wants to raise a \#DE.
    1643  */
    1644 IEM_DECL_NATIVE_HLP_DEF(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu))
    1645 {
    1646     iemRaiseDivideErrorJmp(pVCpu);
    16471683#ifndef _MSC_VER
    16481684    return VINF_IEM_RAISED_XCPT; /* not reached */
    … …  
    29673003    pReNative->Core.u64ArgVars             = UINT64_MAX;
    29683004
    2969     AssertCompile(RT_ELEMENTS(pReNative->aidxUniqueLabels) == 14);
     3005    AssertCompile(RT_ELEMENTS(pReNative->aidxUniqueLabels) == 16);
    29703006    pReNative->aidxUniqueLabels[0]         = UINT32_MAX;
    29713007    pReNative->aidxUniqueLabels[1]         = UINT32_MAX;
    … …  
    29823018    pReNative->aidxUniqueLabels[12]        = UINT32_MAX;
    29833019    pReNative->aidxUniqueLabels[13]        = UINT32_MAX;
     3020    pReNative->aidxUniqueLabels[14]        = UINT32_MAX;
     3021    pReNative->aidxUniqueLabels[15]        = UINT32_MAX;
    29843022
    29853023    /* Full host register reinit: */
    … …  
    33153353    Assert(idxLabel <= UINT16_MAX);
    33163354    Assert((unsigned)enmType <= UINT8_MAX);
     3355#ifdef RT_ARCH_ARM64
     3356    AssertStmt(   enmType != kIemNativeFixupType_RelImm14At5
     3357               || pReNative->paLabels[idxLabel].enmType >= kIemNativeLabelType_FirstWithMultipleInstances
     3358               || pReNative->paLabels[idxLabel].off != UINT32_MAX,
     3359               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_FIXUP_SHORT_JMP_TO_TAIL_LABEL));
     3360#endif
    33173361
    33183362    /*
    … …  
    65386582
    65396583/**
    6540  * Emits the code at the CheckBranchMiss label.
    6541  */
    6542 static uint32_t iemNativeEmitCheckBranchMiss(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6543 {
    6544     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_CheckBranchMiss);
    6545     if (idxLabel != UINT32_MAX)
    6546     {
    6547         iemNativeLabelDefine(pReNative, idxLabel, off);
    6548 
    6549         /* int iemNativeHlpCheckBranchMiss(PVMCPUCC pVCpu) */
    6550         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6551         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpCheckBranchMiss);
    6552 
    6553         /* jump back to the return sequence. */
    6554         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6555     }
    6556     return off;
    6557 }
    6558 
    6559 
    6560 /**
    6561  * Emits the code at the NeedCsLimChecking label.
    6562  */
    6563 static uint32_t iemNativeEmitNeedCsLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6564 {
    6565     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_NeedCsLimChecking);
    6566     if (idxLabel != UINT32_MAX)
    6567     {
    6568         iemNativeLabelDefine(pReNative, idxLabel, off);
    6569 
    6570         /* int iemNativeHlpNeedCsLimChecking(PVMCPUCC pVCpu) */
    6571         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6572         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpNeedCsLimChecking);
    6573 
    6574         /* jump back to the return sequence. */
    6575         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6576     }
    6577     return off;
    6578 }
    6579 
    6580 
    6581 /**
    6582  * Emits the code at the ObsoleteTb label.
    6583  */
    6584 static uint32_t iemNativeEmitObsoleteTb(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6585 {
    6586     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_ObsoleteTb);
    6587     if (idxLabel != UINT32_MAX)
    6588     {
    6589         iemNativeLabelDefine(pReNative, idxLabel, off);
    6590 
    6591         /* int iemNativeHlpObsoleteTb(PVMCPUCC pVCpu) */
    6592         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6593         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpObsoleteTb);
    6594 
    6595         /* jump back to the return sequence. */
    6596         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6597     }
    6598     return off;
    6599 }
    6600 
    6601 
    6602 /**
    6603  * Emits the code at the RaiseGP0 label.
    6604  */
    6605 static uint32_t iemNativeEmitRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6606 {
    6607     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseGp0);
    6608     if (idxLabel != UINT32_MAX)
    6609     {
    6610         iemNativeLabelDefine(pReNative, idxLabel, off);
    6611 
    6612         /* iemNativeHlpExecRaiseGp0(PVMCPUCC pVCpu) */
    6613         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6614         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseGp0);
    6615 
    6616         /* jump back to the return sequence. */
    6617         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6618     }
    6619     return off;
    6620 }
    6621 
    6622 
    6623 /**
    6624  * Emits the code at the RaiseNm label.
    6625  */
    6626 static uint32_t iemNativeEmitRaiseNm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6627 {
    6628     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseNm);
    6629     if (idxLabel != UINT32_MAX)
    6630     {
    6631         iemNativeLabelDefine(pReNative, idxLabel, off);
    6632 
    6633         /* iemNativeHlpExecRaiseNm(PVMCPUCC pVCpu) */
    6634         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6635         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseNm);
    6636 
    6637         /* jump back to the return sequence. */
    6638         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6639     }
    6640     return off;
    6641 }
    6642 
    6643 
    6644 /**
    6645  * Emits the code at the RaiseUd label.
    6646  */
    6647 static uint32_t iemNativeEmitRaiseUd(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6648 {
    6649     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseUd);
    6650     if (idxLabel != UINT32_MAX)
    6651     {
    6652         iemNativeLabelDefine(pReNative, idxLabel, off);
    6653 
    6654         /* iemNativeHlpExecRaiseUd(PVMCPUCC pVCpu) */
    6655         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6656         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseUd);
    6657 
    6658         /* jump back to the return sequence. */
    6659         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6660     }
    6661     return off;
    6662 }
    6663 
    6664 
    6665 /**
    6666  * Emits the code at the RaiseMf label.
    6667  */
    6668 static uint32_t iemNativeEmitRaiseMf(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6669 {
    6670     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseMf);
    6671     if (idxLabel != UINT32_MAX)
    6672     {
    6673         iemNativeLabelDefine(pReNative, idxLabel, off);
    6674 
    6675         /* iemNativeHlpExecRaiseMf(PVMCPUCC pVCpu) */
    6676         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6677         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseMf);
    6678 
    6679         /* jump back to the return sequence. */
    6680         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6681     }
    6682     return off;
    6683 }
    6684 
    6685 
    6686 /**
    6687  * Emits the code at the RaiseXf label.
    6688  */
    6689 static uint32_t iemNativeEmitRaiseXf(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6690 {
    6691     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseXf);
    6692     if (idxLabel != UINT32_MAX)
    6693     {
    6694         iemNativeLabelDefine(pReNative, idxLabel, off);
    6695 
    6696         /* iemNativeHlpExecRaiseXf(PVMCPUCC pVCpu) */
    6697         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6698         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseXf);
    6699 
    6700         /* jump back to the return sequence. */
    6701         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6702     }
    6703     return off;
    6704 }
    6705 
    6706 
    6707 /**
    6708  * Emits the code at the RaiseDe label.
    6709  */
    6710 static uint32_t iemNativeEmitRaiseDe(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnLabel)
    6711 {
    6712     uint32_t const idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_RaiseDe);
    6713     if (idxLabel != UINT32_MAX)
    6714     {
    6715         iemNativeLabelDefine(pReNative, idxLabel, off);
    6716 
    6717         /* iemNativeHlpExecRaiseDe(PVMCPUCC pVCpu) */
    6718         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6719         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecRaiseDe);
    6720 
    6721         /* jump back to the return sequence. */
    6722         off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    6723     }
    6724     return off;
    6725 }
    6726 
    6727 
    6728 /**
    67296584 * Emits the code at the ReturnWithFlags label (returns
    67306585 * VINF_IEM_REEXEC_FINISH_WITH_FLAGS).
    … …  
    84118266        ENTRY(cpum.GstCtx.eflags),
    84128267        ENTRY(cpum.GstCtx.uRipInhibitInt),
     8268        ENTRY(cpum.GstCtx.cr0),
     8269        ENTRY(cpum.GstCtx.cr4),
     8270        ENTRY(cpum.GstCtx.aXcr[0]),
     8271        ENTRY(cpum.GstCtx.aXcr[1]),
    84138272#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    84148273        ENTRY(cpum.GstCtx.XState.x87.aXMM[0]),
    … …  
    87128571                            switch ((IEMNATIVELABELTYPE)pDbgInfo->aEntries[iDbgEntry].Label.enmLabel)
    87138572                            {
    8714                                 case kIemNativeLabelType_Return:
    8715                                     pszName = "Return";
    8716                                     break;
    8717                                 case kIemNativeLabelType_ReturnBreak:
    8718                                     pszName = "ReturnBreak";
    8719                                     break;
    8720                                 case kIemNativeLabelType_ReturnWithFlags:
    8721                                     pszName = "ReturnWithFlags";
    8722                                     break;
    8723                                 case kIemNativeLabelType_NonZeroRetOrPassUp:
    8724                                     pszName = "NonZeroRetOrPassUp";
    8725                                     break;
    8726                                 case kIemNativeLabelType_RaiseGp0:
    8727                                     pszName = "RaiseGp0";
    8728                                     break;
    8729                                 case kIemNativeLabelType_RaiseNm:
    8730                                     pszName = "RaiseNm";
    8731                                     break;
    8732                                 case kIemNativeLabelType_RaiseUd:
    8733                                     pszName = "RaiseUd";
    8734                                     break;
    8735                                 case kIemNativeLabelType_RaiseMf:
    8736                                     pszName = "RaiseMf";
    8737                                     break;
    8738                                 case kIemNativeLabelType_RaiseXf:
    8739                                     pszName = "RaiseXf";
    8740                                     break;
    8741                                 case kIemNativeLabelType_RaiseDe:
    8742                                     pszName = "RaiseDe";
    8743                                     break;
    8744                                 case kIemNativeLabelType_ObsoleteTb:
    8745                                     pszName = "ObsoleteTb";
    8746                                     break;
    8747                                 case kIemNativeLabelType_NeedCsLimChecking:
    8748                                     pszName = "NeedCsLimChecking";
    8749                                     break;
    8750                                 case kIemNativeLabelType_CheckBranchMiss:
    8751                                     pszName = "CheckBranchMiss";
    8752                                     break;
     8573                                case kIemNativeLabelType_Return:                pszName = "Return"; break;
     8574                                case kIemNativeLabelType_ReturnBreak:           pszName = "ReturnBreak"; break;
     8575                                case kIemNativeLabelType_ReturnWithFlags:       pszName = "ReturnWithFlags"; break;
     8576                                case kIemNativeLabelType_NonZeroRetOrPassUp:    pszName = "NonZeroRetOrPassUp"; break;
     8577                                case kIemNativeLabelType_RaiseDe:               pszName = "RaiseDe"; break;
     8578                                case kIemNativeLabelType_RaiseUd:               pszName = "RaiseUd"; break;
     8579                                case kIemNativeLabelType_RaiseSseRelated:       pszName = "RaiseSseRelated"; break;
     8580                                case kIemNativeLabelType_RaiseAvxRelated:       pszName = "RaiseAvxRelated"; break;
     8581                                case kIemNativeLabelType_RaiseNm:               pszName = "RaiseNm"; break;
     8582                                case kIemNativeLabelType_RaiseGp0:              pszName = "RaiseGp0"; break;
     8583                                case kIemNativeLabelType_RaiseMf:               pszName = "RaiseMf"; break;
     8584                                case kIemNativeLabelType_RaiseXf:               pszName = "RaiseXf"; break;
     8585                                case kIemNativeLabelType_ObsoleteTb:            pszName = "ObsoleteTb"; break;
     8586                                case kIemNativeLabelType_NeedCsLimChecking:     pszName = "NeedCsLimChecking"; break;
     8587                                case kIemNativeLabelType_CheckBranchMiss:       pszName = "CheckBranchMiss"; break;
    87538588                                case kIemNativeLabelType_If:
    87548589                                    pszName = "If";
    … …  
    93369171        if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_ReturnWithFlags))
    93379172            off = iemNativeEmitReturnWithFlags(pReNative, off, idxReturnLabel);
    9338         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseGp0))
    9339             off = iemNativeEmitRaiseGp0(pReNative, off, idxReturnLabel);
    9340         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseNm))
    9341             off = iemNativeEmitRaiseNm(pReNative, off, idxReturnLabel);
    9342         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseUd))
    9343             off = iemNativeEmitRaiseUd(pReNative, off, idxReturnLabel);
    9344         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseMf))
    9345             off = iemNativeEmitRaiseMf(pReNative, off, idxReturnLabel);
    9346         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseXf))
    9347             off = iemNativeEmitRaiseXf(pReNative, off, idxReturnLabel);
    9348         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_RaiseDe))
    9349             off = iemNativeEmitRaiseDe(pReNative, off, idxReturnLabel);
    9350         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_ObsoleteTb))
    9351             off = iemNativeEmitObsoleteTb(pReNative, off, idxReturnLabel);
    9352         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_NeedCsLimChecking))
    9353             off = iemNativeEmitNeedCsLimChecking(pReNative, off, idxReturnLabel);
    9354         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_CheckBranchMiss))
    9355             off = iemNativeEmitCheckBranchMiss(pReNative, off, idxReturnLabel);
     9173
     9174        /*
     9175         * Generate simple TB tail labels that just calls a help with a pVCpu
     9176         * arg and either return or longjmps/throws a non-zero status.
     9177         *
     9178         * The array entries must be ordered by enmLabel value so we can index
     9179         * using fTailLabels bit numbers.
     9180         */
     9181        typedef IEM_DECL_NATIVE_HLP_PTR(int, PFNIEMNATIVESIMPLETAILLABELCALL,(PVMCPUCC pVCpu));
     9182        static struct
     9183        {
     9184            IEMNATIVELABELTYPE              enmLabel;
     9185            PFNIEMNATIVESIMPLETAILLABELCALL pfnCallback;
     9186        } const g_aSimpleTailLabels[] =
     9187        {
     9188            {   kIemNativeLabelType_Invalid,                NULL },
     9189            {   kIemNativeLabelType_RaiseDe,                iemNativeHlpExecRaiseDe },
     9190            {   kIemNativeLabelType_RaiseUd,                iemNativeHlpExecRaiseUd },
     9191            {   kIemNativeLabelType_RaiseSseRelated,        iemNativeHlpExecRaiseSseRelated },
     9192            {   kIemNativeLabelType_RaiseAvxRelated,        iemNativeHlpExecRaiseAvxRelated },
     9193            {   kIemNativeLabelType_RaiseNm,                iemNativeHlpExecRaiseNm },
     9194            {   kIemNativeLabelType_RaiseGp0,               iemNativeHlpExecRaiseGp0 },
     9195            {   kIemNativeLabelType_RaiseMf,                iemNativeHlpExecRaiseMf },
     9196            {   kIemNativeLabelType_RaiseXf,                iemNativeHlpExecRaiseXf },
     9197            {   kIemNativeLabelType_ObsoleteTb,             iemNativeHlpObsoleteTb },
     9198            {   kIemNativeLabelType_NeedCsLimChecking,      iemNativeHlpNeedCsLimChecking },
     9199            {   kIemNativeLabelType_CheckBranchMiss,        iemNativeHlpCheckBranchMiss },
     9200        };
     9201        AssertCompile(RT_ELEMENTS(g_aSimpleTailLabels) == (unsigned)kIemNativeLabelType_LastSimple + 1U);
     9202        AssertCompile(kIemNativeLabelType_Invalid == 0);
     9203        uint64_t fTailLabels = pReNative->bmLabelTypes & (RT_BIT_64(kIemNativeLabelType_LastSimple + 1U) - 2U);
     9204        if (fTailLabels)
     9205        {
     9206            do
     9207            {
     9208                IEMNATIVELABELTYPE const enmLabel = (IEMNATIVELABELTYPE)(ASMBitFirstSetU64(fTailLabels) - 1U);
     9209                fTailLabels &= ~RT_BIT_64(enmLabel);
     9210                Assert(g_aSimpleTailLabels[enmLabel].enmLabel == enmLabel);
     9211
     9212                uint32_t const idxLabel = iemNativeLabelFind(pReNative, enmLabel);
     9213                Assert(idxLabel != UINT32_MAX);
     9214                if (idxLabel != UINT32_MAX)
     9215                {
     9216                    iemNativeLabelDefine(pReNative, idxLabel, off);
     9217
     9218                    /* int pfnCallback(PVMCPUCC pVCpu) */
     9219                    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     9220                    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)g_aSimpleTailLabels[enmLabel].pfnCallback);
     9221
     9222                    /* jump back to the return sequence. */
     9223                    off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
     9224                }
     9225
     9226            } while (fTailLabels);
     9227        }
    93569228    }
    93579229    IEMNATIVE_CATCH_LONGJMP_BEGIN(pReNative, rc);
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette