VirtualBox

Changeset 102850 in vbox


Ignore:
Timestamp:
Jan 12, 2024 12:47:47 AM (11 months ago)
Author:
vboxsync
Message:

VMM/IEM: Implemented the first of two code TLB lookups. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAll.cpp

    r102686 r102850  
    965965                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
    966966            }
     967        }
     968
     969        /*
     970         * Set the accessed flags.
     971         * ASSUMES this is set when the address is translated rather than on commit...
     972         */
     973        /** @todo testcase: check when the A bit are actually set by the CPU for code. */
     974        if (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED)
     975        {
     976            int rc2 = PGMGstModifyPage(pVCpu, GCPtrFirst, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
     977            AssertRC(rc2);
     978            /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
     979            Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
     980            pTlbe->fFlagsAndPhysRev &= ~IEMTLBE_F_PT_NO_ACCESSED;
    967981        }
    968982

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r102846 r102850  
    5555#include "IEMN8veRecompiler.h"
    5656#include "IEMN8veRecompilerEmit.h"
     57#include "IEMN8veRecompilerTlbLookup.h"
    5758
    5859
    … …  
    9091IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
    9192{
    92     STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
     93    STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
    9394    pVCpu->iem.s.pbInstrBuf       = NULL;
    9495    pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
    … …  
    11891190     * Define labels and allocate the register for holding the GCPhys of the new page.
    11901191     */
    1191     uint16_t const uTlbSeqNo        = pReNative->uTlbSeqNo++;
    1192     uint32_t const idxLabelTlbMiss  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
    1193     uint32_t const idxLabelTlbDone  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    1194     uint32_t const idxRegGCPhys     = iemNativeRegAllocTmp(pReNative, &off);
     1192    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
     1193    uint32_t const idxRegGCPhys      = iemNativeRegAllocTmp(pReNative, &off);
     1194    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
     1195    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
     1196                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
     1197                                     : UINT32_MAX;
     1198
     1199    //off = iemNativeEmitBrk(pReNative, off, 0x1111);
    11951200
    11961201    /*
    1197      * First we try to go via the TLB.
     1202     * Jump to the TLB lookup code.
    11981203     */
    1199     /** @todo  */
     1204    if (!TlbState.fSkip)
     1205        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
    12001206
    12011207    /*
    1202      * TLB miss: Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
     1208     * TlbMiss:
     1209     *
     1210     * Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
    12031211     */
    1204     iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
     1212    uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
    12051213
    12061214    /* Save variables in volatile registers. */
    1207     uint32_t const fHstRegsNotToSave = /*TlbState.getRegsNotToSave() | */ RT_BIT_32(idxRegGCPhys);
     1215    uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() | RT_BIT_32(idxRegGCPhys);
    12081216    off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
    12091217
    … …  
    12231231    /* Restore variables and guest shadow registers to volatile registers. */
    12241232    off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
    1225     off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, 0 /*TlbState.getActiveRegsWithShadows()*/);
    1226 
    1227     iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     1233    off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /*fCode*/));
     1234
     1235#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     1236    if (!TlbState.fSkip)
     1237    {
     1238        /* end of TlbMiss - Jump to the done label. */
     1239        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
     1240        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
     1241
     1242        /*
     1243         * TlbLookup:
     1244         */
     1245        off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
     1246                                            IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
     1247                                            1 /*cbMem*/, 0 /*fAlignMask*/, IEM_ACCESS_TYPE_EXEC,
     1248                                            idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
     1249
     1250# ifdef VBOX_WITH_STATISTICS
     1251        off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
     1252                                                RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
     1253# endif
     1254
     1255        /*
     1256         * TlbDone:
     1257         */
     1258        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     1259        TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /*idxVarGCPtrMem*/, true /*fIsCode*/);
     1260    }
     1261#else
     1262    RT_NOREF(idxLabelTlbMiss);
     1263#endif
    12281264
    12291265    /*
    … …  
    12511287    RT_NOREF(a_cbInstr); \
    12521288    off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
    1253 
    1254 #if 0
    1255 do { \
    1256         /* Is RIP within the current code page? */ \
    1257         Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu)); \
    1258         uint64_t const uPc = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base; \
    1259         uint64_t const off = uPc - pVCpu->iem.s.uInstrBufPc; \
    1260         if (off < pVCpu->iem.s.cbInstrBufTotal) \
    1261         { \
    1262             Assert(!(pVCpu->iem.s.GCPhysInstrBuf & GUEST_PAGE_OFFSET_MASK)); \
    1263             Assert(pVCpu->iem.s.pbInstrBuf); \
    1264             RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(a_pTb, a_idxRange) \
    1265                                                      | pTb->aRanges[(a_idxRange)].offPhysPage; \
    1266             if (GCPhysRangePageWithOffset == pVCpu->iem.s.GCPhysInstrBuf + off) \
    1267             { /* we're good */ } \
    1268             else \
    1269             { \
    1270                 Log7(("TB jmp miss: %p at %04x:%08RX64 LB %u; branching/1; GCPhysWithOffset=%RGp expected %RGp, pbInstrBuf=%p - #%u\n", \
    1271                       (a_pTb), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (a_cbInstr), \
    1272                       pVCpu->iem.s.GCPhysInstrBuf + off, GCPhysRangePageWithOffset, pVCpu->iem.s.pbInstrBuf, __LINE__)); \
    1273                 RT_NOREF(a_cbInstr); \
    1274                 STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckBranchMisses); \
    1275                 return VINF_IEM_REEXEC_BREAK; \
    1276             } \
    1277         } \
    1278         else \
    1279         { \
    1280             /* Must translate new RIP. */ \
    1281             pVCpu->iem.s.pbInstrBuf       = NULL; \
    1282             pVCpu->iem.s.offCurInstrStart = 0; \
    1283             pVCpu->iem.s.offInstrNextByte = 0; \
    1284             iemOpcodeFetchBytesJmp(pVCpu, 0, NULL); \
    1285             Assert(!(pVCpu->iem.s.GCPhysInstrBuf & GUEST_PAGE_OFFSET_MASK) || !pVCpu->iem.s.pbInstrBuf); \
    1286             \
    1287             RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(a_pTb, a_idxRange) \
    1288                                                      | pTb->aRanges[(a_idxRange)].offPhysPage; \
    1289             uint64_t const offNew                    = uPc - pVCpu->iem.s.uInstrBufPc; \
    1290             if (   GCPhysRangePageWithOffset == pVCpu->iem.s.GCPhysInstrBuf + offNew \
    1291                 && pVCpu->iem.s.pbInstrBuf) \
    1292             { /* likely */ } \
    1293             else \
    1294             { \
    1295                 Log7(("TB jmp miss: %p at %04x:%08RX64 LB %u; branching/2; GCPhysWithOffset=%RGp expected %RGp, pbInstrBuf=%p - #%u\n", \
    1296                       (a_pTb), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (a_cbInstr), \
    1297                       pVCpu->iem.s.GCPhysInstrBuf + offNew, GCPhysRangePageWithOffset, pVCpu->iem.s.pbInstrBuf, __LINE__)); \
    1298                 RT_NOREF(a_cbInstr); \
    1299                 STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckBranchMisses); \
    1300                 return VINF_IEM_REEXEC_BREAK; \
    1301             } \
    1302         } \
    1303     } while(0)
    1304 #endif
    13051289
    13061290DECL_FORCE_INLINE(uint32_t)

  • trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

    r102841 r102850  
    343343#  ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
    344344        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    345                         "Code TLB native misses on new page",       "/IEM/CPU%u/CodeTlb-Misses-New-Page", idCpu);
     345                        "Code TLB native misses on new page",           "/IEM/CPU%u/CodeTlb-Misses-New-Page", idCpu);
     346        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     347                        "Code TLB native misses on new page w/ offset", "/IEM/CPU%u/CodeTlb-Misses-New-Page-With-Offset", idCpu);
     348        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeCodeTlbHitsForNewPage, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     349                        "Code TLB native hits on new page",   "/IEM/CPU%u/CodeTlb-Hits-New-Page", idCpu);
     350        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeCodeTlbHitsForNewPageWithOffset, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     351                        "Code TLB native hits on new page /w offset",   "/IEM/CPU%u/CodeTlb-Hits-New-Page-With-Offset", idCpu);
    346352#  endif
    347353# endif

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r102841 r102850  
    17451745    /** Native recompiled execution: Code TLB misses for new page. */
    17461746    STAMCOUNTER             StatNativeCodeTlbMissesNewPage;
    1747     uint64_t                au64Padding[6];
     1747    /** Native recompiled execution: Code TLB hits for new page. */
     1748    STAMCOUNTER             StatNativeCodeTlbHitsForNewPage;
     1749    /** Native recompiled execution: Code TLB misses for new page with offset. */
     1750    STAMCOUNTER             StatNativeCodeTlbMissesNewPageWithOffset;
     1751    /** Native recompiled execution: Code TLB hits for new page with offset. */
     1752    STAMCOUNTER             StatNativeCodeTlbHitsForNewPageWithOffset;
     1753    uint64_t                au64Padding[3];
    17481754    /** @} */
    17491755

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102803 r102850  
    710710 * Emits a store of a GPR value to a 64-bit VCpu field.
    711711 */
     712DECL_FORCE_INLINE_THROW(uint32_t)
     713iemNativeEmitStoreGprToVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu,
     714                                 uint8_t iGprTmp = UINT8_MAX)
     715{
     716#ifdef RT_ARCH_AMD64
     717    /* mov mem64, reg64 */
     718    if (iGpr < 8)
     719        pCodeBuf[off++] = X86_OP_REX_W;
     720    else
     721        pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     722    pCodeBuf[off++] = 0x89;
     723    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
     724    RT_NOREF(iGprTmp);
     725
     726#elif defined(RT_ARCH_ARM64)
     727    off = iemNativeEmitGprByVCpuLdStEx(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
     728
     729#else
     730# error "port me"
     731#endif
     732    return off;
     733}
     734
     735
     736/**
     737 * Emits a store of a GPR value to a 64-bit VCpu field.
     738 */
    712739DECL_INLINE_THROW(uint32_t)
    713740iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
    714741{
    715742#ifdef RT_ARCH_AMD64
    716     /* mov mem64, reg64 */
    717     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    718     if (iGpr < 8)
    719         pbCodeBuf[off++] = X86_OP_REX_W;
    720     else
    721         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
    722     pbCodeBuf[off++] = 0x89;
    723     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
    724     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    725 
    726 #elif defined(RT_ARCH_ARM64)
    727     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
    728 
    729 #else
    730 # error "port me"
    731 #endif
     743    off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
     744#elif defined(RT_ARCH_ARM64)
     745    off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGpr, offVCpu,
     746                                           IEMNATIVE_REG_FIXED_TMP0);
     747#else
     748# error "port me"
     749#endif
     750    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    732751    return off;
    733752}
    … …  
    802821#elif defined(RT_ARCH_ARM64)
    803822    off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
     823
     824#else
     825# error "port me"
     826#endif
     827    return off;
     828}
     829
     830
     831/**
     832 * Emits a store of an immediate value to a 16-bit VCpu field.
     833 *
     834 * @note ARM64: A idxTmp1 is always required! The idxTmp2 depends on whehter the
     835 *       offset can be encoded as an immediate or not.  The @a offVCpu immediate
     836 *       range is 0..8190 bytes from VMCPU and the same from CPUMCPU.
     837 */
     838DECL_FORCE_INLINE_THROW(uint32_t)
     839iemNativeEmitStoreImmToVCpuU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint32_t offVCpu,
     840                                 uint8_t idxTmp1 = UINT8_MAX, uint8_t idxTmp2 = UINT8_MAX)
     841{
     842#ifdef RT_ARCH_AMD64
     843    /* mov mem16, imm16 */
     844    pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     845    pCodeBuf[off++] = 0xc7;
     846    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
     847    pCodeBuf[off++] = RT_BYTE1(uImm);
     848    pCodeBuf[off++] = RT_BYTE2(uImm);
     849    RT_NOREF(idxTmp1, idxTmp2);
     850
     851#elif defined(RT_ARCH_ARM64)
     852    if (idxTmp1 != UINT8_MAX)
     853    {
     854        pCodeBuf[off++] = Armv8A64MkInstrMovZ(idxTmp1, uImm);
     855        off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, idxTmp1, offVCpu, kArmv8A64InstrLdStType_St_Half,
     856                                           sizeof(uint16_t), idxTmp2);
     857    }
     858    else
     859# ifdef IEM_WITH_THROW_CATCH
     860        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     861# else
     862        AssertReleaseFailedStmt(off = UINT32_MAX);
     863# endif
    804864
    805865#else
    … …  
    38573917 *       and ARM64 hosts.
    38583918 */
    3859 DECL_INLINE_THROW(uint32_t)
    3860 iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     3919DECL_FORCE_INLINE(uint32_t)
     3920iemNativeEmitAndGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
    38613921{
    38623922#if defined(RT_ARCH_AMD64)
    38633923    /* and Gv, Ev */
    3864     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    3865     pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
    3866     pbCodeBuf[off++] = 0x23;
    3867     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     3924    pCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     3925    pCodeBuf[off++] = 0x23;
     3926    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    38683927    RT_NOREF(fSetFlags);
    38693928
    38703929#elif defined(RT_ARCH_ARM64)
    3871     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    38723930    if (!fSetFlags)
    3873         pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
    3874     else
    3875         pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
    3876 
     3931        pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
     3932    else
     3933        pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
     3934
     3935#else
     3936# error "Port me"
     3937#endif
     3938    return off;
     3939}
     3940
     3941
     3942/**
     3943 * Emits code for AND'ing two 64-bit GPRs.
     3944 *
     3945 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
     3946 *       and ARM64 hosts.
     3947 */
     3948DECL_INLINE_THROW(uint32_t)
     3949iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     3950{
     3951#if defined(RT_ARCH_AMD64)
     3952    off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
     3953#elif defined(RT_ARCH_ARM64)
     3954    off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
    38773955#else
    38783956# error "Port me"
    … …  
    40874165}
    40884166
     4167
     4168/**
     4169 * Emits code for AND'ing an 64-bit GPRs with a constant.
     4170 *
     4171 * @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
     4172 *       encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
     4173 *       the same.
     4174 */
     4175DECL_FORCE_INLINE_THROW(uint32_t)
     4176iemNativeEmitGprEqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint64_t uImm,
     4177                              bool fSetFlags = false)
     4178{
     4179#if defined(RT_ARCH_AMD64)
     4180    off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
     4181    off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc);
     4182    RT_NOREF(fSetFlags);
     4183
     4184#elif defined(RT_ARCH_ARM64)
     4185    uint32_t uImmR     = 0;
     4186    uint32_t uImmNandS = 0;
     4187    if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
     4188    {
     4189        if (!fSetFlags)
     4190            pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR);
     4191        else
     4192            pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR);
     4193    }
     4194    else if (iGprDst != iGprSrc)
     4195    {
     4196        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
     4197        off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
     4198    }
     4199    else
     4200# ifdef IEM_WITH_THROW_CATCH
     4201        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     4202# else
     4203        AssertReleaseFailedStmt(off = UINT32_MAX);
     4204# endif
     4205
     4206#else
     4207# error "Port me"
     4208#endif
     4209    return off;
     4210}
    40894211
    40904212/**

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h

    r102847 r102850  
    160160    }
    161161
    162     void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX) const
    163     {
    164         if (idxRegPtr != UINT8_MAX)
    165         {
    166             if (idxRegPtrHlp == UINT8_MAX)
     162    /* Alternative constructor for the code TLB lookups where we implictly use RIP
     163       variable, only a register derived from the guest RSP. */
     164    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, bool a_fFlat, uint32_t *a_poff)
     165#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     166        :           fSkip(false)
     167#else
     168        :           fSkip(true)
     169#endif
     170        ,    idxRegPtrHlp(UINT8_MAX)
     171        ,       idxRegPtr(iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, kIemNativeGstReg_Pc))
     172        ,   idxRegSegBase(a_fFlat || fSkip
     173                          ? UINT8_MAX
     174                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS)))
     175        ,  idxRegSegLimit(/*a_fFlat || fSkip
     176                          ? UINT8_MAX
     177                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS))*/
     178                          UINT8_MAX)
     179        , idxRegSegAttrib(UINT8_MAX)
     180        ,         idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     181        ,         idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     182#if defined(RT_ARCH_ARM64)
     183        ,         idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     184#endif
     185        ,         uAbsPtr(UINT64_MAX)
     186
     187    {
     188    }
     189
     190    void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX, bool fIsCode = false) const
     191    {
     192        if (!fIsCode)
     193        {
     194            if (idxRegPtr != UINT8_MAX)
    167195            {
    168                 if (idxVarGCPtrMem != UINT8_MAX)
    169                     iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     196                if (idxRegPtrHlp == UINT8_MAX)
     197                {
     198                    if (idxVarGCPtrMem != UINT8_MAX)
     199                        iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     200                }
     201                else
     202                {
     203                    Assert(idxRegPtrHlp == idxRegPtr);
     204                    iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
     205                }
    170206            }
    171207            else
    172             {
    173                 Assert(idxRegPtrHlp == idxRegPtr);
    174                 iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
    175             }
    176         }
    177         else
     208                Assert(idxRegPtrHlp == UINT8_MAX);
     209        }
     210        else
     211        {
     212            Assert(idxVarGCPtrMem == UINT8_MAX);
    178213            Assert(idxRegPtrHlp == UINT8_MAX);
     214            iemNativeRegFreeTmp(a_pReNative, idxRegPtr); /* RIP */
     215        }
    179216        if (idxRegSegBase != UINT8_MAX)
    180217            iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
    181218        if (idxRegSegLimit != UINT8_MAX)
    182         {
    183219            iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
     220        if (idxRegSegAttrib != UINT8_MAX)
    184221            iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
    185         }
    186         else
    187             Assert(idxRegSegAttrib == UINT8_MAX);
    188222#if defined(RT_ARCH_ARM64)
    189223        iemNativeRegFreeTmp(a_pReNative, idxReg3);
    … …  
    207241
    208242    /** This is only for avoid assertions. */
    209     uint32_t getActiveRegsWithShadows() const
     243    uint32_t getActiveRegsWithShadows(bool fCode = false) const
    210244    {
    211245#ifdef VBOX_STRICT
    212246        if (!fSkip)
    213             return RT_BIT_32(idxRegSegBase) | RT_BIT_32(idxRegSegLimit) | RT_BIT_32(idxRegSegAttrib);
     247            return (idxRegSegBase   != UINT8_MAX ? RT_BIT_32(idxRegSegBase)   : 0)
     248                 | (idxRegSegLimit  != UINT8_MAX ? RT_BIT_32(idxRegSegLimit)  : 0)
     249                 | (idxRegSegAttrib != UINT8_MAX ? RT_BIT_32(idxRegSegAttrib) : 0)
     250                 | (fCode                        ? RT_BIT_32(idxRegPtr)       : 0);
    214251#endif
    215252        return 0;
    … …  
    229266{
    230267    Assert(!pTlbState->fSkip);
     268    uint32_t const   offVCpuTlb = a_fDataTlb ? RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb) : RT_UOFFSETOF(VMCPUCC, iem.s.CodeTlb);
    231269# if defined(RT_ARCH_AMD64)
    232     uint8_t * const  pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
     270    uint8_t * const  pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, 512);
    233271# elif defined(RT_ARCH_ARM64)
    234     uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     272    uint32_t * const pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, 64);
    235273# endif
    236274
    … …  
    296334     *
    297335     * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
    298      */
    299     if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     336     *
     337     *     This can be skipped for code TLB lookups because limit is checked by jmp, call,
     338     *     ret, and iret prior to making it.  It is also checked by the helpers prior to
     339     *     doing TLB loading.
     340     */
     341    if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
    300342    {
    301343        /* Check that we've got a segment loaded and that it allows the access.
    … …  
    539581    pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
    540582    pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
    541     off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     583    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
    542584# else
    543     off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     585    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
    544586    off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
    545587# endif
    … …  
    548590     * 3b. Calc pTlbe.
    549591     */
     592    uint32_t const offTlbEntries = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries);
    550593# if defined(RT_ARCH_AMD64)
    551594    /* movzx reg2, byte reg1 */
    … …  
    560603    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /*SIB*/);
    561604    pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
    562     pCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    563     pCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    564     pCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    565     pCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     605    pCodeBuf[off++] = RT_BYTE1(offTlbEntries);
     606    pCodeBuf[off++] = RT_BYTE2(offTlbEntries);
     607    pCodeBuf[off++] = RT_BYTE3(offTlbEntries);
     608    pCodeBuf[off++] = RT_BYTE4(offTlbEntries);
    566609
    567610# elif defined(RT_ARCH_ARM64)
    … …  
    569612    pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 5, 8);
    570613    /* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries) */
    571     off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.aEntries),
    572                                    pTlbState->idxReg3 /*iGprTmp*/);
     614    off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, offTlbEntries, pTlbState->idxReg3 /*iGprTmp*/);
    573615    /* reg2 += pVCpu */
    574616    off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, IEMNATIVE_REG_FIXED_PVMCPU);
    … …  
    602644    uint64_t       fTlbe   = IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PT_NO_ACCESSED
    603645                           | fNoUser;
     646    if (fAccess & IEM_ACCESS_TYPE_EXEC)
     647        fTlbe |= IEMTLBE_F_PT_NO_EXEC /*| IEMTLBE_F_PG_NO_READ?*/;
    604648    if (fAccess & IEM_ACCESS_TYPE_READ)
    605649        fTlbe |= IEMTLBE_F_PG_NO_READ;
    … …  
    618662    pCodeBuf[off++] = 0x3b;
    619663    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
    620                                     RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     664                                    offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
    621665# elif defined(RT_ARCH_ARM64)
    622666    off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
    623667                                         RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    624668    pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
    625     off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     669    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
    626670    off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
    627671# else
    … …  
    634678     * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
    635679     *    resulting pointer.
     680     *
     681     *    For code TLB lookups we have some more work to do here to set various
     682     *    IEMCPU members and we return a GCPhys address rather than a host pointer.
    636683     */
    637684    /* mov  reg1, [reg2->pbMappingR3] */
    … …  
    643690                                                      true /*f64Bit*/, idxLabelTlbMiss);
    644691
    645     if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    646     {
    647         /* and result, 0xfff */
    648         off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     692    if (a_fDataTlb)
     693    {
     694        if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     695        {
     696            /* and result, 0xfff */
     697            off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     698        }
     699        else
     700        {
     701            Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
     702            /* result = regflat & 0xfff */
     703            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
     704        }
     705
     706        /* add result, reg1 */
     707        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
    649708    }
    650709    else
    651710    {
    652         Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
    653         /* result = regflat & 0xfff */
    654         off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
    655     }
    656     /* add result, reg1 */
    657     off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
     711        /*
     712         * Code TLB use a la iemOpcodeFetchBytesJmp - keep reg2 pointing to the TLBE.
     713         *
     714         * Note. We do not need to set offCurInstrStart or offInstrNextByte.
     715         */
     716# ifdef RT_ARCH_AMD64
     717        uint8_t const idxReg3 = UINT8_MAX;
     718# else
     719        uint8_t const idxReg3 = pTlbState->idxReg3;
     720# endif
     721        /* Set pbInstrBuf first since we've got it loaded already. */
     722        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     723                                               RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf), idxReg3);
     724        /* Set uInstrBufPc to (FlatPC & ~GUEST_PAGE_OFFSET_MASK). */
     725        off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
     726        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     727                                               RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc), idxReg3);
     728        /* Set cbInstrBufTotal to GUEST_PAGE_SIZE. */ /** @todo this is a simplifications. Calc right size using CS.LIM and EIP? */
     729        off = iemNativeEmitStoreImmToVCpuU16Ex(pCodeBuf, off, GUEST_PAGE_SIZE, RT_UOFFSETOF(VMCPUCC, iem.s.cbInstrBufTotal),
     730                                               pTlbState->idxReg1, idxReg3);
     731        /* Now set GCPhysInstrBuf last as we'll be returning it in idxRegMemResult. */
     732        off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     733                                             pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys));
     734        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     735                                               RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
     736        /* Set idxRegMemResult. */
     737        if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     738            off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     739        else
     740            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
     741        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
     742    }
    658743
    659744# if 0
    … …  
    665750     */
    666751#  ifdef RT_ARCH_AMD64
    667     /* push     seg | (cbMem << 8) | (fAccess << 16) */
    668     pCodeBuf[off++] = 0x68;
    669     pCodeBuf[off++] = iSegReg;
    670     pCodeBuf[off++] = cbMem;
    671     pCodeBuf[off++] = RT_BYTE1(fAccess);
    672     pCodeBuf[off++] = RT_BYTE2(fAccess);
    673     /* push     pTlbState->idxRegPtr / immediate address. */
    674     if (pTlbState->idxRegPtr != UINT8_MAX)
    675     {
    676         if (pTlbState->idxRegPtr >= 8)
     752    if (a_fDataTlb)
     753    {
     754        /* push     seg | (cbMem << 8) | (fAccess << 16) */
     755        pCodeBuf[off++] = 0x68;
     756        pCodeBuf[off++] = iSegReg;
     757        pCodeBuf[off++] = cbMem;
     758        pCodeBuf[off++] = RT_BYTE1(fAccess);
     759        pCodeBuf[off++] = RT_BYTE2(fAccess);
     760        /* push     pTlbState->idxRegPtr / immediate address. */
     761        if (pTlbState->idxRegPtr != UINT8_MAX)
     762        {
     763            if (pTlbState->idxRegPtr >= 8)
     764                pCodeBuf[off++] = X86_OP_REX_B;
     765            pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
     766        }
     767        else
     768        {
     769            off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
     770            if (pTlbState->idxReg1 >= 8)
     771                pCodeBuf[off++] = X86_OP_REX_B;
     772            pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
     773        }
     774        /* push     idxRegMemResult */
     775        if (idxRegMemResult >= 8)
    677776            pCodeBuf[off++] = X86_OP_REX_B;
    678         pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
    679     }
    680     else
    681     {
    682         off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
    683         if (pTlbState->idxReg1 >= 8)
    684             pCodeBuf[off++] = X86_OP_REX_B;
    685         pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
    686     }
    687     /* push     idxRegMemResult */
    688     if (idxRegMemResult >= 8)
    689         pCodeBuf[off++] = X86_OP_REX_B;
    690     pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
    691     /* push     pVCpu */
    692     pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
    693     /* mov      reg1, helper */
    694     off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
    695     /* call     [reg1] */
    696     pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
    697     pCodeBuf[off++] = 0xff;
    698     pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
    699     /* The stack is cleaned up by helper function. */
     777        pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
     778        /* push     pVCpu */
     779        pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
     780        /* mov      reg1, helper */
     781        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
     782        /* call     [reg1] */
     783        pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
     784        pCodeBuf[off++] = 0xff;
     785        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
     786        /* The stack is cleaned up by helper function. */
     787    }
    700788
    701789#  else
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette