Index: trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllOpcodeFetch-x86.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllOpcodeFetch-x86.cpp	(revision 108244)
+++ trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllOpcodeFetch-x86.cpp	(revision 108244)
@@ -0,0 +1,1138 @@
+/* $Id$ */
+/** @file
+ * IEM - Interpreted Execution Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2011-2024 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#ifdef IN_RING0
+# define VBOX_VMM_TARGET_X86
+#endif
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/dbgf.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#include "IEMInline.h"
+#ifdef VBOX_VMM_TARGET_X86
+# include "IEMAllTlbInline-x86.h"
+#endif
+
+
+#ifndef IEM_WITH_CODE_TLB
+/**
+ * Prefetch opcodes the first time when starting executing.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling
+ *                  thread.
+ */
+VBOXSTRICTRC iemOpcodeFetchPrefetch(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     *
+     * Note! The iemOpcodeFetchMoreBytes code depends on this here code to fetch
+     *       all relevant bytes from the first page, as it ASSUMES it's only ever
+     *       called for dealing with CS.LIM, page crossing and instructions that
+     *       are too long.
+     */
+    uint32_t    cbToTryRead;
+    RTGCPTR     GCPtrPC;
+    if (IEM_IS_64BIT_CODE(pVCpu))
+    {
+        cbToTryRead = GUEST_PAGE_SIZE;
+        GCPtrPC     = pVCpu->cpum.GstCtx.rip;
+        if (IEM_IS_CANONICAL(GCPtrPC))
+            cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    else
+    {
+        uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
+        AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || IEM_IS_32BIT_CODE(pVCpu), ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+        if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
+            cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
+        else
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        if (cbToTryRead) { /* likely */ }
+        else /* overflowed */
+        {
+            Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+        }
+        GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
+        Assert(GCPtrPC <= UINT32_MAX);
+    }
+
+    PGMPTWALKFAST WalkFast;
+    int rc = PGMGstQueryPageFast(pVCpu, GCPtrPC,
+                                 IEM_GET_CPL(pVCpu) == 3 ? PGMQPAGE_F_EXECUTE | PGMQPAGE_F_USER_MODE : PGMQPAGE_F_EXECUTE,
+                                 &WalkFast);
+    if (RT_SUCCESS(rc))
+        Assert(WalkFast.fInfo & PGM_WALKINFO_SUCCEEDED);
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/** @todo This isn't quite right yet, as PGM_GST_SLAT_NAME_EPT(Walk) doesn't
+ * know about what kind of access we're making! See PGM_GST_NAME(WalkFast). */
+        if (WalkFast.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &WalkFast, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, rc);
+    }
+#if 0
+    if ((WalkFast.fEffective & X86_PTE_US) || IEM_GET_CPL(pVCpu) != 3) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/** @todo this is completely wrong for EPT. WalkFast.fFailed is always zero here!*/
+#  error completely wrong
+        if (WalkFast.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &WalkFast, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    if (!(WalkFast.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/** @todo this is completely wrong for EPT. WalkFast.fFailed is always zero here!*/
+#  error completely wrong.
+        if (WalkFast.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &WalkFast, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+#else
+    Assert((WalkFast.fEffective & X86_PTE_US) || IEM_GET_CPL(pVCpu) != 3);
+    Assert(!(WalkFast.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE));
+#endif
+    RTGCPHYS const GCPhys = WalkFast.GCPhys;
+
+    /*
+     * Read the bytes at this address.
+     */
+    uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
+    if (cbToTryRead > cbLeftOnPage)
+        cbToTryRead = cbLeftOnPage;
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
+
+    if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, rc, cbToTryRead));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode = cbToTryRead;
+    return VINF_SUCCESS;
+}
+#endif /* !IEM_WITH_CODE_TLB */
+
+
+/**
+ * Flushes the prefetch buffer, light version.
+ */
+void iemOpcodeFlushLight(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+#ifndef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbOpcode = cbInstr;
+#else
+    RT_NOREF(pVCpu, cbInstr);
+#endif
+}
+
+
+/**
+ * Flushes the prefetch buffer, heavy version.
+ */
+void iemOpcodeFlushHeavy(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+#ifndef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbOpcode = cbInstr; /* Note! SVM and VT-x may set this to zero on exit, rather than the instruction length. */
+#elif 1
+    pVCpu->iem.s.cbInstrBufTotal = 0;
+    RT_NOREF(cbInstr);
+#else
+    RT_NOREF(pVCpu, cbInstr);
+#endif
+}
+
+
+
+#ifdef IEM_WITH_CODE_TLB
+
+/**
+ * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
+ * failure and jumps.
+ *
+ * We end up here for a number of reasons:
+ *      - pbInstrBuf isn't yet initialized.
+ *      - Advancing beyond the buffer boundrary (e.g. cross page).
+ *      - Advancing beyond the CS segment limit.
+ *      - Fetching from non-mappable page (e.g. MMIO).
+ *      - TLB loading in the recompiler (@a pvDst = NULL, @a cbDst = 0).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pvDst               Where to return the bytes.
+ * @param   cbDst               Number of bytes to read.  A value of zero is
+ *                              allowed for initializing pbInstrBuf (the
+ *                              recompiler does this).  In this case it is best
+ *                              to set pbInstrBuf to NULL prior to the call.
+ */
+void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IN_RING3
+    for (;;)
+    {
+        Assert(cbDst <= 8);
+        uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
+
+        /*
+         * We might have a partial buffer match, deal with that first to make the
+         * rest simpler.  This is the first part of the cross page/buffer case.
+         */
+        uint8_t const * const pbInstrBuf = pVCpu->iem.s.pbInstrBuf;
+        if (pbInstrBuf != NULL)
+        {
+            Assert(cbDst != 0); /* pbInstrBuf shall be NULL in case of a TLB load */
+            uint32_t const cbInstrBuf = pVCpu->iem.s.cbInstrBuf;
+            if (offBuf < cbInstrBuf)
+            {
+                Assert(offBuf + cbDst > cbInstrBuf);
+                uint32_t const cbCopy = cbInstrBuf - offBuf;
+                memcpy(pvDst, &pbInstrBuf[offBuf], cbCopy);
+
+                cbDst  -= cbCopy;
+                pvDst   = (uint8_t *)pvDst + cbCopy;
+                offBuf += cbCopy;
+            }
+        }
+
+        /*
+         * Check segment limit, figuring how much we're allowed to access at this point.
+         *
+         * We will fault immediately if RIP is past the segment limit / in non-canonical
+         * territory.  If we do continue, there are one or more bytes to read before we
+         * end up in trouble and we need to do that first before faulting.
+         */
+        RTGCPTR  GCPtrFirst;
+        uint32_t cbMaxRead;
+        if (IEM_IS_64BIT_CODE(pVCpu))
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
+            { /* likely */ }
+            else
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+            cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+        }
+        else
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            /* Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || IEM_IS_32BIT_CODE(pVCpu)); - this is allowed */
+            if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
+            { /* likely */ }
+            else /** @todo For CPUs older than the 386, we should not necessarily generate \#GP here but wrap around! */
+                iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+            cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
+            if (cbMaxRead != 0)
+            { /* likely */ }
+            else
+            {
+                /* Overflowed because address is 0 and limit is max. */
+                Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+                cbMaxRead = X86_PAGE_SIZE;
+            }
+            GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
+            uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            if (cbMaxRead2 < cbMaxRead)
+                cbMaxRead = cbMaxRead2;
+            /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
+        }
+
+        /*
+         * Get the TLB entry for this piece of code.
+         */
+        uint64_t const uTagNoRev = IEMTLB_CALC_TAG_NO_REV(GCPtrFirst);
+        PIEMTLBENTRY   pTlbe     = IEMTLB_TAG_TO_EVEN_ENTRY(&pVCpu->iem.s.CodeTlb, uTagNoRev);
+        if (   pTlbe->uTag               == (uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevision)
+            || (pTlbe = pTlbe + 1)->uTag == (uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevisionGlobal))
+        {
+            /* likely when executing lots of code, otherwise unlikely */
+#  ifdef IEM_WITH_TLB_STATISTICS
+            pVCpu->iem.s.CodeTlb.cTlbCoreHits++;
+#  endif
+            Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED));
+
+            /* Check TLB page table level access flags. */
+            if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
+            {
+                if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && IEM_GET_CPL(pVCpu) == 3)
+                {
+                    Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
+                    iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+                }
+                if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
+                {
+                    Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
+                    iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+                }
+            }
+
+            /* Look up the physical page info if necessary. */
+            if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+            { /* not necessary */ }
+            else
+            {
+                if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
+                { /* likely */ }
+                else
+                    iemTlbInvalidateAllPhysicalSlow(pVCpu);
+                pTlbe->fFlagsAndPhysRev &= ~IEMTLBE_GCPHYS2PTR_MASK;
+                int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
+                                                    &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
+                AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
+            }
+        }
+        else
+        {
+            pVCpu->iem.s.CodeTlb.cTlbCoreMisses++;
+
+            /* This page table walking will set A bits as required by the access while performing the walk.
+               ASSUMES these are set when the address is translated rather than on commit... */
+            /** @todo testcase: check when A bits are actually set by the CPU for code.  */
+            PGMPTWALKFAST WalkFast;
+            int rc = PGMGstQueryPageFast(pVCpu, GCPtrFirst,
+                                         IEM_GET_CPL(pVCpu) == 3 ? PGMQPAGE_F_EXECUTE | PGMQPAGE_F_USER_MODE : PGMQPAGE_F_EXECUTE,
+                                         &WalkFast);
+            if (RT_SUCCESS(rc))
+                Assert((WalkFast.fInfo & PGM_WALKINFO_SUCCEEDED) && WalkFast.fFailed == PGM_WALKFAIL_SUCCESS);
+            else
+            {
+#  ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                /** @todo Nested VMX: Need to handle EPT violation/misconfig here?  OF COURSE! */
+                Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
+#  endif
+                Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, rc);
+            }
+
+            AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
+            if (   !(WalkFast.fEffective & PGM_PTATTRS_G_MASK)
+                || IEM_GET_CPL(pVCpu) != 0) /* optimization: Only use the PTE.G=1 entries in ring-0. */
+            {
+                pTlbe--;
+                pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevision;
+                if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
+                    iemTlbLoadedLargePage<false>(pVCpu, &pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+#  ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.CodeTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev));
+#  endif
+            }
+            else
+            {
+                pVCpu->iem.s.CodeTlb.cTlbCoreGlobalLoads++;
+                pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevisionGlobal;
+                if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
+                    iemTlbLoadedLargePage<true>(pVCpu, &pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+#  ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.CodeTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev) + 1);
+#  endif
+            }
+            pTlbe->fFlagsAndPhysRev = (~WalkFast.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A))
+                                    | (WalkFast.fEffective >> X86_PTE_PAE_BIT_NX) /*IEMTLBE_F_PT_NO_EXEC*/
+                                    | (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE);
+            RTGCPHYS const GCPhysPg = WalkFast.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+            pTlbe->GCPhys           = GCPhysPg;
+            pTlbe->pbMappingR3      = NULL;
+            Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE));
+            Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) || IEM_GET_CPL(pVCpu) != 3);
+            Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED));
+
+            if (!((uintptr_t)pTlbe & (sizeof(*pTlbe) * 2 - 1)))
+                IEMTLBTRACE_LOAD(       pVCpu, GCPtrFirst, pTlbe->GCPhys, (uint32_t)pTlbe->fFlagsAndPhysRev, false);
+            else
+                IEMTLBTRACE_LOAD_GLOBAL(pVCpu, GCPtrFirst, pTlbe->GCPhys, (uint32_t)pTlbe->fFlagsAndPhysRev, false);
+
+            /* Resolve the physical address. */
+            if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
+            { /* likely */ }
+            else
+                iemTlbInvalidateAllPhysicalSlow(pVCpu);
+            Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_GCPHYS2PTR_MASK));
+            rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, GCPhysPg, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
+                                            &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
+            AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
+        }
+
+# if defined(IN_RING3) || defined(IN_RING0) /** @todo fixme */
+        /*
+         * Try do a direct read using the pbMappingR3 pointer.
+         * Note! Do not recheck the physical TLB revision number here as we have the
+         *       wrong response to changes in the else case.  If someone is updating
+         *       pVCpu->iem.s.CodeTlb.uTlbPhysRev in parallel to us, we should be fine
+         *       pretending we always won the race.
+         */
+        if (    (pTlbe->fFlagsAndPhysRev & (/*IEMTLBE_F_PHYS_REV |*/ IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
+             == /*pVCpu->iem.s.CodeTlb.uTlbPhysRev*/ 0U)
+        {
+            uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
+            if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
+            {
+                pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(15, cbMaxRead);
+                pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
+            }
+            else
+            {
+                uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
+                if (cbInstr + (uint32_t)cbDst <= 15)
+                {
+                    pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
+                    pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
+                }
+                else
+                {
+                    Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
+                         pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
+                    iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+                }
+            }
+            if (cbDst <= cbMaxRead)
+            {
+                pVCpu->iem.s.fTbCrossedPage     |= offPg == 0 || pVCpu->iem.s.fTbBranched != 0; /** @todo Spurious load effect on branch handling? */
+#  if 0 /* unused */
+                pVCpu->iem.s.GCPhysInstrBufPrev  = pVCpu->iem.s.GCPhysInstrBuf;
+#  endif
+                pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
+                pVCpu->iem.s.uInstrBufPc      = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
+                pVCpu->iem.s.GCPhysInstrBuf   = pTlbe->GCPhys;
+                pVCpu->iem.s.pbInstrBuf       = pTlbe->pbMappingR3;
+                if (cbDst > 0) /* To make ASAN happy in the TLB load case. */
+                    memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
+                else
+                    Assert(!pvDst);
+                return;
+            }
+            pVCpu->iem.s.pbInstrBuf = NULL;
+
+            memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
+            pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
+        }
+# else
+#  error "refactor as needed"
+        /*
+         * If there is no special read handling, so we can read a bit more and
+         * put it in the prefetch buffer.
+         */
+        if (   cbDst < cbMaxRead
+            && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
+                                                &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log((RT_SUCCESS(rcStrict)
+                     ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                     : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+            }
+        }
+# endif
+        /*
+         * Special read handling, so only read exactly what's needed.
+         * This is a highly unlikely scenario.
+         */
+        else
+        {
+            pVCpu->iem.s.CodeTlb.cTlbSlowCodeReadPath++;
+
+            /* Check instruction length. */
+            uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
+            if (RT_LIKELY(cbInstr + cbDst <= 15))
+            { /* likely */ }
+            else
+            {
+                Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0) [slow]\n",
+                     pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+            }
+
+            /* Do the reading. */
+            uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
+            if (cbToRead > 0)
+            {
+                VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
+                                                    pvDst, cbToRead, PGMACCESSORIGIN_IEM);
+                if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                { /* likely */ }
+                else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                {
+                    Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                         GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)));
+                }
+                else
+                {
+                    Log((RT_SUCCESS(rcStrict)
+                         ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                         : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                         GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+                }
+            }
+
+            /* Update the state and probably return. */
+            uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            pVCpu->iem.s.fTbCrossedPage     |= offPg == 0 || pVCpu->iem.s.fTbBranched != 0;
+#  if 0 /* unused */
+            pVCpu->iem.s.GCPhysInstrBufPrev  = pVCpu->iem.s.GCPhysInstrBuf;
+#  endif
+            pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
+            pVCpu->iem.s.offInstrNextByte = offPg + cbInstr + cbToRead;
+            pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(15, cbMaxRead + cbInstr) - cbToRead - cbInstr;
+            pVCpu->iem.s.cbInstrBufTotal  = X86_PAGE_SIZE; /** @todo ??? */
+            pVCpu->iem.s.GCPhysInstrBuf   = pTlbe->GCPhys;
+            pVCpu->iem.s.uInstrBufPc      = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
+            pVCpu->iem.s.pbInstrBuf       = NULL;
+            if (cbToRead == cbDst)
+                return;
+            Assert(cbToRead == cbMaxRead);
+        }
+
+        /*
+         * More to read, loop.
+         */
+        cbDst -= cbMaxRead;
+        pvDst  = (uint8_t *)pvDst + cbMaxRead;
+    }
+# else  /* !IN_RING3 */
+    RT_NOREF(pvDst, cbDst);
+    if (pvDst || cbDst)
+        IEM_DO_LONGJMP(pVCpu, VERR_INTERNAL_ERROR);
+# endif /* !IN_RING3 */
+}
+
+#else /* !IEM_WITH_CODE_TLB */
+
+/**
+ * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
+ * exception if it fails.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   cbMin               The minimum number of bytes relative offOpcode
+ *                              that must be read.
+ */
+VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
+{
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     */
+    uint8_t const   cbOpcode  = pVCpu->iem.s.cbOpcode;
+    uint8_t const   offOpcode = pVCpu->iem.s.offOpcode;
+    uint8_t const   cbLeft    = cbOpcode - offOpcode;
+    Assert(cbLeft < cbMin);
+    Assert(cbOpcode <= sizeof(pVCpu->iem.s.abOpcode));
+
+    uint32_t        cbToTryRead;
+    RTGCPTR         GCPtrNext;
+    if (IEM_IS_64BIT_CODE(pVCpu))
+    {
+        GCPtrNext   = pVCpu->cpum.GstCtx.rip + cbOpcode;
+        if (!IEM_IS_CANONICAL(GCPtrNext))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        cbToTryRead = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
+    }
+    else
+    {
+        uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
+        /* Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || IEM_IS_32BIT_CODE(pVCpu)); - this is allowed */
+        GCPtrNext32 += cbOpcode;
+        if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
+            /** @todo For CPUs older than the 386, we should not generate \#GP here but wrap around! */
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
+        if (!cbToTryRead) /* overflowed */
+        {
+            Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+            /** @todo check out wrapping around the code segment.  */
+        }
+        if (cbToTryRead < cbMin - cbLeft)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
+
+        uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
+        if (cbToTryRead > cbLeftOnPage)
+            cbToTryRead = cbLeftOnPage;
+    }
+
+    /* Restrict to opcode buffer space.
+
+       We're making ASSUMPTIONS here based on work done previously in
+       iemInitDecoderAndPrefetchOpcodes, where bytes from the first page will
+       be fetched in case of an instruction crossing two pages. */
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - cbOpcode)
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - cbOpcode;
+    if (RT_LIKELY(cbToTryRead + cbLeft >= cbMin))
+    { /* likely */ }
+    else
+    {
+        Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
+             pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, offOpcode, cbMin));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    PGMPTWALKFAST WalkFast;
+    int rc = PGMGstQueryPageFast(pVCpu, GCPtrNext,
+                                 IEM_GET_CPL(pVCpu) == 3 ? PGMQPAGE_F_EXECUTE | PGMQPAGE_F_USER_MODE : PGMQPAGE_F_EXECUTE,
+                                 &WalkFast);
+    if (RT_SUCCESS(rc))
+        Assert((WalkFast.fInfo & PGM_WALKINFO_SUCCEEDED) && WalkFast.fFailed == PGM_WALKFAIL_SUCCESS);
+    else
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (WalkFast.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &WalkFast, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+#endif
+        return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, rc);
+    }
+    Assert((WalkFast.fEffective & X86_PTE_US) || IEM_GET_CPL(pVCpu) != 3);
+    Assert(!(WalkFast.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE));
+
+    RTGCPHYS const GCPhys = WalkFast.GCPhys;
+    Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n",  GCPtrNext,  GCPhys, cbOpcode));
+
+    /*
+     * Read the bytes at this address.
+     *
+     * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
+     * and since PATM should only patch the start of an instruction there
+     * should be no need to check again here.
+     */
+    if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[cbOpcode],
+                                            cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[cbOpcode], GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode = cbOpcode + cbToTryRead;
+    Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_CODE_TLB */
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pb                  Where to return the opcode byte.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pb = pVCpu->iem.s.abOpcode[offOpcode];
+        pVCpu->iem.s.offOpcode = offOpcode + 1;
+    }
+    else
+        *pb = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint8_t u8;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
+    return u8;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+        return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu16 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu32 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu64 = (int8_t)u8;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu16 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
+ *
+ * @returns The opcode word.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint16_t u16;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
+    return u16;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode += 2;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode double word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode dword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint32_t u32;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
+    return u32;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 7]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint64_t u64;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
+    return u64;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 7]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
Index: trunk/src/VBox/VMM/VMMAll/target-x86/IEMInternal-x86.h
===================================================================
--- trunk/src/VBox/VMM/VMMAll/target-x86/IEMInternal-x86.h	(revision 108243)
+++ trunk/src/VBox/VMM/VMMAll/target-x86/IEMInternal-x86.h	(revision 108244)
@@ -41,4 +41,6 @@
  * @{
  */
+
+VBOXSTRICTRC iemOpcodeFetchPrefetch(PVMCPUCC pVCpu) RT_NOEXCEPT;
 
 
