Changeset 108245 in vbox for trunk/src/VBox/VMM/VMMAll/target-x86/IEMAll-x86.cpp
- Timestamp:
- Feb 17, 2025 12:13:38 AM (3 months ago)
- svn:sync-xref-src-repo-rev:
- 167566
- File:
-
- 1 copied
-
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAll-x86.cpp (copied) (copied from trunk/src/VBox/VMM/VMMAll/IEMAll.cpp ) (7 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAll-x86.cpp
r108244 r108245 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - All Contexts.3 * IEM - Interpreted Execution Manager - x86 target, miscellaneous. 4 4 */ 5 5 … … 25 25 * SPDX-License-Identifier: GPL-3.0-only 26 26 */ 27 28 29 /** @page pg_iem IEM - Interpreted Execution Manager30 *31 * The interpreted exeuction manager (IEM) is for executing short guest code32 * sequences that are causing too many exits / virtualization traps. It will33 * also be used to interpret single instructions, thus replacing the selective34 * interpreters in EM and IOM.35 *36 * Design goals:37 * - Relatively small footprint, although we favour speed and correctness38 * over size.39 * - Reasonably fast.40 * - Correctly handle lock prefixed instructions.41 * - Complete instruction set - eventually.42 * - Refactorable into a recompiler, maybe.43 * - Replace EMInterpret*.44 *45 * Using the existing disassembler has been considered, however this is thought46 * to conflict with speed as the disassembler chews things a bit too much while47 * leaving us with a somewhat complicated state to interpret afterwards.48 *49 *50 * The current code is very much work in progress. You've been warned!51 *52 *53 * @section sec_iem_fpu_instr FPU Instructions54 *55 * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the56 * same or equivalent instructions on the host FPU. To make life easy, we also57 * let the FPU prioritize the unmasked exceptions for us. This however, only58 * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 1359 * for FPU exception delivery, because with CR0.NE=0 there is a window where we60 * can trigger spurious FPU exceptions.61 *62 * The guest FPU state is not loaded into the host CPU and kept there till we63 * leave IEM because the calling conventions have declared an all year open64 * season on much of the FPU state. For instance an innocent looking call to65 * memcpy might end up using a whole bunch of XMM or MM registers if the66 * particular implementation finds it worthwhile.67 *68 *69 * @section sec_iem_logging Logging70 *71 * The IEM code uses the \"IEM\" log group for the main logging. The different72 * logging levels/flags are generally used for the following purposes:73 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events.74 * - Flow (LogFlow) : Basic enter/exit IEM state info.75 * - Level 2 (Log2) : ?76 * - Level 3 (Log3) : More detailed enter/exit IEM state info.77 * - Level 4 (Log4) : Decoding mnemonics w/ EIP.78 * - Level 5 (Log5) : Decoding details.79 * - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.80 * - Level 7 (Log7) : iret++ execution logging.81 * - Level 8 (Log8) :82 * - Level 9 (Log9) :83 * - Level 10 (Log10): TLBs.84 * - Level 11 (Log11): Unmasked FPU exceptions.85 *86 * The \"IEM_MEM\" log group covers most of memory related details logging,87 * except for errors and exceptions:88 * - Level 1 (Log) : Reads.89 * - Level 2 (Log2) : Read fallbacks.90 * - Level 3 (Log3) : MemMap read.91 * - Level 4 (Log4) : MemMap read fallbacks.92 * - Level 5 (Log5) : Writes93 * - Level 6 (Log6) : Write fallbacks.94 * - Level 7 (Log7) : MemMap writes and read-writes.95 * - Level 8 (Log8) : MemMap write and read-write fallbacks.96 * - Level 9 (Log9) : Stack reads.97 * - Level 10 (Log10): Stack read fallbacks.98 * - Level 11 (Log11): Stack writes.99 * - Level 12 (Log12): Stack write fallbacks.100 * - Flow (LogFlow) :101 *102 * The SVM (AMD-V) and VMX (VT-x) code has the following assignments:103 * - Level 1 (Log) : Errors and other major events.104 * - Flow (LogFlow) : Misc flow stuff (cleanup?)105 * - Level 2 (Log2) : VM exits.106 *107 * The syscall logging level assignments:108 * - Level 1: DOS and BIOS.109 * - Level 2: Windows 3.x110 * - Level 3: Linux.111 */112 113 /* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */114 #ifdef _MSC_VER115 # pragma warning(disable:4505)116 #endif117 27 118 28 … … 127 37 #include <VBox/vmm/iem.h> 128 38 #include <VBox/vmm/cpum.h> 129 #include <VBox/vmm/pdmapic.h>130 #include <VBox/vmm/pdm.h>131 #include <VBox/vmm/pgm.h>132 #include <VBox/vmm/iom.h>133 #include <VBox/vmm/em.h>134 #include <VBox/vmm/hm.h>135 #include <VBox/vmm/nem.h>136 #include <VBox/vmm/gcm.h>137 #include <VBox/vmm/gim.h>138 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM139 # include <VBox/vmm/em.h>140 # include <VBox/vmm/hm_svm.h>141 #endif142 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX143 # include <VBox/vmm/hmvmxinline.h>144 #endif145 #include <VBox/vmm/tm.h>146 39 #include <VBox/vmm/dbgf.h> 147 #include <VBox/vmm/dbgftrace.h>148 40 #include "IEMInternal.h" 149 41 #include <VBox/vmm/vmcc.h> … … 151 43 #include <VBox/err.h> 152 44 #include <VBox/param.h> 153 #include <VBox/dis.h>154 #include <iprt/asm-math.h>155 #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)156 # include <iprt/asm-amd64-x86.h>157 #elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32)158 # include <iprt/asm-arm.h>159 #endif160 45 #include <iprt/assert.h> 161 46 #include <iprt/string.h> … … 163 48 164 49 #include "IEMInline.h" 165 #ifdef VBOX_VMM_TARGET_X86166 # include "target-x86/IEMAllTlbInline-x86.h"167 #endif168 169 170 /*********************************************************************************************************************************171 * Global Variables *172 *********************************************************************************************************************************/173 #if defined(IEM_LOG_MEMORY_WRITES)174 /** What IEM just wrote. */175 uint8_t g_abIemWrote[256];176 /** How much IEM just wrote. */177 size_t g_cbIemWrote;178 #endif179 50 180 51 … … 271 142 272 143 273 /**274 * Initializes the decoder state.275 *276 * iemReInitDecoder is mostly a copy of this function.277 *278 * @param pVCpu The cross context virtual CPU structure of the279 * calling thread.280 * @param fExecOpts Optional execution flags:281 * - IEM_F_BYPASS_HANDLERS282 * - IEM_F_X86_DISREGARD_LOCK283 */284 DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, uint32_t fExecOpts)285 {286 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);287 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));288 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));289 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));290 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));291 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));292 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));293 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));294 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));295 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));296 297 /* Execution state: */298 uint32_t fExec;299 pVCpu->iem.s.fExec = fExec = iemCalcExecFlags(pVCpu) | fExecOpts;300 301 /* Decoder state: */302 pVCpu->iem.s.enmDefAddrMode = fExec & IEM_F_MODE_CPUMODE_MASK; /** @todo check if this is correct... */303 pVCpu->iem.s.enmEffAddrMode = fExec & IEM_F_MODE_CPUMODE_MASK;304 if ((fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)305 {306 pVCpu->iem.s.enmDefOpSize = fExec & IEM_F_MODE_CPUMODE_MASK; /** @todo check if this is correct... */307 pVCpu->iem.s.enmEffOpSize = fExec & IEM_F_MODE_CPUMODE_MASK;308 }309 else310 {311 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;312 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;313 }314 pVCpu->iem.s.fPrefixes = 0;315 pVCpu->iem.s.uRexReg = 0;316 pVCpu->iem.s.uRexB = 0;317 pVCpu->iem.s.uRexIndex = 0;318 pVCpu->iem.s.idxPrefix = 0;319 pVCpu->iem.s.uVex3rdReg = 0;320 pVCpu->iem.s.uVexLength = 0;321 pVCpu->iem.s.fEvexStuff = 0;322 pVCpu->iem.s.iEffSeg = X86_SREG_DS;323 #ifdef IEM_WITH_CODE_TLB324 pVCpu->iem.s.pbInstrBuf = NULL;325 pVCpu->iem.s.offInstrNextByte = 0;326 pVCpu->iem.s.offCurInstrStart = 0;327 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF328 pVCpu->iem.s.offOpcode = 0;329 # endif330 # ifdef VBOX_STRICT331 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;332 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;333 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;334 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);335 # endif336 #else337 pVCpu->iem.s.offOpcode = 0;338 pVCpu->iem.s.cbOpcode = 0;339 #endif340 pVCpu->iem.s.offModRm = 0;341 pVCpu->iem.s.cActiveMappings = 0;342 pVCpu->iem.s.iNextMapping = 0;343 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;344 345 #ifdef DBGFTRACE_ENABLED346 switch (IEM_GET_CPU_MODE(pVCpu))347 {348 case IEMMODE_64BIT:349 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);350 break;351 case IEMMODE_32BIT:352 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);353 break;354 case IEMMODE_16BIT:355 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);356 break;357 }358 #endif359 }360 361 362 /**363 * Reinitializes the decoder state 2nd+ loop of IEMExecLots.364 *365 * This is mostly a copy of iemInitDecoder.366 *367 * @param pVCpu The cross context virtual CPU structure of the calling EMT.368 */369 DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)370 {371 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));372 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));373 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));374 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));375 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));376 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));377 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));378 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));379 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));380 381 /* ASSUMES: Anyone changing CPU state affecting the fExec bits will update them! */382 AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),383 ("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));384 385 IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);386 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */387 pVCpu->iem.s.enmEffAddrMode = enmMode;388 if (enmMode != IEMMODE_64BIT)389 {390 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */391 pVCpu->iem.s.enmEffOpSize = enmMode;392 }393 else394 {395 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;396 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;397 }398 pVCpu->iem.s.fPrefixes = 0;399 pVCpu->iem.s.uRexReg = 0;400 pVCpu->iem.s.uRexB = 0;401 pVCpu->iem.s.uRexIndex = 0;402 pVCpu->iem.s.idxPrefix = 0;403 pVCpu->iem.s.uVex3rdReg = 0;404 pVCpu->iem.s.uVexLength = 0;405 pVCpu->iem.s.fEvexStuff = 0;406 pVCpu->iem.s.iEffSeg = X86_SREG_DS;407 #ifdef IEM_WITH_CODE_TLB408 if (pVCpu->iem.s.pbInstrBuf)409 {410 uint64_t off = (enmMode == IEMMODE_64BIT411 ? pVCpu->cpum.GstCtx.rip412 : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)413 - pVCpu->iem.s.uInstrBufPc;414 if (off < pVCpu->iem.s.cbInstrBufTotal)415 {416 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;417 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;418 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)419 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;420 else421 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;422 }423 else424 {425 pVCpu->iem.s.pbInstrBuf = NULL;426 pVCpu->iem.s.offInstrNextByte = 0;427 pVCpu->iem.s.offCurInstrStart = 0;428 pVCpu->iem.s.cbInstrBuf = 0;429 pVCpu->iem.s.cbInstrBufTotal = 0;430 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;431 }432 }433 else434 {435 pVCpu->iem.s.offInstrNextByte = 0;436 pVCpu->iem.s.offCurInstrStart = 0;437 pVCpu->iem.s.cbInstrBuf = 0;438 pVCpu->iem.s.cbInstrBufTotal = 0;439 # ifdef VBOX_STRICT440 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;441 # endif442 }443 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF444 pVCpu->iem.s.offOpcode = 0;445 # endif446 #else /* !IEM_WITH_CODE_TLB */447 pVCpu->iem.s.cbOpcode = 0;448 pVCpu->iem.s.offOpcode = 0;449 #endif /* !IEM_WITH_CODE_TLB */450 pVCpu->iem.s.offModRm = 0;451 Assert(pVCpu->iem.s.cActiveMappings == 0);452 pVCpu->iem.s.iNextMapping = 0;453 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);454 Assert(!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS));455 456 #ifdef DBGFTRACE_ENABLED457 switch (enmMode)458 {459 case IEMMODE_64BIT:460 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);461 break;462 case IEMMODE_32BIT:463 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);464 break;465 case IEMMODE_16BIT:466 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);467 break;468 }469 #endif470 }471 472 473 474 /**475 * Prefetch opcodes the first time when starting executing.476 *477 * @returns Strict VBox status code.478 * @param pVCpu The cross context virtual CPU structure of the479 * calling thread.480 * @param fExecOpts Optional execution flags:481 * - IEM_F_BYPASS_HANDLERS482 * - IEM_F_X86_DISREGARD_LOCK483 */484 DECLINLINE(VBOXSTRICTRC) iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT485 {486 iemInitDecoder(pVCpu, fExecOpts);487 488 #ifndef IEM_WITH_CODE_TLB489 return iemOpcodeFetchPrefetch(pVCpu);490 #else491 return VINF_SUCCESS;492 #endif493 }494 495 496 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)497 /**498 * Worker for iemTlbInvalidateAll.499 */500 template<bool a_fGlobal>501 DECL_FORCE_INLINE(void) iemTlbInvalidateOne(IEMTLB *pTlb)502 {503 if (!a_fGlobal)504 pTlb->cTlsFlushes++;505 else506 pTlb->cTlsGlobalFlushes++;507 508 pTlb->uTlbRevision += IEMTLB_REVISION_INCR;509 if (RT_LIKELY(pTlb->uTlbRevision != 0))510 { /* very likely */ }511 else512 {513 pTlb->uTlbRevision = IEMTLB_REVISION_INCR;514 pTlb->cTlbRevisionRollovers++;515 unsigned i = RT_ELEMENTS(pTlb->aEntries) / 2;516 while (i-- > 0)517 pTlb->aEntries[i * 2].uTag = 0;518 }519 520 pTlb->cTlbNonGlobalLargePageCurLoads = 0;521 pTlb->NonGlobalLargePageRange.uLastTag = 0;522 pTlb->NonGlobalLargePageRange.uFirstTag = UINT64_MAX;523 524 if (a_fGlobal)525 {526 pTlb->uTlbRevisionGlobal += IEMTLB_REVISION_INCR;527 if (RT_LIKELY(pTlb->uTlbRevisionGlobal != 0))528 { /* very likely */ }529 else530 {531 pTlb->uTlbRevisionGlobal = IEMTLB_REVISION_INCR;532 pTlb->cTlbRevisionRollovers++;533 unsigned i = RT_ELEMENTS(pTlb->aEntries) / 2;534 while (i-- > 0)535 pTlb->aEntries[i * 2 + 1].uTag = 0;536 }537 538 pTlb->cTlbGlobalLargePageCurLoads = 0;539 pTlb->GlobalLargePageRange.uLastTag = 0;540 pTlb->GlobalLargePageRange.uFirstTag = UINT64_MAX;541 }542 }543 #endif544 545 546 /**547 * Worker for IEMTlbInvalidateAll and IEMTlbInvalidateAllGlobal.548 */549 template<bool a_fGlobal>550 DECL_FORCE_INLINE(void) iemTlbInvalidateAll(PVMCPUCC pVCpu)551 {552 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)553 Log10(("IEMTlbInvalidateAll\n"));554 555 # ifdef IEM_WITH_CODE_TLB556 pVCpu->iem.s.cbInstrBufTotal = 0;557 iemTlbInvalidateOne<a_fGlobal>(&pVCpu->iem.s.CodeTlb);558 if (a_fGlobal)559 IEMTLBTRACE_FLUSH_GLOBAL(pVCpu, pVCpu->iem.s.CodeTlb.uTlbRevision, pVCpu->iem.s.CodeTlb.uTlbRevisionGlobal, false);560 else561 IEMTLBTRACE_FLUSH(pVCpu, pVCpu->iem.s.CodeTlb.uTlbRevision, false);562 # endif563 564 # ifdef IEM_WITH_DATA_TLB565 iemTlbInvalidateOne<a_fGlobal>(&pVCpu->iem.s.DataTlb);566 if (a_fGlobal)567 IEMTLBTRACE_FLUSH_GLOBAL(pVCpu, pVCpu->iem.s.DataTlb.uTlbRevision, pVCpu->iem.s.DataTlb.uTlbRevisionGlobal, true);568 else569 IEMTLBTRACE_FLUSH(pVCpu, pVCpu->iem.s.DataTlb.uTlbRevision, true);570 # endif571 #else572 RT_NOREF(pVCpu);573 #endif574 }575 576 577 /**578 * Invalidates non-global the IEM TLB entries.579 *580 * This is called internally as well as by PGM when moving GC mappings.581 *582 * @param pVCpu The cross context virtual CPU structure of the calling583 * thread.584 */585 VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)586 {587 iemTlbInvalidateAll<false>(pVCpu);588 }589 590 591 /**592 * Invalidates all the IEM TLB entries.593 *594 * This is called internally as well as by PGM when moving GC mappings.595 *596 * @param pVCpu The cross context virtual CPU structure of the calling597 * thread.598 */599 VMM_INT_DECL(void) IEMTlbInvalidateAllGlobal(PVMCPUCC pVCpu)600 {601 iemTlbInvalidateAll<true>(pVCpu);602 }603 604 605 /**606 * Invalidates a page in the TLBs.607 *608 * @param pVCpu The cross context virtual CPU structure of the calling609 * thread.610 * @param GCPtr The address of the page to invalidate611 * @thread EMT(pVCpu)612 */613 VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)614 {615 IEMTLBTRACE_INVLPG(pVCpu, GCPtr);616 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)617 Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));618 GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);619 Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));620 uintptr_t const idxEven = IEMTLB_TAG_TO_EVEN_INDEX(GCPtr);621 622 # ifdef IEM_WITH_CODE_TLB623 iemTlbInvalidatePageWorker<false>(pVCpu, &pVCpu->iem.s.CodeTlb, GCPtr, idxEven);624 # endif625 # ifdef IEM_WITH_DATA_TLB626 iemTlbInvalidatePageWorker<true>(pVCpu, &pVCpu->iem.s.DataTlb, GCPtr, idxEven);627 # endif628 #else629 NOREF(pVCpu); NOREF(GCPtr);630 #endif631 }632 633 634 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)635 /**636 * Invalid both TLBs slow fashion following a rollover.637 *638 * Worker for IEMTlbInvalidateAllPhysical,639 * IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,640 * iemMemMapJmp and others.641 *642 * @thread EMT(pVCpu)643 */644 void iemTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) RT_NOEXCEPT645 {646 Log10(("iemTlbInvalidateAllPhysicalSlow\n"));647 ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);648 ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);649 650 unsigned i;651 # ifdef IEM_WITH_CODE_TLB652 i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);653 while (i-- > 0)654 {655 pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;656 pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ657 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);658 }659 pVCpu->iem.s.CodeTlb.cTlbPhysRevRollovers++;660 pVCpu->iem.s.CodeTlb.cTlbPhysRevFlushes++;661 # endif662 # ifdef IEM_WITH_DATA_TLB663 i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);664 while (i-- > 0)665 {666 pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;667 pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ668 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);669 }670 pVCpu->iem.s.DataTlb.cTlbPhysRevRollovers++;671 pVCpu->iem.s.DataTlb.cTlbPhysRevFlushes++;672 # endif673 674 }675 #endif676 677 678 /**679 * Invalidates the host physical aspects of the IEM TLBs.680 *681 * This is called internally as well as by PGM when moving GC mappings.682 *683 * @param pVCpu The cross context virtual CPU structure of the calling684 * thread.685 * @note Currently not used.686 */687 VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)688 {689 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)690 /* Note! This probably won't end up looking exactly like this, but it give an idea... */691 Log10(("IEMTlbInvalidateAllPhysical\n"));692 693 # ifdef IEM_WITH_CODE_TLB694 pVCpu->iem.s.cbInstrBufTotal = 0;695 # endif696 uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;697 if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))698 {699 pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;700 pVCpu->iem.s.CodeTlb.cTlbPhysRevFlushes++;701 pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;702 pVCpu->iem.s.DataTlb.cTlbPhysRevFlushes++;703 }704 else705 iemTlbInvalidateAllPhysicalSlow(pVCpu);706 #else707 NOREF(pVCpu);708 #endif709 }710 711 712 /**713 * Invalidates the host physical aspects of the IEM TLBs.714 *715 * This is called internally as well as by PGM when moving GC mappings.716 *717 * @param pVM The cross context VM structure.718 * @param idCpuCaller The ID of the calling EMT if available to the caller,719 * otherwise NIL_VMCPUID.720 * @param enmReason The reason we're called.721 *722 * @remarks Caller holds the PGM lock.723 */724 VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller, IEMTLBPHYSFLUSHREASON enmReason)725 {726 #if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)727 PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);728 if (pVCpuCaller)729 VMCPU_ASSERT_EMT(pVCpuCaller);730 Log10(("IEMTlbInvalidateAllPhysicalAllCpus: %d\n", enmReason)); RT_NOREF(enmReason);731 732 VMCC_FOR_EACH_VMCPU(pVM)733 {734 # ifdef IEM_WITH_CODE_TLB735 if (pVCpuCaller == pVCpu)736 pVCpu->iem.s.cbInstrBufTotal = 0;737 # endif738 739 uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);740 uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;741 if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))742 { /* likely */}743 else if (pVCpuCaller != pVCpu)744 uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;745 else746 {747 iemTlbInvalidateAllPhysicalSlow(pVCpu);748 continue;749 }750 if (ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev))751 pVCpu->iem.s.CodeTlb.cTlbPhysRevFlushes++;752 753 if (ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev))754 pVCpu->iem.s.DataTlb.cTlbPhysRevFlushes++;755 }756 VMCC_FOR_EACH_VMCPU_END(pVM);757 758 #else759 RT_NOREF(pVM, idCpuCaller, enmReason);760 #endif761 }762 763 764 144 /** @name Register Access. 765 145 * @{ … … 916 296 /** @} */ 917 297 918 919 /** @name Memory access.920 *921 * @{922 */923 924 #undef LOG_GROUP925 #define LOG_GROUP LOG_GROUP_IEM_MEM926 927 #if 0 /*unused*/928 /**929 * Looks up a memory mapping entry.930 *931 * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).932 * @param pVCpu The cross context virtual CPU structure of the calling thread.933 * @param pvMem The memory address.934 * @param fAccess The access to.935 */936 DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)937 {938 Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));939 fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;940 if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem941 && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)942 return 0;943 if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem944 && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)945 return 1;946 if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem947 && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)948 return 2;949 return VERR_NOT_FOUND;950 }951 #endif952 953 /**954 * Finds a free memmap entry when using iNextMapping doesn't work.955 *956 * @returns Memory mapping index, 1024 on failure.957 * @param pVCpu The cross context virtual CPU structure of the calling thread.958 */959 static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)960 {961 /*962 * The easy case.963 */964 if (pVCpu->iem.s.cActiveMappings == 0)965 {966 pVCpu->iem.s.iNextMapping = 1;967 return 0;968 }969 970 /* There should be enough mappings for all instructions. */971 AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);972 973 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)974 if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)975 return i;976 977 AssertFailedReturn(1024);978 }979 980 981 /**982 * Commits a bounce buffer that needs writing back and unmaps it.983 *984 * @returns Strict VBox status code.985 * @param pVCpu The cross context virtual CPU structure of the calling thread.986 * @param iMemMap The index of the buffer to commit.987 * @param fPostponeFail Whether we can postpone writer failures to ring-3.988 * Always false in ring-3, obviously.989 */990 static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)991 {992 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);993 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);994 #ifdef IN_RING3995 Assert(!fPostponeFail);996 RT_NOREF_PV(fPostponeFail);997 #endif998 999 /*1000 * Do the writing.1001 */1002 PVMCC pVM = pVCpu->CTX_SUFF(pVM);1003 if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)1004 {1005 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;1006 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;1007 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];1008 if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))1009 {1010 /*1011 * Carefully and efficiently dealing with access handler return1012 * codes make this a little bloated.1013 */1014 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,1015 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,1016 pbBuf,1017 cbFirst,1018 PGMACCESSORIGIN_IEM);1019 if (rcStrict == VINF_SUCCESS)1020 {1021 if (cbSecond)1022 {1023 rcStrict = PGMPhysWrite(pVM,1024 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,1025 pbBuf + cbFirst,1026 cbSecond,1027 PGMACCESSORIGIN_IEM);1028 if (rcStrict == VINF_SUCCESS)1029 { /* nothing */ }1030 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))1031 {1032 LogEx(LOG_GROUP_IEM,1033 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",1034 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1035 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));1036 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1037 }1038 #ifndef IN_RING31039 else if (fPostponeFail)1040 {1041 LogEx(LOG_GROUP_IEM,1042 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",1043 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1044 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));1045 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;1046 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);1047 return iemSetPassUpStatus(pVCpu, rcStrict);1048 }1049 #endif1050 else1051 {1052 LogEx(LOG_GROUP_IEM,1053 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",1054 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1055 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));1056 return rcStrict;1057 }1058 }1059 }1060 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))1061 {1062 if (!cbSecond)1063 {1064 LogEx(LOG_GROUP_IEM,1065 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",1066 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));1067 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1068 }1069 else1070 {1071 VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,1072 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,1073 pbBuf + cbFirst,1074 cbSecond,1075 PGMACCESSORIGIN_IEM);1076 if (rcStrict2 == VINF_SUCCESS)1077 {1078 LogEx(LOG_GROUP_IEM,1079 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",1080 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),1081 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));1082 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1083 }1084 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))1085 {1086 LogEx(LOG_GROUP_IEM,1087 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",1088 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),1089 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));1090 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);1091 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1092 }1093 #ifndef IN_RING31094 else if (fPostponeFail)1095 {1096 LogEx(LOG_GROUP_IEM,1097 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",1098 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1099 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));1100 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;1101 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);1102 return iemSetPassUpStatus(pVCpu, rcStrict);1103 }1104 #endif1105 else1106 {1107 LogEx(LOG_GROUP_IEM,1108 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",1109 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),1110 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));1111 return rcStrict2;1112 }1113 }1114 }1115 #ifndef IN_RING31116 else if (fPostponeFail)1117 {1118 LogEx(LOG_GROUP_IEM,1119 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",1120 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1121 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));1122 if (!cbSecond)1123 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;1124 else1125 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;1126 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);1127 return iemSetPassUpStatus(pVCpu, rcStrict);1128 }1129 #endif1130 else1131 {1132 LogEx(LOG_GROUP_IEM,1133 ("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",1134 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),1135 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));1136 return rcStrict;1137 }1138 }1139 else1140 {1141 /*1142 * No access handlers, much simpler.1143 */1144 int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);1145 if (RT_SUCCESS(rc))1146 {1147 if (cbSecond)1148 {1149 rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);1150 if (RT_SUCCESS(rc))1151 { /* likely */ }1152 else1153 {1154 LogEx(LOG_GROUP_IEM,1155 ("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",1156 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,1157 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));1158 return rc;1159 }1160 }1161 }1162 else1163 {1164 LogEx(LOG_GROUP_IEM,1165 ("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",1166 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,1167 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));1168 return rc;1169 }1170 }1171 }1172 1173 #if defined(IEM_LOG_MEMORY_WRITES)1174 Log5(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,1175 RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));1176 if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)1177 Log5(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,1178 RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),1179 &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));1180 1181 size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;1182 g_cbIemWrote = cbWrote;1183 memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));1184 #endif1185 1186 /*1187 * Free the mapping entry.1188 */1189 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1190 Assert(pVCpu->iem.s.cActiveMappings != 0);1191 pVCpu->iem.s.cActiveMappings--;1192 return VINF_SUCCESS;1193 }1194 1195 1196 /**1197 * Helper for iemMemMap, iemMemMapJmp and iemMemBounceBufferMapCrossPage.1198 * @todo duplicated1199 */1200 DECL_FORCE_INLINE(uint32_t)1201 iemMemCheckDataBreakpoint(PVMCC pVM, PVMCPUCC pVCpu, RTGCPTR GCPtrMem, size_t cbMem, uint32_t fAccess)1202 {1203 bool const fSysAccess = (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_SYS;1204 if (fAccess & IEM_ACCESS_TYPE_WRITE)1205 return DBGFBpCheckDataWrite(pVM, pVCpu, GCPtrMem, (uint32_t)cbMem, fSysAccess);1206 return DBGFBpCheckDataRead(pVM, pVCpu, GCPtrMem, (uint32_t)cbMem, fSysAccess);1207 }1208 1209 1210 /**1211 * iemMemMap worker that deals with a request crossing pages.1212 */1213 VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, uint8_t *pbUnmapInfo,1214 size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) RT_NOEXCEPT1215 {1216 STAM_COUNTER_INC(&pVCpu->iem.s.StatMemBounceBufferCrossPage);1217 Assert(cbMem <= GUEST_PAGE_SIZE);1218 1219 /*1220 * Do the address translations.1221 */1222 uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (uint32_t)(GCPtrFirst & GUEST_PAGE_OFFSET_MASK);1223 RTGCPHYS GCPhysFirst;1224 VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, cbFirstPage, fAccess, &GCPhysFirst);1225 if (rcStrict != VINF_SUCCESS)1226 return rcStrict;1227 Assert((GCPhysFirst & GUEST_PAGE_OFFSET_MASK) == (GCPtrFirst & GUEST_PAGE_OFFSET_MASK));1228 1229 uint32_t const cbSecondPage = (uint32_t)cbMem - cbFirstPage;1230 RTGCPHYS GCPhysSecond;1231 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,1232 cbSecondPage, fAccess, &GCPhysSecond);1233 if (rcStrict != VINF_SUCCESS)1234 return rcStrict;1235 Assert((GCPhysSecond & GUEST_PAGE_OFFSET_MASK) == 0);1236 GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; /** @todo why? */1237 1238 PVMCC pVM = pVCpu->CTX_SUFF(pVM);1239 1240 /*1241 * Check for data breakpoints.1242 */1243 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_DATA)))1244 { /* likely */ }1245 else1246 {1247 uint32_t fDataBps = iemMemCheckDataBreakpoint(pVM, pVCpu, GCPtrFirst, cbFirstPage, fAccess);1248 fDataBps |= iemMemCheckDataBreakpoint(pVM, pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,1249 cbSecondPage, fAccess);1250 pVCpu->cpum.GstCtx.eflags.uBoth |= fDataBps & (CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);1251 if (fDataBps > 1)1252 LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapCrossPage: Data breakpoint: fDataBps=%#x for %RGv LB %zx; fAccess=%#x cs:rip=%04x:%08RX64\n",1253 fDataBps, GCPtrFirst, cbMem, fAccess, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));1254 }1255 1256 /*1257 * Read in the current memory content if it's a read, execute or partial1258 * write access.1259 */1260 uint8_t * const pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];1261 1262 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))1263 {1264 if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))1265 {1266 /*1267 * Must carefully deal with access handler status codes here,1268 * makes the code a bit bloated.1269 */1270 rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);1271 if (rcStrict == VINF_SUCCESS)1272 {1273 rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);1274 if (rcStrict == VINF_SUCCESS)1275 { /*likely */ }1276 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))1277 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1278 else1279 {1280 LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",1281 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));1282 return rcStrict;1283 }1284 }1285 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))1286 {1287 VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);1288 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))1289 {1290 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);1291 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1292 }1293 else1294 {1295 LogEx(LOG_GROUP_IEM,1296 ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",1297 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));1298 return rcStrict2;1299 }1300 }1301 else1302 {1303 LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",1304 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));1305 return rcStrict;1306 }1307 }1308 else1309 {1310 /*1311 * No informational status codes here, much more straight forward.1312 */1313 int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);1314 if (RT_SUCCESS(rc))1315 {1316 Assert(rc == VINF_SUCCESS);1317 rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);1318 if (RT_SUCCESS(rc))1319 Assert(rc == VINF_SUCCESS);1320 else1321 {1322 LogEx(LOG_GROUP_IEM,1323 ("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));1324 return rc;1325 }1326 }1327 else1328 {1329 LogEx(LOG_GROUP_IEM,1330 ("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));1331 return rc;1332 }1333 }1334 }1335 #ifdef VBOX_STRICT1336 else1337 memset(pbBuf, 0xcc, cbMem);1338 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))1339 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);1340 #endif1341 AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64);1342 1343 /*1344 * Commit the bounce buffer entry.1345 */1346 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;1347 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;1348 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;1349 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;1350 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;1351 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;1352 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;1353 pVCpu->iem.s.iNextMapping = iMemMap + 1;1354 pVCpu->iem.s.cActiveMappings++;1355 1356 *ppvMem = pbBuf;1357 *pbUnmapInfo = iMemMap | 0x08 | ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);1358 return VINF_SUCCESS;1359 }1360 1361 1362 /**1363 * iemMemMap woker that deals with iemMemPageMap failures.1364 */1365 VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, uint8_t *pbUnmapInfo, size_t cbMem,1366 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) RT_NOEXCEPT1367 {1368 STAM_COUNTER_INC(&pVCpu->iem.s.StatMemBounceBufferMapPhys);1369 1370 /*1371 * Filter out conditions we can handle and the ones which shouldn't happen.1372 */1373 if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE1374 && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL1375 && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)1376 {1377 AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);1378 return rcMap;1379 }1380 pVCpu->iem.s.cPotentialExits++;1381 1382 /*1383 * Read in the current memory content if it's a read, execute or partial1384 * write access.1385 */1386 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];1387 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))1388 {1389 if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)1390 memset(pbBuf, 0xff, cbMem);1391 else1392 {1393 int rc;1394 if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))1395 {1396 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);1397 if (rcStrict == VINF_SUCCESS)1398 { /* nothing */ }1399 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))1400 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);1401 else1402 {1403 LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",1404 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));1405 return rcStrict;1406 }1407 }1408 else1409 {1410 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);1411 if (RT_SUCCESS(rc))1412 { /* likely */ }1413 else1414 {1415 LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",1416 GCPhysFirst, rc));1417 return rc;1418 }1419 }1420 }1421 }1422 #ifdef VBOX_STRICT1423 else1424 memset(pbBuf, 0xcc, cbMem);1425 #endif1426 #ifdef VBOX_STRICT1427 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))1428 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);1429 #endif1430 1431 /*1432 * Commit the bounce buffer entry.1433 */1434 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;1435 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;1436 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;1437 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;1438 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;1439 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;1440 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;1441 pVCpu->iem.s.iNextMapping = iMemMap + 1;1442 pVCpu->iem.s.cActiveMappings++;1443 1444 *ppvMem = pbBuf;1445 *pbUnmapInfo = iMemMap | 0x08 | ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);1446 return VINF_SUCCESS;1447 }1448 1449 1450 1451 /**1452 * Commits the guest memory if bounce buffered and unmaps it.1453 *1454 * @returns Strict VBox status code.1455 * @param pVCpu The cross context virtual CPU structure of the calling thread.1456 * @param bUnmapInfo Unmap info set by iemMemMap.1457 */1458 VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT1459 {1460 uintptr_t const iMemMap = bUnmapInfo & 0x7;1461 AssertMsgReturn( (bUnmapInfo & 0x08)1462 && iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)1463 && (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK | 0xf)) == ((unsigned)bUnmapInfo >> 4),1464 ("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess),1465 VERR_NOT_FOUND);1466 1467 /* If it's bounce buffered, we may need to write back the buffer. */1468 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)1469 {1470 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)1471 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);1472 }1473 /* Otherwise unlock it. */1474 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))1475 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);1476 1477 /* Free the entry. */1478 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1479 Assert(pVCpu->iem.s.cActiveMappings != 0);1480 pVCpu->iem.s.cActiveMappings--;1481 return VINF_SUCCESS;1482 }1483 1484 1485 /**1486 * Rolls back the guest memory (conceptually only) and unmaps it.1487 *1488 * @param pVCpu The cross context virtual CPU structure of the calling thread.1489 * @param bUnmapInfo Unmap info set by iemMemMap.1490 */1491 void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT1492 {1493 uintptr_t const iMemMap = bUnmapInfo & 0x7;1494 AssertMsgReturnVoid( (bUnmapInfo & 0x08)1495 && iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)1496 && (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK | 0xf))1497 == ((unsigned)bUnmapInfo >> 4),1498 ("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess));1499 1500 /* Unlock it if necessary. */1501 if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))1502 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);1503 1504 /* Free the entry. */1505 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1506 Assert(pVCpu->iem.s.cActiveMappings != 0);1507 pVCpu->iem.s.cActiveMappings--;1508 }1509 1510 #ifdef IEM_WITH_SETJMP1511 1512 /**1513 * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.1514 *1515 * @param pVCpu The cross context virtual CPU structure of the calling thread.1516 * @param pvMem The mapping.1517 * @param fAccess The kind of access.1518 */1519 void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP1520 {1521 uintptr_t const iMemMap = bUnmapInfo & 0x7;1522 AssertMsgReturnVoid( (bUnmapInfo & 0x08)1523 && iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)1524 && (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK | 0xf))1525 == ((unsigned)bUnmapInfo >> 4),1526 ("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess));1527 1528 /* If it's bounce buffered, we may need to write back the buffer. */1529 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)1530 {1531 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)1532 {1533 VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);1534 if (rcStrict == VINF_SUCCESS)1535 return;1536 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));1537 }1538 }1539 /* Otherwise unlock it. */1540 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))1541 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);1542 1543 /* Free the entry. */1544 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1545 Assert(pVCpu->iem.s.cActiveMappings != 0);1546 pVCpu->iem.s.cActiveMappings--;1547 }1548 1549 1550 /** Fallback for iemMemCommitAndUnmapRwJmp. */1551 void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP1552 {1553 Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE));1554 iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);1555 }1556 1557 1558 /** Fallback for iemMemCommitAndUnmapAtJmp. */1559 void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP1560 {1561 Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE));1562 iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);1563 }1564 1565 1566 /** Fallback for iemMemCommitAndUnmapWoJmp. */1567 void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP1568 {1569 Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);1570 iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);1571 }1572 1573 1574 /** Fallback for iemMemCommitAndUnmapRoJmp. */1575 void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP1576 {1577 Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_READ);1578 iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);1579 }1580 1581 1582 /** Fallback for iemMemRollbackAndUnmapWo. */1583 void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT1584 {1585 Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);1586 iemMemRollbackAndUnmap(pVCpu, bUnmapInfo);1587 }1588 1589 #endif /* IEM_WITH_SETJMP */1590 1591 #ifndef IN_RING31592 /**1593 * Commits the guest memory if bounce buffered and unmaps it, if any bounce1594 * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).1595 *1596 * Allows the instruction to be completed and retired, while the IEM user will1597 * return to ring-3 immediately afterwards and do the postponed writes there.1598 *1599 * @returns VBox status code (no strict statuses). Caller must check1600 * VMCPU_FF_IEM before repeating string instructions and similar stuff.1601 * @param pVCpu The cross context virtual CPU structure of the calling thread.1602 * @param pvMem The mapping.1603 * @param fAccess The kind of access.1604 */1605 VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT1606 {1607 uintptr_t const iMemMap = bUnmapInfo & 0x7;1608 AssertMsgReturn( (bUnmapInfo & 0x08)1609 && iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)1610 && (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK | 0xf))1611 == ((unsigned)bUnmapInfo >> 4),1612 ("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess),1613 VERR_NOT_FOUND);1614 1615 /* If it's bounce buffered, we may need to write back the buffer. */1616 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)1617 {1618 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)1619 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);1620 }1621 /* Otherwise unlock it. */1622 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))1623 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);1624 1625 /* Free the entry. */1626 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1627 Assert(pVCpu->iem.s.cActiveMappings != 0);1628 pVCpu->iem.s.cActiveMappings--;1629 return VINF_SUCCESS;1630 }1631 #endif1632 1633 1634 /**1635 * Rollbacks mappings, releasing page locks and such.1636 *1637 * The caller shall only call this after checking cActiveMappings.1638 *1639 * @param pVCpu The cross context virtual CPU structure of the calling thread.1640 */1641 void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT1642 {1643 Assert(pVCpu->iem.s.cActiveMappings > 0);1644 1645 uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);1646 while (iMemMap-- > 0)1647 {1648 uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;1649 if (fAccess != IEM_ACCESS_INVALID)1650 {1651 AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));1652 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;1653 if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED | IEM_ACCESS_NOT_LOCKED)))1654 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);1655 AssertMsg(pVCpu->iem.s.cActiveMappings > 0,1656 ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",1657 iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,1658 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));1659 pVCpu->iem.s.cActiveMappings--;1660 }1661 }1662 }1663 1664 #undef LOG_GROUP1665 #define LOG_GROUP LOG_GROUP_IEM1666 1667 /** @} */1668 1669 1670 #ifdef LOG_ENABLED1671 /**1672 * Logs the current instruction.1673 * @param pVCpu The cross context virtual CPU structure of the calling EMT.1674 * @param fSameCtx Set if we have the same context information as the VMM,1675 * clear if we may have already executed an instruction in1676 * our debug context. When clear, we assume IEMCPU holds1677 * valid CPU mode info.1678 *1679 * The @a fSameCtx parameter is now misleading and obsolete.1680 * @param pszFunction The IEM function doing the execution.1681 */1682 static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT1683 {1684 # ifdef IN_RING31685 if (LogIs2Enabled())1686 {1687 char szInstr[256];1688 uint32_t cbInstr = 0;1689 if (fSameCtx)1690 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,1691 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,1692 szInstr, sizeof(szInstr), &cbInstr);1693 else1694 {1695 uint32_t fFlags = 0;1696 switch (IEM_GET_CPU_MODE(pVCpu))1697 {1698 case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;1699 case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;1700 case IEMMODE_16BIT:1701 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)1702 fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;1703 else1704 fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;1705 break;1706 }1707 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,1708 szInstr, sizeof(szInstr), &cbInstr);1709 }1710 1711 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;1712 Log2(("**** %s fExec=%x\n"1713 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"1714 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"1715 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"1716 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"1717 " %s\n"1718 , pszFunction, pVCpu->iem.s.fExec,1719 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,1720 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,1721 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,1722 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,1723 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,1724 szInstr));1725 1726 /* This stuff sucks atm. as it fills the log with MSRs. */1727 //if (LogIs3Enabled())1728 // DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);1729 }1730 else1731 # endif1732 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,1733 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));1734 RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);1735 }1736 #endif /* LOG_ENABLED */1737 1738 1739 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX1740 /**1741 * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,1742 * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.1743 *1744 * @returns Modified rcStrict.1745 * @param pVCpu The cross context virtual CPU structure of the calling thread.1746 * @param rcStrict The instruction execution status.1747 */1748 static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT1749 {1750 Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));1751 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))1752 {1753 /* VMX preemption timer takes priority over NMI-window exits. */1754 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))1755 {1756 rcStrict = iemVmxVmexitPreemptTimer(pVCpu);1757 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));1758 }1759 /*1760 * Check remaining intercepts.1761 *1762 * NMI-window and Interrupt-window VM-exits.1763 * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.1764 * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.1765 *1766 * See Intel spec. 26.7.6 "NMI-Window Exiting".1767 * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".1768 */1769 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)1770 && !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)1771 && !TRPMHasTrap(pVCpu))1772 {1773 Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));1774 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)1775 && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))1776 {1777 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);1778 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));1779 }1780 else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)1781 && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))1782 {1783 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);1784 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));1785 }1786 }1787 }1788 /* TPR-below threshold/APIC write has the highest priority. */1789 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))1790 {1791 rcStrict = iemVmxApicWriteEmulation(pVCpu);1792 Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));1793 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));1794 }1795 /* MTF takes priority over VMX-preemption timer. */1796 else1797 {1798 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);1799 Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));1800 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));1801 }1802 return rcStrict;1803 }1804 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */1805 1806 1807 /**1808 * The actual code execution bits of IEMExecOne, IEMExecOneWithPrefetchedByPC,1809 * IEMExecOneBypass and friends.1810 *1811 * Similar code is found in IEMExecLots.1812 *1813 * @return Strict VBox status code.1814 * @param pVCpu The cross context virtual CPU structure of the calling EMT.1815 * @param fExecuteInhibit If set, execute the instruction following CLI,1816 * POP SS and MOV SS,GR.1817 * @param pszFunction The calling function name.1818 */1819 DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)1820 {1821 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));1822 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));1823 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));1824 RT_NOREF_PV(pszFunction);1825 1826 #ifdef IEM_WITH_SETJMP1827 VBOXSTRICTRC rcStrict;1828 IEM_TRY_SETJMP(pVCpu, rcStrict)1829 {1830 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);1831 rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);1832 }1833 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);1834 {1835 pVCpu->iem.s.cLongJumps++;1836 }1837 IEM_CATCH_LONGJMP_END(pVCpu);1838 #else1839 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);1840 VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);1841 #endif1842 if (rcStrict == VINF_SUCCESS)1843 pVCpu->iem.s.cInstructions++;1844 if (pVCpu->iem.s.cActiveMappings > 0)1845 {1846 Assert(rcStrict != VINF_SUCCESS);1847 iemMemRollback(pVCpu);1848 }1849 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));1850 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));1851 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));1852 1853 //#ifdef DEBUG1854 // AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));1855 //#endif1856 1857 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX1858 /*1859 * Perform any VMX nested-guest instruction boundary actions.1860 *1861 * If any of these causes a VM-exit, we must skip executing the next1862 * instruction (would run into stale page tables). A VM-exit makes sure1863 * there is no interrupt-inhibition, so that should ensure we don't go1864 * to try execute the next instruction. Clearing fExecuteInhibit is1865 * problematic because of the setjmp/longjmp clobbering above.1866 */1867 if ( !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER1868 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)1869 || rcStrict != VINF_SUCCESS)1870 { /* likely */ }1871 else1872 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);1873 #endif1874 1875 /* Execute the next instruction as well if a cli, pop ss or1876 mov ss, Gr has just completed successfully. */1877 if ( fExecuteInhibit1878 && rcStrict == VINF_SUCCESS1879 && CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))1880 {1881 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fExec & (IEM_F_BYPASS_HANDLERS | IEM_F_X86_DISREGARD_LOCK));1882 if (rcStrict == VINF_SUCCESS)1883 {1884 #ifdef LOG_ENABLED1885 iemLogCurInstr(pVCpu, false, pszFunction);1886 #endif1887 #ifdef IEM_WITH_SETJMP1888 IEM_TRY_SETJMP_AGAIN(pVCpu, rcStrict)1889 {1890 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);1891 rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);1892 }1893 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);1894 {1895 pVCpu->iem.s.cLongJumps++;1896 }1897 IEM_CATCH_LONGJMP_END(pVCpu);1898 #else1899 IEM_OPCODE_GET_FIRST_U8(&b);1900 rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);1901 #endif1902 if (rcStrict == VINF_SUCCESS)1903 {1904 pVCpu->iem.s.cInstructions++;1905 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX1906 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER1907 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))1908 { /* likely */ }1909 else1910 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);1911 #endif1912 }1913 if (pVCpu->iem.s.cActiveMappings > 0)1914 {1915 Assert(rcStrict != VINF_SUCCESS);1916 iemMemRollback(pVCpu);1917 }1918 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));1919 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));1920 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));1921 }1922 else if (pVCpu->iem.s.cActiveMappings > 0)1923 iemMemRollback(pVCpu);1924 /** @todo drop this after we bake this change into RIP advancing. */1925 CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); /* hope this is correct for all exceptional cases... */1926 }1927 1928 /*1929 * Return value fiddling, statistics and sanity assertions.1930 */1931 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);1932 1933 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));1934 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));1935 return rcStrict;1936 }1937 1938 1939 /**1940 * Execute one instruction.1941 *1942 * @return Strict VBox status code.1943 * @param pVCpu The cross context virtual CPU structure of the calling EMT.1944 */1945 VMM_INT_DECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)1946 {1947 AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */1948 #ifdef LOG_ENABLED1949 iemLogCurInstr(pVCpu, true, "IEMExecOne");1950 #endif1951 1952 /*1953 * Do the decoding and emulation.1954 */1955 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /*fExecOpts*/);1956 if (rcStrict == VINF_SUCCESS)1957 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");1958 else if (pVCpu->iem.s.cActiveMappings > 0)1959 iemMemRollback(pVCpu);1960 1961 if (rcStrict != VINF_SUCCESS)1962 LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",1963 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));1964 return rcStrict;1965 }1966 1967 1968 VMM_INT_DECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,1969 const void *pvOpcodeBytes, size_t cbOpcodeBytes)1970 {1971 VBOXSTRICTRC rcStrict;1972 if ( cbOpcodeBytes1973 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)1974 {1975 iemInitDecoder(pVCpu, 0 /*fExecOpts*/);1976 #ifdef IEM_WITH_CODE_TLB1977 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;1978 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;1979 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);1980 pVCpu->iem.s.offCurInstrStart = 0;1981 pVCpu->iem.s.offInstrNextByte = 0;1982 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;1983 #else1984 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));1985 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);1986 #endif1987 rcStrict = VINF_SUCCESS;1988 }1989 else1990 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /*fExecOpts*/);1991 if (rcStrict == VINF_SUCCESS)1992 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");1993 else if (pVCpu->iem.s.cActiveMappings > 0)1994 iemMemRollback(pVCpu);1995 1996 return rcStrict;1997 }1998 1999 2000 VMM_INT_DECL(VBOXSTRICTRC) IEMExecOneBypass(PVMCPUCC pVCpu)2001 {2002 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_BYPASS_HANDLERS);2003 if (rcStrict == VINF_SUCCESS)2004 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypass");2005 else if (pVCpu->iem.s.cActiveMappings > 0)2006 iemMemRollback(pVCpu);2007 2008 return rcStrict;2009 }2010 2011 2012 VMM_INT_DECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,2013 const void *pvOpcodeBytes, size_t cbOpcodeBytes)2014 {2015 VBOXSTRICTRC rcStrict;2016 if ( cbOpcodeBytes2017 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)2018 {2019 iemInitDecoder(pVCpu, IEM_F_BYPASS_HANDLERS);2020 #ifdef IEM_WITH_CODE_TLB2021 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;2022 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;2023 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);2024 pVCpu->iem.s.offCurInstrStart = 0;2025 pVCpu->iem.s.offInstrNextByte = 0;2026 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;2027 #else2028 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));2029 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);2030 #endif2031 rcStrict = VINF_SUCCESS;2032 }2033 else2034 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_BYPASS_HANDLERS);2035 if (rcStrict == VINF_SUCCESS)2036 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");2037 else if (pVCpu->iem.s.cActiveMappings > 0)2038 iemMemRollback(pVCpu);2039 2040 return rcStrict;2041 }2042 2043 2044 /**2045 * For handling split cacheline lock operations when the host has split-lock2046 * detection enabled.2047 *2048 * This will cause the interpreter to disregard the lock prefix and implicit2049 * locking (xchg).2050 *2051 * @returns Strict VBox status code.2052 * @param pVCpu The cross context virtual CPU structure of the calling EMT.2053 */2054 VMM_INT_DECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)2055 {2056 /*2057 * Do the decoding and emulation.2058 */2059 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_X86_DISREGARD_LOCK);2060 if (rcStrict == VINF_SUCCESS)2061 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");2062 else if (pVCpu->iem.s.cActiveMappings > 0)2063 iemMemRollback(pVCpu);2064 2065 if (rcStrict != VINF_SUCCESS)2066 LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",2067 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));2068 return rcStrict;2069 }2070 2071 2072 /**2073 * Code common to IEMExecLots and IEMExecRecompilerThreaded that attempts to2074 * inject a pending TRPM trap.2075 */2076 VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu)2077 {2078 Assert(TRPMHasTrap(pVCpu));2079 2080 if ( !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)2081 && !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))2082 {2083 /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */2084 #if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)2085 bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);2086 if (fIntrEnabled)2087 {2088 if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))2089 fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;2090 else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))2091 fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));2092 else2093 {2094 Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));2095 fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));2096 }2097 }2098 #else2099 bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;2100 #endif2101 if (fIntrEnabled)2102 {2103 uint8_t u8TrapNo;2104 TRPMEVENT enmType;2105 uint32_t uErrCode;2106 RTGCPTR uCr2;2107 int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /*pu8InstLen*/, NULL /*fIcebp*/);2108 AssertRC(rc2);2109 Assert(enmType == TRPM_HARDWARE_INT);2110 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /*cbInstr*/);2111 2112 TRPMResetTrap(pVCpu);2113 2114 #if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)2115 /* Injecting an event may cause a VM-exit. */2116 if ( rcStrict != VINF_SUCCESS2117 && rcStrict != VINF_IEM_RAISED_XCPT)2118 return iemExecStatusCodeFiddling(pVCpu, rcStrict);2119 #else2120 NOREF(rcStrict);2121 #endif2122 }2123 }2124 2125 return VINF_SUCCESS;2126 }2127 2128 2129 VMM_INT_DECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)2130 {2131 uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;2132 AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));2133 Assert(cMaxInstructions > 0);2134 2135 /*2136 * See if there is an interrupt pending in TRPM, inject it if we can.2137 */2138 /** @todo What if we are injecting an exception and not an interrupt? Is that2139 * possible here? For now we assert it is indeed only an interrupt. */2140 if (!TRPMHasTrap(pVCpu))2141 { /* likely */ }2142 else2143 {2144 VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);2145 if (RT_LIKELY(rcStrict == VINF_SUCCESS))2146 { /*likely */ }2147 else2148 return rcStrict;2149 }2150 2151 /*2152 * Initial decoder init w/ prefetch, then setup setjmp.2153 */2154 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /*fExecOpts*/);2155 if (rcStrict == VINF_SUCCESS)2156 {2157 #ifdef IEM_WITH_SETJMP2158 pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf? */2159 IEM_TRY_SETJMP(pVCpu, rcStrict)2160 #endif2161 {2162 /*2163 * The run loop. We limit ourselves to 4096 instructions right now.2164 */2165 uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;2166 PVMCC pVM = pVCpu->CTX_SUFF(pVM);2167 for (;;)2168 {2169 /*2170 * Log the state.2171 */2172 #ifdef LOG_ENABLED2173 iemLogCurInstr(pVCpu, true, "IEMExecLots");2174 #endif2175 2176 /*2177 * Do the decoding and emulation.2178 */2179 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);2180 rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);2181 #ifdef VBOX_STRICT2182 CPUMAssertGuestRFlagsCookie(pVM, pVCpu);2183 #endif2184 if (RT_LIKELY(rcStrict == VINF_SUCCESS))2185 {2186 Assert(pVCpu->iem.s.cActiveMappings == 0);2187 pVCpu->iem.s.cInstructions++;2188 2189 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX2190 /* Perform any VMX nested-guest instruction boundary actions. */2191 uint64_t fCpu = pVCpu->fLocalForcedActions;2192 if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER2193 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))2194 { /* likely */ }2195 else2196 {2197 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);2198 if (RT_LIKELY(rcStrict == VINF_SUCCESS))2199 fCpu = pVCpu->fLocalForcedActions;2200 else2201 {2202 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2203 break;2204 }2205 }2206 #endif2207 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))2208 {2209 #ifndef VBOX_WITH_NESTED_HWVIRT_VMX2210 uint64_t fCpu = pVCpu->fLocalForcedActions;2211 #endif2212 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR32213 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL2214 | VMCPU_FF_TLB_FLUSH2215 | VMCPU_FF_UNHALT );2216 2217 if (RT_LIKELY( ( !fCpu2218 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))2219 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )2220 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))2221 {2222 if (--cMaxInstructionsGccStupidity > 0)2223 {2224 /* Poll timers every now an then according to the caller's specs. */2225 if ( (cMaxInstructionsGccStupidity & cPollRate) != 02226 || !TMTimerPollBool(pVM, pVCpu))2227 {2228 Assert(pVCpu->iem.s.cActiveMappings == 0);2229 iemReInitDecoder(pVCpu);2230 continue;2231 }2232 }2233 }2234 }2235 Assert(pVCpu->iem.s.cActiveMappings == 0);2236 }2237 else if (pVCpu->iem.s.cActiveMappings > 0)2238 iemMemRollback(pVCpu);2239 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2240 break;2241 }2242 }2243 #ifdef IEM_WITH_SETJMP2244 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);2245 {2246 if (pVCpu->iem.s.cActiveMappings > 0)2247 iemMemRollback(pVCpu);2248 # if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)2249 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2250 # endif2251 pVCpu->iem.s.cLongJumps++;2252 }2253 IEM_CATCH_LONGJMP_END(pVCpu);2254 #endif2255 2256 /*2257 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).2258 */2259 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));2260 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));2261 }2262 else2263 {2264 if (pVCpu->iem.s.cActiveMappings > 0)2265 iemMemRollback(pVCpu);2266 2267 #if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)2268 /*2269 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching2270 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.2271 */2272 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2273 #endif2274 }2275 2276 /*2277 * Maybe re-enter raw-mode and log.2278 */2279 if (rcStrict != VINF_SUCCESS)2280 LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",2281 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));2282 if (pcInstructions)2283 *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;2284 return rcStrict;2285 }2286 2287 2288 /**2289 * Interface used by EMExecuteExec, does exit statistics and limits.2290 *2291 * @returns Strict VBox status code.2292 * @param pVCpu The cross context virtual CPU structure.2293 * @param fWillExit To be defined.2294 * @param cMinInstructions Minimum number of instructions to execute before checking for FFs.2295 * @param cMaxInstructions Maximum number of instructions to execute.2296 * @param cMaxInstructionsWithoutExits2297 * The max number of instructions without exits.2298 * @param pStats Where to return statistics.2299 */2300 VMM_INT_DECL(VBOXSTRICTRC)2301 IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,2302 uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)2303 {2304 NOREF(fWillExit); /** @todo define flexible exit crits */2305 2306 /*2307 * Initialize return stats.2308 */2309 pStats->cInstructions = 0;2310 pStats->cExits = 0;2311 pStats->cMaxExitDistance = 0;2312 pStats->cReserved = 0;2313 2314 /*2315 * Initial decoder init w/ prefetch, then setup setjmp.2316 */2317 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /*fExecOpts*/);2318 if (rcStrict == VINF_SUCCESS)2319 {2320 #ifdef IEM_WITH_SETJMP2321 pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf?!? */2322 IEM_TRY_SETJMP(pVCpu, rcStrict)2323 #endif2324 {2325 #ifdef IN_RING02326 bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);2327 #endif2328 uint32_t cInstructionSinceLastExit = 0;2329 2330 /*2331 * The run loop. We limit ourselves to 4096 instructions right now.2332 */2333 PVM pVM = pVCpu->CTX_SUFF(pVM);2334 for (;;)2335 {2336 /*2337 * Log the state.2338 */2339 #ifdef LOG_ENABLED2340 iemLogCurInstr(pVCpu, true, "IEMExecForExits");2341 #endif2342 2343 /*2344 * Do the decoding and emulation.2345 */2346 uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;2347 2348 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);2349 rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);2350 2351 if ( cPotentialExits != pVCpu->iem.s.cPotentialExits2352 && cInstructionSinceLastExit > 0 /* don't count the first */ )2353 {2354 pStats->cExits += 1;2355 if (cInstructionSinceLastExit > pStats->cMaxExitDistance)2356 pStats->cMaxExitDistance = cInstructionSinceLastExit;2357 cInstructionSinceLastExit = 0;2358 }2359 2360 if (RT_LIKELY(rcStrict == VINF_SUCCESS))2361 {2362 Assert(pVCpu->iem.s.cActiveMappings == 0);2363 pVCpu->iem.s.cInstructions++;2364 pStats->cInstructions++;2365 cInstructionSinceLastExit++;2366 2367 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX2368 /* Perform any VMX nested-guest instruction boundary actions. */2369 uint64_t fCpu = pVCpu->fLocalForcedActions;2370 if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER2371 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))2372 { /* likely */ }2373 else2374 {2375 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);2376 if (RT_LIKELY(rcStrict == VINF_SUCCESS))2377 fCpu = pVCpu->fLocalForcedActions;2378 else2379 {2380 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2381 break;2382 }2383 }2384 #endif2385 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))2386 {2387 #ifndef VBOX_WITH_NESTED_HWVIRT_VMX2388 uint64_t fCpu = pVCpu->fLocalForcedActions;2389 #endif2390 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR32391 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL2392 | VMCPU_FF_TLB_FLUSH2393 | VMCPU_FF_UNHALT );2394 if (RT_LIKELY( ( ( !fCpu2395 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))2396 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))2397 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )2398 || pStats->cInstructions < cMinInstructions))2399 {2400 if (pStats->cInstructions < cMaxInstructions)2401 {2402 if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)2403 {2404 #ifdef IN_RING02405 if ( !fCheckPreemptionPending2406 || !RTThreadPreemptIsPending(NIL_RTTHREAD))2407 #endif2408 {2409 Assert(pVCpu->iem.s.cActiveMappings == 0);2410 iemReInitDecoder(pVCpu);2411 continue;2412 }2413 #ifdef IN_RING02414 rcStrict = VINF_EM_RAW_INTERRUPT;2415 break;2416 #endif2417 }2418 }2419 }2420 Assert(!(fCpu & VMCPU_FF_IEM));2421 }2422 Assert(pVCpu->iem.s.cActiveMappings == 0);2423 }2424 else if (pVCpu->iem.s.cActiveMappings > 0)2425 iemMemRollback(pVCpu);2426 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2427 break;2428 }2429 }2430 #ifdef IEM_WITH_SETJMP2431 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);2432 {2433 if (pVCpu->iem.s.cActiveMappings > 0)2434 iemMemRollback(pVCpu);2435 pVCpu->iem.s.cLongJumps++;2436 }2437 IEM_CATCH_LONGJMP_END(pVCpu);2438 #endif2439 2440 /*2441 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).2442 */2443 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));2444 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));2445 }2446 else2447 {2448 if (pVCpu->iem.s.cActiveMappings > 0)2449 iemMemRollback(pVCpu);2450 2451 #if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)2452 /*2453 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching2454 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.2455 */2456 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);2457 #endif2458 }2459 2460 /*2461 * Maybe re-enter raw-mode and log.2462 */2463 if (rcStrict != VINF_SUCCESS)2464 LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",2465 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,2466 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));2467 return rcStrict;2468 }2469 2470 2471 /**2472 * Injects a trap, fault, abort, software interrupt or external interrupt.2473 *2474 * The parameter list matches TRPMQueryTrapAll pretty closely.2475 *2476 * @returns Strict VBox status code.2477 * @param pVCpu The cross context virtual CPU structure of the calling EMT.2478 * @param u8TrapNo The trap number.2479 * @param enmType What type is it (trap/fault/abort), software2480 * interrupt or hardware interrupt.2481 * @param uErrCode The error code if applicable.2482 * @param uCr2 The CR2 value if applicable.2483 * @param cbInstr The instruction length (only relevant for2484 * software interrupts).2485 * @note x86 specific, but difficult to move due to iemInitDecoder dep.2486 */2487 VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,2488 uint8_t cbInstr)2489 {2490 iemInitDecoder(pVCpu, 0 /*fExecOpts*/); /** @todo wrong init function! */2491 #ifdef DBGFTRACE_ENABLED2492 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",2493 u8TrapNo, enmType, uErrCode, uCr2);2494 #endif2495 2496 uint32_t fFlags;2497 switch (enmType)2498 {2499 case TRPM_HARDWARE_INT:2500 Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));2501 fFlags = IEM_XCPT_FLAGS_T_EXT_INT;2502 uErrCode = uCr2 = 0;2503 break;2504 2505 case TRPM_SOFTWARE_INT:2506 Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));2507 fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;2508 uErrCode = uCr2 = 0;2509 break;2510 2511 case TRPM_TRAP:2512 case TRPM_NMI: /** @todo Distinguish NMI from exception 2. */2513 Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));2514 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;2515 if (u8TrapNo == X86_XCPT_PF)2516 fFlags |= IEM_XCPT_FLAGS_CR2;2517 switch (u8TrapNo)2518 {2519 case X86_XCPT_DF:2520 case X86_XCPT_TS:2521 case X86_XCPT_NP:2522 case X86_XCPT_SS:2523 case X86_XCPT_PF:2524 case X86_XCPT_AC:2525 case X86_XCPT_GP:2526 fFlags |= IEM_XCPT_FLAGS_ERR;2527 break;2528 }2529 break;2530 2531 IEM_NOT_REACHED_DEFAULT_CASE_RET();2532 }2533 2534 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);2535 2536 if (pVCpu->iem.s.cActiveMappings > 0)2537 iemMemRollback(pVCpu);2538 2539 return rcStrict;2540 }2541 2542 2543 /**2544 * Injects the active TRPM event.2545 *2546 * @returns Strict VBox status code.2547 * @param pVCpu The cross context virtual CPU structure.2548 */2549 VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)2550 {2551 #ifndef IEM_IMPLEMENTS_TASKSWITCH2552 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));2553 #else2554 uint8_t u8TrapNo;2555 TRPMEVENT enmType;2556 uint32_t uErrCode;2557 RTGCUINTPTR uCr2;2558 uint8_t cbInstr;2559 int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);2560 if (RT_FAILURE(rc))2561 return rc;2562 2563 /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle2564 * ICEBP \#DB injection as a special case. */2565 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);2566 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM2567 if (rcStrict == VINF_SVM_VMEXIT)2568 rcStrict = VINF_SUCCESS;2569 #endif2570 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX2571 if (rcStrict == VINF_VMX_VMEXIT)2572 rcStrict = VINF_SUCCESS;2573 #endif2574 /** @todo Are there any other codes that imply the event was successfully2575 * delivered to the guest? See @bugref{6607}. */2576 if ( rcStrict == VINF_SUCCESS2577 || rcStrict == VINF_IEM_RAISED_XCPT)2578 TRPMResetTrap(pVCpu);2579 2580 return rcStrict;2581 #endif2582 }2583 2584 2585 VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)2586 {2587 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);2588 return VERR_NOT_IMPLEMENTED;2589 }2590 2591 2592 VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)2593 {2594 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);2595 return VERR_NOT_IMPLEMENTED;2596 }2597 2598 #ifdef IN_RING32599 2600 /**2601 * Handles the unlikely and probably fatal merge cases.2602 *2603 * @returns Merged status code.2604 * @param rcStrict Current EM status code.2605 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge2606 * with @a rcStrict.2607 * @param iMemMap The memory mapping index. For error reporting only.2608 * @param pVCpu The cross context virtual CPU structure of the calling2609 * thread, for error reporting only.2610 */2611 DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,2612 unsigned iMemMap, PVMCPUCC pVCpu)2613 {2614 if (RT_FAILURE_NP(rcStrict))2615 return rcStrict;2616 2617 if (RT_FAILURE_NP(rcStrictCommit))2618 return rcStrictCommit;2619 2620 if (rcStrict == rcStrictCommit)2621 return rcStrictCommit;2622 2623 AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",2624 VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,2625 pVCpu->iem.s.aMemMappings[iMemMap].fAccess,2626 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,2627 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));2628 return VERR_IOM_FF_STATUS_IPE;2629 }2630 2631 2632 /**2633 * Helper for IOMR3ProcessForceFlag.2634 *2635 * @returns Merged status code.2636 * @param rcStrict Current EM status code.2637 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge2638 * with @a rcStrict.2639 * @param iMemMap The memory mapping index. For error reporting only.2640 * @param pVCpu The cross context virtual CPU structure of the calling2641 * thread, for error reporting only.2642 */2643 DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)2644 {2645 /* Simple. */2646 if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))2647 return rcStrictCommit;2648 2649 if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))2650 return rcStrict;2651 2652 /* EM scheduling status codes. */2653 if (RT_LIKELY( rcStrict >= VINF_EM_FIRST2654 && rcStrict <= VINF_EM_LAST))2655 {2656 if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST2657 && rcStrictCommit <= VINF_EM_LAST))2658 return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;2659 }2660 2661 /* Unlikely */2662 return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);2663 }2664 2665 2666 /**2667 * Called by force-flag handling code when VMCPU_FF_IEM is set.2668 *2669 * @returns Merge between @a rcStrict and what the commit operation returned.2670 * @param pVM The cross context VM structure.2671 * @param pVCpu The cross context virtual CPU structure of the calling EMT.2672 * @param rcStrict The status code returned by ring-0 or raw-mode.2673 */2674 VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)2675 {2676 /*2677 * Reset the pending commit.2678 */2679 AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)2680 & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),2681 ("%#x %#x %#x\n",2682 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));2683 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);2684 2685 /*2686 * Commit the pending bounce buffers (usually just one).2687 */2688 unsigned cBufs = 0;2689 unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);2690 while (iMemMap-- > 0)2691 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))2692 {2693 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);2694 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);2695 Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);2696 2697 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;2698 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;2699 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];2700 2701 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)2702 {2703 VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,2704 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,2705 pbBuf,2706 cbFirst,2707 PGMACCESSORIGIN_IEM);2708 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);2709 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",2710 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,2711 VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));2712 }2713 2714 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)2715 {2716 VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,2717 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,2718 pbBuf + cbFirst,2719 cbSecond,2720 PGMACCESSORIGIN_IEM);2721 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);2722 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",2723 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,2724 VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));2725 }2726 cBufs++;2727 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;2728 }2729 2730 AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,2731 ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,2732 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));2733 pVCpu->iem.s.cActiveMappings = 0;2734 return rcStrict;2735 }2736 2737 #endif /* IN_RING3 */2738
Note:
See TracChangeset
for help on using the changeset viewer.
