Changeset 108260 in vbox
- Timestamp:
- Feb 17, 2025 3:24:14 PM (3 months ago)
- svn:sync-xref-src-repo-rev:
- 167582
- Location:
- trunk/src/VBox/VMM
- Files:
-
- 1 added
- 19 edited
- 3 copied
-
VMMAll/IEMAll.cpp (modified) (1 diff)
-
VMMAll/IEMAllMem.cpp (modified) (1 diff)
-
VMMAll/IEMAllThrdFuncsBltIn.cpp (modified) (1 diff)
-
VMMAll/IEMAllThrdRecompiler.cpp (modified) (2 diffs)
-
VMMAll/target-x86/IEMAll-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllCImpl-x86.cpp (modified) (3 diffs)
-
VMMAll/target-x86/IEMAllCImplSvmInstr-x86.cpp (modified) (2 diffs)
-
VMMAll/target-x86/IEMAllCImplVmxInstr-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllExec-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllHlpFpu-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllIntprTables-x86.h (modified) (1 diff)
-
VMMAll/target-x86/IEMAllMem-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllOpHlp-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllOpcodeFetch-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllThrdFuncs-x86.cpp (modified) (1 diff)
-
VMMAll/target-x86/IEMAllThrdTables-x86.h (modified) (1 diff)
-
VMMAll/target-x86/IEMAllXcpt-x86.cpp (modified) (2 diffs)
-
VMMAll/target-x86/IEMInline-x86.h (copied) (copied from trunk/src/VBox/VMM/include/IEMInline.h ) (12 diffs)
-
VMMAll/target-x86/IEMInlineDecode-x86.h (copied) (copied from trunk/src/VBox/VMM/include/IEMInline.h ) (5 diffs)
-
VMMAll/target-x86/IEMInlineMem-x86.h (copied) (copied from trunk/src/VBox/VMM/include/IEMInline.h ) (5 diffs)
-
VMMAll/target-x86/Makefile.kup (added)
-
VMMR3/IEMR3.cpp (modified) (1 diff)
-
include/IEMInline.h (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/VBox/VMM/VMMAll/IEMAll.cpp
r108249 r108260 158 158 159 159 #include "IEMInline.h" 160 #ifdef VBOX_VMM_TARGET_X86 161 # include "target-x86/IEMInline-x86.h" 162 # include "target-x86/IEMInlineDecode-x86.h" 163 #endif 160 164 161 165 -
trunk/src/VBox/VMM/VMMAll/IEMAllMem.cpp
r108250 r108260 50 50 #include "IEMInline.h" 51 51 #ifdef VBOX_VMM_TARGET_X86 52 # include "target-x86/IEMInline-x86.h" /* not really required. sigh. */ 52 53 # include "target-x86/IEMAllTlbInline-x86.h" 53 54 #endif -
trunk/src/VBox/VMM/VMMAll/IEMAllThrdFuncsBltIn.cpp
r108195 r108260 71 71 72 72 #include "IEMInline.h" 73 #ifdef VBOX_VMM_TARGET_X86 74 # include "target-x86/IEMInline-x86.h" 75 #endif 73 76 74 77 -
trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp
r108195 r108260 55 55 #include <VBox/vmm/iem.h> 56 56 #include <VBox/vmm/cpum.h> 57 #include <VBox/vmm/pdmapic.h>58 #include <VBox/vmm/pdm.h>59 #include <VBox/vmm/pgm.h>60 #include <VBox/vmm/iom.h>61 #include <VBox/vmm/em.h>62 #include <VBox/vmm/hm.h>63 #include <VBox/vmm/nem.h>64 #include <VBox/vmm/gim.h>65 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM66 # include <VBox/vmm/em.h>67 # include <VBox/vmm/hm_svm.h>68 #endif69 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX70 # include <VBox/vmm/hmvmxinline.h>71 #endif72 57 #include <VBox/vmm/tm.h> 73 58 #include <VBox/vmm/dbgf.h> … … 89 74 #include <iprt/x86.h> 90 75 91 #ifndef TST_IEM_CHECK_MC 92 # include "IEMInline.h" 93 # include "IEMOpHlp.h" 94 # include "IEMMc.h" 95 #endif 76 #include "IEMInline.h" 77 #ifdef VBOX_VMM_TARGET_X86 78 # include "target-x86/IEMInline-x86.h" 79 # include "target-x86/IEMInlineDecode-x86.h" 80 #endif 81 #include "IEMOpHlp.h" 82 #include "IEMMc.h" 96 83 97 84 #include "IEMThreadedFunctions.h" -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAll-x86.cpp
r108246 r108260 43 43 #include <VBox/param.h> 44 44 #include <iprt/assert.h> 45 #include <iprt/errcore.h> 45 46 #include <iprt/string.h> 46 47 #include <iprt/x86.h> 47 48 48 #include "IEMInline .h"49 #include "IEMInline-x86.h" /* iemRegFinishClearingRF */ 49 50 50 51 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllCImpl-x86.cpp
r108196 r108260 39 39 #include <VBox/vmm/cpum.h> 40 40 #include <VBox/vmm/pdmapic.h> 41 #include <VBox/vmm/pdm.h>42 41 #include <VBox/vmm/pgm.h> 43 42 #include <VBox/vmm/iom.h> 44 43 #include <VBox/vmm/em.h> 45 #include <VBox/vmm/hm.h>46 #include <VBox/vmm/nem.h>47 #include <VBox/vmm/gim.h>48 44 #include <VBox/vmm/gcm.h> 49 45 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM 50 # include <VBox/vmm/em.h>51 46 # include <VBox/vmm/hm_svm.h> 52 47 #endif … … 65 60 #include <VBox/err.h> 66 61 #include <VBox/param.h> 67 #include <VBox/dis.h>68 #include <iprt/asm-math.h>69 62 #include <iprt/assert.h> 70 63 #include <iprt/string.h> … … 72 65 73 66 #include "IEMInline.h" 67 #include "IEMInline-x86.h" 68 #include "IEMInlineMem-x86.h" 74 69 75 70 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllCImplSvmInstr-x86.cpp
r108196 r108260 36 36 #endif 37 37 #include <VBox/vmm/iem.h> 38 #include <VBox/vmm/pdmapic.h>39 38 #include <VBox/vmm/cpum.h> 40 39 #include <VBox/vmm/dbgf.h> … … 58 57 59 58 #include "IEMInline.h" 59 #include "IEMInline-x86.h" 60 60 61 61 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM /* Almost the whole file. */ -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllCImplVmxInstr-x86.cpp
r108196 r108260 59 59 60 60 #include "IEMInline.h" 61 #include "IEMInline-x86.h" 61 62 62 63 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllExec-x86.cpp
r108220 r108260 50 50 51 51 #include "IEMInline.h" 52 #include "IEMInline-x86.h" 52 53 53 54 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllHlpFpu-x86.cpp
r108226 r108260 50 50 #include <iprt/x86.h> 51 51 52 #include "IEMInline .h"52 #include "IEMInline-x86.h" 53 53 54 54 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllIntprTables-x86.h
r108205 r108260 78 78 #ifndef TST_IEM_CHECK_MC 79 79 # include "IEMInline.h" 80 # include "IEMInline-x86.h" 81 # include "IEMInlineDecode-x86.h" 82 # include "IEMInlineMem-x86.h" 80 83 # include "IEMOpHlp.h" 81 84 # include "IEMMc.h" -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllMem-x86.cpp
r108232 r108260 48 48 49 49 #include "IEMInline.h" 50 #include "IEMInline-x86.h" 51 #include "IEMInlineMem-x86.h" 50 52 #include "IEMAllTlbInline-x86.h" 51 53 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllOpHlp-x86.cpp
r108243 r108260 46 46 #include <iprt/x86.h> 47 47 48 #include "IEMInline .h"48 #include "IEMInlineDecode-x86.h" 49 49 50 50 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllOpcodeFetch-x86.cpp
r108246 r108260 44 44 #include <VBox/param.h> 45 45 #include <iprt/assert.h> 46 #include <iprt/errcore.h> 46 47 #include <iprt/string.h> 47 48 #include <iprt/x86.h> 48 49 49 50 #include "IEMInline.h" 50 #ifdef VBOX_VMM_TARGET_X86 51 # include "IEMAllTlbInline-x86.h" 52 #endif 51 #include "IEMInline-x86.h" 52 #include "IEMAllTlbInline-x86.h" 53 53 54 54 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllThrdFuncs-x86.cpp
r108204 r108260 71 71 72 72 #include "IEMInline.h" 73 #include "IEMInline-x86.h" 74 #include "IEMInlineMem-x86.h" 73 75 #include "IEMMc.h" 74 76 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllThrdTables-x86.h
r108205 r108260 81 81 #ifndef TST_IEM_CHECK_MC 82 82 # include "IEMInline.h" 83 # include "IEMInline-x86.h" 84 # include "IEMInlineDecode-x86.h" 85 # include "IEMInlineMem-x86.h" 83 86 # include "IEMOpHlp.h" 84 87 # include "IEMMc.h" -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllXcpt-x86.cpp
r108248 r108260 37 37 #include <VBox/vmm/iem.h> 38 38 #include <VBox/vmm/cpum.h> 39 #include <VBox/vmm/ pdmapic.h>39 #include <VBox/vmm/em.h> 40 40 #include <VBox/vmm/pdm.h> 41 #include <VBox/vmm/pgm.h>42 #include <VBox/vmm/iom.h>43 #include <VBox/vmm/em.h>44 #include <VBox/vmm/hm.h>45 #include <VBox/vmm/nem.h>46 41 #include <VBox/vmm/gcm.h> 47 #include <VBox/vmm/gim.h>48 42 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM 49 # include <VBox/vmm/em.h>50 43 # include <VBox/vmm/hm_svm.h> 51 44 #endif … … 61 54 #include <VBox/err.h> 62 55 #include <VBox/param.h> 63 #include <VBox/dis.h>64 #include <iprt/asm-math.h>65 #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)66 # include <iprt/asm-amd64-x86.h>67 #elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32)68 # include <iprt/asm-arm.h>69 #endif70 56 #include <iprt/assert.h> 71 57 #include <iprt/string.h> 72 58 #include <iprt/x86.h> 73 59 74 #include "IEMInline .h"60 #include "IEMInline-x86.h" 75 61 76 62 -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMInline-x86.h
r108246 r108260 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - Inlined Functions .3 * IEM - Interpreted Execution Manager - Inlined Functions, x86 target. 4 4 */ 5 5 … … 26 26 */ 27 27 28 #ifndef VMM_INCLUDED_SRC_ include_IEMInline_h29 #define VMM_INCLUDED_SRC_ include_IEMInline_h28 #ifndef VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInline_x86_h 29 #define VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInline_x86_h 30 30 #ifndef RT_WITHOUT_PRAGMA_ONCE 31 31 # pragma once … … 33 33 34 34 #include <VBox/err.h> 35 36 37 /**38 * Makes status code addjustments (pass up from I/O and access handler)39 * as well as maintaining statistics.40 *41 * @returns Strict VBox status code to pass up.42 * @param pVCpu The cross context virtual CPU structure of the calling thread.43 * @param rcStrict The status from executing an instruction.44 */45 DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT46 {47 if (rcStrict != VINF_SUCCESS)48 {49 /* Deal with the cases that should be treated as VINF_SUCCESS first. */50 if ( rcStrict == VINF_IEM_YIELD_PENDING_FF51 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */52 || rcStrict == VINF_VMX_VMEXIT53 #endif54 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM55 || rcStrict == VINF_SVM_VMEXIT56 #endif57 )58 {59 rcStrict = pVCpu->iem.s.rcPassUp;60 if (RT_LIKELY(rcStrict == VINF_SUCCESS))61 { /* likely */ }62 else63 pVCpu->iem.s.cRetPassUpStatus++;64 }65 else if (RT_SUCCESS(rcStrict))66 {67 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)68 || rcStrict == VINF_IOM_R3_IOPORT_READ69 || rcStrict == VINF_IOM_R3_IOPORT_WRITE70 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE71 || rcStrict == VINF_IOM_R3_MMIO_READ72 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE73 || rcStrict == VINF_IOM_R3_MMIO_WRITE74 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE75 || rcStrict == VINF_CPUM_R3_MSR_READ76 || rcStrict == VINF_CPUM_R3_MSR_WRITE77 || rcStrict == VINF_EM_RAW_EMULATE_INSTR78 || rcStrict == VINF_EM_RAW_TO_R379 || rcStrict == VINF_EM_TRIPLE_FAULT80 || rcStrict == VINF_EM_EMULATE_SPLIT_LOCK81 || rcStrict == VINF_GIM_R3_HYPERCALL82 /* raw-mode / virt handlers only: */83 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT84 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT85 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT86 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT87 || rcStrict == VINF_SELM_SYNC_GDT88 || rcStrict == VINF_CSAM_PENDING_ACTION89 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE90 /* nested hw.virt codes: */91 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE92 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR93 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));94 /** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */95 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;96 if (rcPassUp == VINF_SUCCESS)97 pVCpu->iem.s.cRetInfStatuses++;98 else if ( rcPassUp < VINF_EM_FIRST99 || rcPassUp > VINF_EM_LAST100 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))101 {102 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));103 pVCpu->iem.s.cRetPassUpStatus++;104 rcStrict = rcPassUp;105 }106 else107 {108 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));109 pVCpu->iem.s.cRetInfStatuses++;110 }111 }112 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)113 pVCpu->iem.s.cRetAspectNotImplemented++;114 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)115 pVCpu->iem.s.cRetInstrNotImplemented++;116 else117 pVCpu->iem.s.cRetErrStatuses++;118 }119 else120 {121 rcStrict = pVCpu->iem.s.rcPassUp;122 if (rcStrict != VINF_SUCCESS)123 pVCpu->iem.s.cRetPassUpStatus++;124 }125 126 /* Just clear it here as well. */127 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;128 129 return rcStrict;130 }131 132 133 /**134 * Sets the pass up status.135 *136 * @returns VINF_SUCCESS.137 * @param pVCpu The cross context virtual CPU structure of the138 * calling thread.139 * @param rcPassUp The pass up status. Must be informational.140 * VINF_SUCCESS is not allowed.141 */142 DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT143 {144 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);145 146 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;147 if (rcOldPassUp == VINF_SUCCESS)148 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);149 /* If both are EM scheduling codes, use EM priority rules. */150 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST151 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)152 {153 if (rcPassUp < rcOldPassUp)154 {155 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));156 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);157 }158 else159 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));160 }161 /* Override EM scheduling with specific status code. */162 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)163 {164 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));165 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);166 }167 /* Don't override specific status code, first come first served. */168 else169 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));170 return VINF_SUCCESS;171 }172 35 173 36 … … 383 246 } 384 247 385 /** 386 * Calculates the the IEM_F_XXX flags. 387 * 388 * @returns IEM_F_XXX combination match the current CPU state. 389 * @param pVCpu The cross context virtual CPU structure of the 390 * calling thread. 391 */ 248 392 249 DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT 393 250 { … … 429 286 430 287 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 431 432 # if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */ 433 434 /** 435 * Initializes the execution state. 436 * 437 * @param pVCpu The cross context virtual CPU structure of the 438 * calling thread. 439 * @param fExecOpts Optional execution flags: 440 * - IEM_F_BYPASS_HANDLERS 441 * - IEM_F_X86_DISREGARD_LOCK 442 * 443 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal 444 * side-effects in strict builds. 445 */ 446 DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT 447 { 448 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK); 449 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); 288 # ifdef VBOX_STRICT 289 DECLINLINE(void) iemInitExecTargetStrict(PVMCPUCC pVCpu) RT_NOEXCEPT 290 { 450 291 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); 451 292 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); … … 457 298 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr)); 458 299 459 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;460 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts;461 pVCpu->iem.s.cActiveMappings = 0;462 pVCpu->iem.s.iNextMapping = 0;463 464 # ifdef VBOX_STRICT465 300 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe; 466 301 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe; … … 478 313 pVCpu->iem.s.fEvexStuff = 127; 479 314 pVCpu->iem.s.uFpuOpcode = UINT16_MAX; 480 # 315 # ifdef IEM_WITH_CODE_TLB 481 316 pVCpu->iem.s.offInstrNextByte = UINT16_MAX; 482 317 pVCpu->iem.s.pbInstrBuf = NULL; … … 485 320 pVCpu->iem.s.offCurInstrStart = INT16_MAX; 486 321 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff); 487 # 322 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 488 323 pVCpu->iem.s.offOpcode = 127; 489 # 490 # 324 # endif 325 # else 491 326 pVCpu->iem.s.offOpcode = 127; 492 327 pVCpu->iem.s.cbOpcode = 127; 493 # endif494 # endif /* VBOX_STRICT */495 }496 497 498 # if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)499 /**500 * Performs a minimal reinitialization of the execution state.501 *502 * This is intended to be used by VM-exits, SMM, LOADALL and other similar503 * 'world-switch' types operations on the CPU. Currently only nested504 * hardware-virtualization uses it.505 *506 * @param pVCpu The cross context virtual CPU structure of the calling EMT.507 * @param cbInstr The instruction length (for flushing).508 */509 DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT510 {511 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);512 iemOpcodeFlushHeavy(pVCpu, cbInstr);513 }514 328 # endif 515 516 # endif /* VBOX_INCLUDED_vmm_dbgf_h || DOXYGEN_RUNNING */ 517 518 /** 519 * Counterpart to #iemInitExec that undoes evil strict-build stuff. 520 * 521 * @param pVCpu The cross context virtual CPU structure of the 522 * calling thread. 523 */ 524 DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT 525 { 526 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */ 527 # ifdef VBOX_STRICT 528 # ifdef IEM_WITH_CODE_TLB 529 NOREF(pVCpu); 530 # else 531 pVCpu->iem.s.cbOpcode = 0; 532 # endif 533 # else 534 NOREF(pVCpu); 535 # endif 536 } 537 538 539 /** 540 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter. 541 * 542 * Only calling iemRCRawMaybeReenter in raw-mode, obviously. 543 * 544 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller. 545 * @param pVCpu The cross context virtual CPU structure of the calling thread. 546 * @param rcStrict The status code to fiddle. 547 */ 548 DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT 549 { 550 iemUninitExec(pVCpu); 551 return iemExecStatusCodeFiddling(pVCpu, rcStrict); 552 } 329 } 330 # endif /* VBOX_STRICT */ 331 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */ 553 332 554 333 … … 561 340 * @param a_cbMin The minimum length. 562 341 */ 563 # 342 #define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \ 564 343 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \ 565 344 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH) 566 567 568 # ifndef IEM_WITH_SETJMP569 570 /**571 * Fetches the first opcode byte.572 *573 * @returns Strict VBox status code.574 * @param pVCpu The cross context virtual CPU structure of the575 * calling thread.576 * @param pu8 Where to return the opcode byte.577 */578 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT579 {580 /*581 * Check for hardware instruction breakpoints.582 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs.583 */584 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))585 { /* likely */ }586 else587 {588 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,589 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base,590 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS)591 || IEM_IS_GUEST_CPU_AMD(pVCpu));592 if (RT_LIKELY(rcStrict == VINF_SUCCESS))593 { /* likely */ }594 else595 {596 *pu8 = 0xff; /* shut up gcc. sigh */597 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)598 return iemRaiseDebugException(pVCpu);599 return rcStrict;600 }601 }602 603 /*604 * Fetch the first opcode byte.605 */606 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;607 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))608 {609 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;610 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];611 return VINF_SUCCESS;612 }613 return iemOpcodeGetNextU8Slow(pVCpu, pu8);614 }615 616 # else /* IEM_WITH_SETJMP */617 618 /**619 * Fetches the first opcode byte, longjmp on error.620 *621 * @returns The opcode byte.622 * @param pVCpu The cross context virtual CPU structure of the calling thread.623 */624 DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP625 {626 /*627 * Check for hardware instruction breakpoints.628 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs.629 */630 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))631 { /* likely */ }632 else633 {634 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,635 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base,636 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS)637 || IEM_IS_GUEST_CPU_AMD(pVCpu));638 if (RT_LIKELY(rcStrict == VINF_SUCCESS))639 { /* likely */ }640 else641 {642 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)643 rcStrict = iemRaiseDebugException(pVCpu);644 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));645 }646 }647 648 /*649 * Fetch the first opcode byte.650 */651 # ifdef IEM_WITH_CODE_TLB652 uint8_t bRet;653 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;654 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;655 if (RT_LIKELY( pbBuf != NULL656 && offBuf < pVCpu->iem.s.cbInstrBuf))657 {658 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;659 bRet = pbBuf[offBuf];660 }661 else662 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);663 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF664 Assert(pVCpu->iem.s.offOpcode == 0);665 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;666 # endif667 return bRet;668 669 # else /* !IEM_WITH_CODE_TLB */670 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;671 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))672 {673 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;674 return pVCpu->iem.s.abOpcode[offOpcode];675 }676 return iemOpcodeGetNextU8SlowJmp(pVCpu);677 # endif678 }679 680 # endif /* IEM_WITH_SETJMP */681 682 /**683 * Fetches the first opcode byte, returns/throws automatically on failure.684 *685 * @param a_pu8 Where to return the opcode byte.686 * @remark Implicitly references pVCpu.687 */688 # ifndef IEM_WITH_SETJMP689 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) \690 do \691 { \692 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \693 if (rcStrict2 == VINF_SUCCESS) \694 { /* likely */ } \695 else \696 return rcStrict2; \697 } while (0)698 # else699 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))700 # endif /* IEM_WITH_SETJMP */701 702 703 # ifndef IEM_WITH_SETJMP704 705 /**706 * Fetches the next opcode byte.707 *708 * @returns Strict VBox status code.709 * @param pVCpu The cross context virtual CPU structure of the710 * calling thread.711 * @param pu8 Where to return the opcode byte.712 */713 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT714 {715 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;716 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))717 {718 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;719 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];720 return VINF_SUCCESS;721 }722 return iemOpcodeGetNextU8Slow(pVCpu, pu8);723 }724 725 # else /* IEM_WITH_SETJMP */726 727 /**728 * Fetches the next opcode byte, longjmp on error.729 *730 * @returns The opcode byte.731 * @param pVCpu The cross context virtual CPU structure of the calling thread.732 */733 DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP734 {735 # ifdef IEM_WITH_CODE_TLB736 uint8_t bRet;737 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;738 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;739 if (RT_LIKELY( pbBuf != NULL740 && offBuf < pVCpu->iem.s.cbInstrBuf))741 {742 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;743 bRet = pbBuf[offBuf];744 }745 else746 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);747 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF748 Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));749 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;750 # endif751 return bRet;752 753 # else /* !IEM_WITH_CODE_TLB */754 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;755 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))756 {757 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;758 return pVCpu->iem.s.abOpcode[offOpcode];759 }760 return iemOpcodeGetNextU8SlowJmp(pVCpu);761 # endif762 }763 764 # endif /* IEM_WITH_SETJMP */765 766 /**767 * Fetches the next opcode byte, returns automatically on failure.768 *769 * @param a_pu8 Where to return the opcode byte.770 * @remark Implicitly references pVCpu.771 */772 # ifndef IEM_WITH_SETJMP773 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) \774 do \775 { \776 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \777 if (rcStrict2 == VINF_SUCCESS) \778 { /* likely */ } \779 else \780 return rcStrict2; \781 } while (0)782 # else783 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))784 # endif /* IEM_WITH_SETJMP */785 786 787 # ifndef IEM_WITH_SETJMP788 /**789 * Fetches the next signed byte from the opcode stream.790 *791 * @returns Strict VBox status code.792 * @param pVCpu The cross context virtual CPU structure of the calling thread.793 * @param pi8 Where to return the signed byte.794 */795 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT796 {797 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);798 }799 # endif /* !IEM_WITH_SETJMP */800 801 802 /**803 * Fetches the next signed byte from the opcode stream, returning automatically804 * on failure.805 *806 * @param a_pi8 Where to return the signed byte.807 * @remark Implicitly references pVCpu.808 */809 # ifndef IEM_WITH_SETJMP810 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) \811 do \812 { \813 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \814 if (rcStrict2 != VINF_SUCCESS) \815 return rcStrict2; \816 } while (0)817 # else /* IEM_WITH_SETJMP */818 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))819 820 # endif /* IEM_WITH_SETJMP */821 822 823 # ifndef IEM_WITH_SETJMP824 /**825 * Fetches the next signed byte from the opcode stream, extending it to826 * unsigned 16-bit.827 *828 * @returns Strict VBox status code.829 * @param pVCpu The cross context virtual CPU structure of the calling thread.830 * @param pu16 Where to return the unsigned word.831 */832 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT833 {834 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;835 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))836 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);837 838 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];839 pVCpu->iem.s.offOpcode = offOpcode + 1;840 return VINF_SUCCESS;841 }842 # endif /* !IEM_WITH_SETJMP */843 844 /**845 * Fetches the next signed byte from the opcode stream and sign-extending it to846 * a word, returning automatically on failure.847 *848 * @param a_pu16 Where to return the word.849 * @remark Implicitly references pVCpu.850 */851 # ifndef IEM_WITH_SETJMP852 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \853 do \854 { \855 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \856 if (rcStrict2 != VINF_SUCCESS) \857 return rcStrict2; \858 } while (0)859 # else860 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))861 # endif862 863 # ifndef IEM_WITH_SETJMP864 /**865 * Fetches the next signed byte from the opcode stream, extending it to866 * unsigned 32-bit.867 *868 * @returns Strict VBox status code.869 * @param pVCpu The cross context virtual CPU structure of the calling thread.870 * @param pu32 Where to return the unsigned dword.871 */872 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT873 {874 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;875 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))876 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);877 878 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];879 pVCpu->iem.s.offOpcode = offOpcode + 1;880 return VINF_SUCCESS;881 }882 # endif /* !IEM_WITH_SETJMP */883 884 /**885 * Fetches the next signed byte from the opcode stream and sign-extending it to886 * a word, returning automatically on failure.887 *888 * @param a_pu32 Where to return the word.889 * @remark Implicitly references pVCpu.890 */891 # ifndef IEM_WITH_SETJMP892 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \893 do \894 { \895 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \896 if (rcStrict2 != VINF_SUCCESS) \897 return rcStrict2; \898 } while (0)899 # else900 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))901 # endif902 903 904 # ifndef IEM_WITH_SETJMP905 /**906 * Fetches the next signed byte from the opcode stream, extending it to907 * unsigned 64-bit.908 *909 * @returns Strict VBox status code.910 * @param pVCpu The cross context virtual CPU structure of the calling thread.911 * @param pu64 Where to return the unsigned qword.912 */913 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT914 {915 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;916 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))917 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);918 919 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];920 pVCpu->iem.s.offOpcode = offOpcode + 1;921 return VINF_SUCCESS;922 }923 # endif /* !IEM_WITH_SETJMP */924 925 /**926 * Fetches the next signed byte from the opcode stream and sign-extending it to927 * a word, returning automatically on failure.928 *929 * @param a_pu64 Where to return the word.930 * @remark Implicitly references pVCpu.931 */932 # ifndef IEM_WITH_SETJMP933 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \934 do \935 { \936 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \937 if (rcStrict2 != VINF_SUCCESS) \938 return rcStrict2; \939 } while (0)940 # else941 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))942 # endif943 944 945 # ifndef IEM_WITH_SETJMP946 947 /**948 * Fetches the next opcode word.949 *950 * @returns Strict VBox status code.951 * @param pVCpu The cross context virtual CPU structure of the calling thread.952 * @param pu16 Where to return the opcode word.953 */954 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT955 {956 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;957 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))958 {959 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;960 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS961 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];962 # else963 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);964 # endif965 return VINF_SUCCESS;966 }967 return iemOpcodeGetNextU16Slow(pVCpu, pu16);968 }969 970 # else /* IEM_WITH_SETJMP */971 972 /**973 * Fetches the next opcode word, longjmp on error.974 *975 * @returns The opcode word.976 * @param pVCpu The cross context virtual CPU structure of the calling thread.977 */978 DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP979 {980 # ifdef IEM_WITH_CODE_TLB981 uint16_t u16Ret;982 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;983 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;984 if (RT_LIKELY( pbBuf != NULL985 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))986 {987 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;988 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS989 u16Ret = *(uint16_t const *)&pbBuf[offBuf];990 # else991 u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);992 # endif993 }994 else995 u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);996 997 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF998 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;999 Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));1000 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1001 *(uint16_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;1002 # else1003 pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);1004 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);1005 # endif1006 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;1007 # endif1008 1009 return u16Ret;1010 1011 # else /* !IEM_WITH_CODE_TLB */1012 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1013 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))1014 {1015 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;1016 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1017 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1018 # else1019 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1020 # endif1021 }1022 return iemOpcodeGetNextU16SlowJmp(pVCpu);1023 # endif /* !IEM_WITH_CODE_TLB */1024 }1025 1026 # endif /* IEM_WITH_SETJMP */1027 1028 /**1029 * Fetches the next opcode word, returns automatically on failure.1030 *1031 * @param a_pu16 Where to return the opcode word.1032 * @remark Implicitly references pVCpu.1033 */1034 # ifndef IEM_WITH_SETJMP1035 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) \1036 do \1037 { \1038 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \1039 if (rcStrict2 != VINF_SUCCESS) \1040 return rcStrict2; \1041 } while (0)1042 # else1043 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))1044 # endif1045 1046 # ifndef IEM_WITH_SETJMP1047 /**1048 * Fetches the next opcode word, zero extending it to a double word.1049 *1050 * @returns Strict VBox status code.1051 * @param pVCpu The cross context virtual CPU structure of the calling thread.1052 * @param pu32 Where to return the opcode double word.1053 */1054 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT1055 {1056 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1057 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))1058 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);1059 1060 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1061 pVCpu->iem.s.offOpcode = offOpcode + 2;1062 return VINF_SUCCESS;1063 }1064 # endif /* !IEM_WITH_SETJMP */1065 1066 /**1067 * Fetches the next opcode word and zero extends it to a double word, returns1068 * automatically on failure.1069 *1070 * @param a_pu32 Where to return the opcode double word.1071 * @remark Implicitly references pVCpu.1072 */1073 # ifndef IEM_WITH_SETJMP1074 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \1075 do \1076 { \1077 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \1078 if (rcStrict2 != VINF_SUCCESS) \1079 return rcStrict2; \1080 } while (0)1081 # else1082 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))1083 # endif1084 1085 # ifndef IEM_WITH_SETJMP1086 /**1087 * Fetches the next opcode word, zero extending it to a quad word.1088 *1089 * @returns Strict VBox status code.1090 * @param pVCpu The cross context virtual CPU structure of the calling thread.1091 * @param pu64 Where to return the opcode quad word.1092 */1093 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1094 {1095 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1096 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))1097 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);1098 1099 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1100 pVCpu->iem.s.offOpcode = offOpcode + 2;1101 return VINF_SUCCESS;1102 }1103 # endif /* !IEM_WITH_SETJMP */1104 1105 /**1106 * Fetches the next opcode word and zero extends it to a quad word, returns1107 * automatically on failure.1108 *1109 * @param a_pu64 Where to return the opcode quad word.1110 * @remark Implicitly references pVCpu.1111 */1112 # ifndef IEM_WITH_SETJMP1113 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \1114 do \1115 { \1116 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \1117 if (rcStrict2 != VINF_SUCCESS) \1118 return rcStrict2; \1119 } while (0)1120 # else1121 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))1122 # endif1123 1124 1125 # ifndef IEM_WITH_SETJMP1126 /**1127 * Fetches the next signed word from the opcode stream.1128 *1129 * @returns Strict VBox status code.1130 * @param pVCpu The cross context virtual CPU structure of the calling thread.1131 * @param pi16 Where to return the signed word.1132 */1133 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT1134 {1135 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);1136 }1137 # endif /* !IEM_WITH_SETJMP */1138 1139 1140 /**1141 * Fetches the next signed word from the opcode stream, returning automatically1142 * on failure.1143 *1144 * @param a_pi16 Where to return the signed word.1145 * @remark Implicitly references pVCpu.1146 */1147 # ifndef IEM_WITH_SETJMP1148 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) \1149 do \1150 { \1151 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \1152 if (rcStrict2 != VINF_SUCCESS) \1153 return rcStrict2; \1154 } while (0)1155 # else1156 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))1157 # endif1158 1159 # ifndef IEM_WITH_SETJMP1160 1161 /**1162 * Fetches the next opcode dword.1163 *1164 * @returns Strict VBox status code.1165 * @param pVCpu The cross context virtual CPU structure of the calling thread.1166 * @param pu32 Where to return the opcode double word.1167 */1168 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT1169 {1170 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1171 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))1172 {1173 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;1174 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1175 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1176 # else1177 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1178 pVCpu->iem.s.abOpcode[offOpcode + 1],1179 pVCpu->iem.s.abOpcode[offOpcode + 2],1180 pVCpu->iem.s.abOpcode[offOpcode + 3]);1181 # endif1182 return VINF_SUCCESS;1183 }1184 return iemOpcodeGetNextU32Slow(pVCpu, pu32);1185 }1186 1187 # else /* IEM_WITH_SETJMP */1188 1189 /**1190 * Fetches the next opcode dword, longjmp on error.1191 *1192 * @returns The opcode dword.1193 * @param pVCpu The cross context virtual CPU structure of the calling thread.1194 */1195 DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP1196 {1197 # ifdef IEM_WITH_CODE_TLB1198 uint32_t u32Ret;1199 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;1200 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;1201 if (RT_LIKELY( pbBuf != NULL1202 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))1203 {1204 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;1205 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1206 u32Ret = *(uint32_t const *)&pbBuf[offBuf];1207 # else1208 u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],1209 pbBuf[offBuf + 1],1210 pbBuf[offBuf + 2],1211 pbBuf[offBuf + 3]);1212 # endif1213 }1214 else1215 u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);1216 1217 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF1218 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1219 Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));1220 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1221 *(uint32_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;1222 # else1223 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);1224 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);1225 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);1226 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);1227 # endif1228 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;1229 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */1230 1231 return u32Ret;1232 1233 # else /* !IEM_WITH_CODE_TLB */1234 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1235 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))1236 {1237 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;1238 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1239 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1240 # else1241 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1242 pVCpu->iem.s.abOpcode[offOpcode + 1],1243 pVCpu->iem.s.abOpcode[offOpcode + 2],1244 pVCpu->iem.s.abOpcode[offOpcode + 3]);1245 # endif1246 }1247 return iemOpcodeGetNextU32SlowJmp(pVCpu);1248 # endif1249 }1250 1251 # endif /* IEM_WITH_SETJMP */1252 1253 /**1254 * Fetches the next opcode dword, returns automatically on failure.1255 *1256 * @param a_pu32 Where to return the opcode dword.1257 * @remark Implicitly references pVCpu.1258 */1259 # ifndef IEM_WITH_SETJMP1260 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) \1261 do \1262 { \1263 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \1264 if (rcStrict2 != VINF_SUCCESS) \1265 return rcStrict2; \1266 } while (0)1267 # else1268 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))1269 # endif1270 1271 # ifndef IEM_WITH_SETJMP1272 /**1273 * Fetches the next opcode dword, zero extending it to a quad word.1274 *1275 * @returns Strict VBox status code.1276 * @param pVCpu The cross context virtual CPU structure of the calling thread.1277 * @param pu64 Where to return the opcode quad word.1278 */1279 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1280 {1281 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1282 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))1283 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);1284 1285 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1286 pVCpu->iem.s.abOpcode[offOpcode + 1],1287 pVCpu->iem.s.abOpcode[offOpcode + 2],1288 pVCpu->iem.s.abOpcode[offOpcode + 3]);1289 pVCpu->iem.s.offOpcode = offOpcode + 4;1290 return VINF_SUCCESS;1291 }1292 # endif /* !IEM_WITH_SETJMP */1293 1294 /**1295 * Fetches the next opcode dword and zero extends it to a quad word, returns1296 * automatically on failure.1297 *1298 * @param a_pu64 Where to return the opcode quad word.1299 * @remark Implicitly references pVCpu.1300 */1301 # ifndef IEM_WITH_SETJMP1302 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \1303 do \1304 { \1305 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \1306 if (rcStrict2 != VINF_SUCCESS) \1307 return rcStrict2; \1308 } while (0)1309 # else1310 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))1311 # endif1312 1313 1314 # ifndef IEM_WITH_SETJMP1315 /**1316 * Fetches the next signed double word from the opcode stream.1317 *1318 * @returns Strict VBox status code.1319 * @param pVCpu The cross context virtual CPU structure of the calling thread.1320 * @param pi32 Where to return the signed double word.1321 */1322 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT1323 {1324 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);1325 }1326 # endif1327 1328 /**1329 * Fetches the next signed double word from the opcode stream, returning1330 * automatically on failure.1331 *1332 * @param a_pi32 Where to return the signed double word.1333 * @remark Implicitly references pVCpu.1334 */1335 # ifndef IEM_WITH_SETJMP1336 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) \1337 do \1338 { \1339 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \1340 if (rcStrict2 != VINF_SUCCESS) \1341 return rcStrict2; \1342 } while (0)1343 # else1344 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))1345 # endif1346 1347 # ifndef IEM_WITH_SETJMP1348 /**1349 * Fetches the next opcode dword, sign extending it into a quad word.1350 *1351 * @returns Strict VBox status code.1352 * @param pVCpu The cross context virtual CPU structure of the calling thread.1353 * @param pu64 Where to return the opcode quad word.1354 */1355 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1356 {1357 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1358 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))1359 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);1360 1361 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1362 pVCpu->iem.s.abOpcode[offOpcode + 1],1363 pVCpu->iem.s.abOpcode[offOpcode + 2],1364 pVCpu->iem.s.abOpcode[offOpcode + 3]);1365 *pu64 = (uint64_t)(int64_t)i32;1366 pVCpu->iem.s.offOpcode = offOpcode + 4;1367 return VINF_SUCCESS;1368 }1369 # endif /* !IEM_WITH_SETJMP */1370 1371 /**1372 * Fetches the next opcode double word and sign extends it to a quad word,1373 * returns automatically on failure.1374 *1375 * @param a_pu64 Where to return the opcode quad word.1376 * @remark Implicitly references pVCpu.1377 */1378 # ifndef IEM_WITH_SETJMP1379 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \1380 do \1381 { \1382 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \1383 if (rcStrict2 != VINF_SUCCESS) \1384 return rcStrict2; \1385 } while (0)1386 # else1387 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))1388 # endif1389 1390 # ifndef IEM_WITH_SETJMP1391 1392 /**1393 * Fetches the next opcode qword.1394 *1395 * @returns Strict VBox status code.1396 * @param pVCpu The cross context virtual CPU structure of the calling thread.1397 * @param pu64 Where to return the opcode qword.1398 */1399 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1400 {1401 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1402 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))1403 {1404 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1405 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1406 # else1407 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1408 pVCpu->iem.s.abOpcode[offOpcode + 1],1409 pVCpu->iem.s.abOpcode[offOpcode + 2],1410 pVCpu->iem.s.abOpcode[offOpcode + 3],1411 pVCpu->iem.s.abOpcode[offOpcode + 4],1412 pVCpu->iem.s.abOpcode[offOpcode + 5],1413 pVCpu->iem.s.abOpcode[offOpcode + 6],1414 pVCpu->iem.s.abOpcode[offOpcode + 7]);1415 # endif1416 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;1417 return VINF_SUCCESS;1418 }1419 return iemOpcodeGetNextU64Slow(pVCpu, pu64);1420 }1421 1422 # else /* IEM_WITH_SETJMP */1423 1424 /**1425 * Fetches the next opcode qword, longjmp on error.1426 *1427 * @returns The opcode qword.1428 * @param pVCpu The cross context virtual CPU structure of the calling thread.1429 */1430 DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP1431 {1432 # ifdef IEM_WITH_CODE_TLB1433 uint64_t u64Ret;1434 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;1435 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;1436 if (RT_LIKELY( pbBuf != NULL1437 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))1438 {1439 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;1440 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1441 u64Ret = *(uint64_t const *)&pbBuf[offBuf];1442 # else1443 u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],1444 pbBuf[offBuf + 1],1445 pbBuf[offBuf + 2],1446 pbBuf[offBuf + 3],1447 pbBuf[offBuf + 4],1448 pbBuf[offBuf + 5],1449 pbBuf[offBuf + 6],1450 pbBuf[offBuf + 7]);1451 # endif1452 }1453 else1454 u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);1455 1456 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF1457 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1458 Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));1459 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1460 *(uint64_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;1461 # else1462 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);1463 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);1464 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);1465 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);1466 pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);1467 pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);1468 pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);1469 pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);1470 # endif1471 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;1472 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */1473 1474 return u64Ret;1475 1476 # else /* !IEM_WITH_CODE_TLB */1477 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1478 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))1479 {1480 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;1481 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1482 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1483 # else1484 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1485 pVCpu->iem.s.abOpcode[offOpcode + 1],1486 pVCpu->iem.s.abOpcode[offOpcode + 2],1487 pVCpu->iem.s.abOpcode[offOpcode + 3],1488 pVCpu->iem.s.abOpcode[offOpcode + 4],1489 pVCpu->iem.s.abOpcode[offOpcode + 5],1490 pVCpu->iem.s.abOpcode[offOpcode + 6],1491 pVCpu->iem.s.abOpcode[offOpcode + 7]);1492 # endif1493 }1494 return iemOpcodeGetNextU64SlowJmp(pVCpu);1495 # endif /* !IEM_WITH_CODE_TLB */1496 }1497 1498 # endif /* IEM_WITH_SETJMP */1499 1500 /**1501 * Fetches the next opcode quad word, returns automatically on failure.1502 *1503 * @param a_pu64 Where to return the opcode quad word.1504 * @remark Implicitly references pVCpu.1505 */1506 # ifndef IEM_WITH_SETJMP1507 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) \1508 do \1509 { \1510 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \1511 if (rcStrict2 != VINF_SUCCESS) \1512 return rcStrict2; \1513 } while (0)1514 # else1515 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )1516 # endif1517 1518 /**1519 * For fetching the opcode bytes for an ModR/M effective address, but throw1520 * away the result.1521 *1522 * This is used when decoding undefined opcodes and such where we want to avoid1523 * unnecessary MC blocks.1524 *1525 * @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is1526 * used instead. At least for now...1527 */1528 # ifndef IEM_WITH_SETJMP1529 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \1530 RTGCPTR GCPtrEff; \1531 VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \1532 if (rcStrict != VINF_SUCCESS) \1533 return rcStrict; \1534 } while (0)1535 # else1536 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \1537 (void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \1538 } while (0)1539 # endif1540 1541 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */1542 345 1543 346 … … 1594 397 1595 398 /** @} */ 1596 1597 1598 /*1599 *1600 * Helpers routines.1601 * Helpers routines.1602 * Helpers routines.1603 *1604 */1605 1606 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1607 1608 /**1609 * Recalculates the effective operand size.1610 *1611 * @param pVCpu The cross context virtual CPU structure of the calling thread.1612 */1613 DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT1614 {1615 switch (IEM_GET_CPU_MODE(pVCpu))1616 {1617 case IEMMODE_16BIT:1618 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;1619 break;1620 case IEMMODE_32BIT:1621 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;1622 break;1623 case IEMMODE_64BIT:1624 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))1625 {1626 case 0:1627 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;1628 break;1629 case IEM_OP_PRF_SIZE_OP:1630 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1631 break;1632 case IEM_OP_PRF_SIZE_REX_W:1633 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:1634 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1635 break;1636 }1637 break;1638 default:1639 AssertFailed();1640 }1641 }1642 1643 1644 /**1645 * Sets the default operand size to 64-bit and recalculates the effective1646 * operand size.1647 *1648 * @param pVCpu The cross context virtual CPU structure of the calling thread.1649 */1650 DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT1651 {1652 Assert(IEM_IS_64BIT_CODE(pVCpu));1653 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1654 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)1655 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1656 else1657 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1658 }1659 1660 1661 /**1662 * Sets the default operand size to 64-bit and recalculates the effective1663 * operand size, with intel ignoring any operand size prefix (AMD respects it).1664 *1665 * This is for the relative jumps.1666 *1667 * @param pVCpu The cross context virtual CPU structure of the calling thread.1668 */1669 DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT1670 {1671 Assert(IEM_IS_64BIT_CODE(pVCpu));1672 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1673 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP1674 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)1675 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1676 else1677 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1678 }1679 1680 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */1681 1682 399 1683 400 … … 4562 3279 4563 3280 4564 /** @name Memory access.4565 *4566 * @{4567 */4568 4569 4570 /**4571 * Checks whether alignment checks are enabled or not.4572 *4573 * @returns true if enabled, false if not.4574 * @param pVCpu The cross context virtual CPU structure of the calling thread.4575 */4576 DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT4577 {4578 #if 04579 AssertCompile(X86_CR0_AM == X86_EFL_AC);4580 return IEM_GET_CPL(pVCpu) == 34581 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);4582 #else4583 return RT_BOOL(pVCpu->iem.s.fExec & IEM_F_X86_AC);4584 #endif4585 }4586 4587 /**4588 * Checks if the given segment can be written to, raise the appropriate4589 * exception if not.4590 *4591 * @returns VBox strict status code.4592 *4593 * @param pVCpu The cross context virtual CPU structure of the calling thread.4594 * @param pHid Pointer to the hidden register.4595 * @param iSegReg The register number.4596 * @param pu64BaseAddr Where to return the base address to use for the4597 * segment. (In 64-bit code it may differ from the4598 * base in the hidden segment.)4599 */4600 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,4601 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT4602 {4603 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4604 4605 if (IEM_IS_64BIT_CODE(pVCpu))4606 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;4607 else4608 {4609 if (!pHid->Attr.n.u1Present)4610 {4611 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);4612 AssertRelease(uSel == 0);4613 LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));4614 return iemRaiseGeneralProtectionFault0(pVCpu);4615 }4616 4617 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)4618 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )4619 && !IEM_IS_64BIT_CODE(pVCpu) )4620 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4621 *pu64BaseAddr = pHid->u64Base;4622 }4623 return VINF_SUCCESS;4624 }4625 4626 4627 /**4628 * Checks if the given segment can be read from, raise the appropriate4629 * exception if not.4630 *4631 * @returns VBox strict status code.4632 *4633 * @param pVCpu The cross context virtual CPU structure of the calling thread.4634 * @param pHid Pointer to the hidden register.4635 * @param iSegReg The register number.4636 * @param pu64BaseAddr Where to return the base address to use for the4637 * segment. (In 64-bit code it may differ from the4638 * base in the hidden segment.)4639 */4640 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,4641 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT4642 {4643 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4644 4645 if (IEM_IS_64BIT_CODE(pVCpu))4646 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;4647 else4648 {4649 if (!pHid->Attr.n.u1Present)4650 {4651 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);4652 AssertRelease(uSel == 0);4653 LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));4654 return iemRaiseGeneralProtectionFault0(pVCpu);4655 }4656 4657 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)4658 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4659 *pu64BaseAddr = pHid->u64Base;4660 }4661 return VINF_SUCCESS;4662 }4663 4664 4665 /**4666 * Maps a physical page.4667 *4668 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).4669 * @param pVCpu The cross context virtual CPU structure of the calling thread.4670 * @param GCPhysMem The physical address.4671 * @param fAccess The intended access.4672 * @param ppvMem Where to return the mapping address.4673 * @param pLock The PGM lock.4674 */4675 DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,4676 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT4677 {4678 #ifdef IEM_LOG_MEMORY_WRITES4679 if (fAccess & IEM_ACCESS_TYPE_WRITE)4680 return VERR_PGM_PHYS_TLB_CATCH_ALL;4681 #endif4682 4683 /** @todo This API may require some improving later. A private deal with PGM4684 * regarding locking and unlocking needs to be struct. A couple of TLBs4685 * living in PGM, but with publicly accessible inlined access methods4686 * could perhaps be an even better solution. */4687 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,4688 GCPhysMem,4689 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),4690 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),4691 ppvMem,4692 pLock);4693 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/4694 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));4695 4696 return rc;4697 }4698 4699 4700 /**4701 * Unmap a page previously mapped by iemMemPageMap.4702 *4703 * @param pVCpu The cross context virtual CPU structure of the calling thread.4704 * @param GCPhysMem The physical address.4705 * @param fAccess The intended access.4706 * @param pvMem What iemMemPageMap returned.4707 * @param pLock The PGM lock.4708 */4709 DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,4710 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT4711 {4712 NOREF(pVCpu);4713 NOREF(GCPhysMem);4714 NOREF(fAccess);4715 NOREF(pvMem);4716 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);4717 }4718 4719 #ifdef IEM_WITH_SETJMP4720 4721 /** @todo slim this down */4722 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,4723 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP4724 {4725 Assert(cbMem >= 1);4726 Assert(iSegReg < X86_SREG_COUNT);4727 4728 /*4729 * 64-bit mode is simpler.4730 */4731 if (IEM_IS_64BIT_CODE(pVCpu))4732 {4733 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)4734 {4735 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4736 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);4737 GCPtrMem += pSel->u64Base;4738 }4739 4740 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))4741 return GCPtrMem;4742 iemRaiseGeneralProtectionFault0Jmp(pVCpu);4743 }4744 /*4745 * 16-bit and 32-bit segmentation.4746 */4747 else if (iSegReg != UINT8_MAX)4748 {4749 /** @todo Does this apply to segments with 4G-1 limit? */4750 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4751 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))4752 {4753 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4754 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);4755 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE4756 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */4757 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */4758 | X86_SEL_TYPE_CODE))4759 {4760 case X86DESCATTR_P: /* readonly data, expand up */4761 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */4762 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */4763 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */4764 /* expand up */4765 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))4766 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4767 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",4768 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));4769 break;4770 4771 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */4772 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */4773 /* expand down */4774 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit4775 && ( pSel->Attr.n.u1DefBig4776 || GCPtrLast32 <= UINT32_C(0xffff)) ))4777 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4778 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",4779 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));4780 break;4781 4782 default:4783 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));4784 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4785 break;4786 }4787 }4788 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));4789 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4790 }4791 /*4792 * 32-bit flat address.4793 */4794 else4795 return GCPtrMem;4796 }4797 4798 4799 /** @todo slim this down */4800 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,4801 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP4802 {4803 Assert(cbMem >= 1);4804 Assert(iSegReg < X86_SREG_COUNT);4805 4806 /*4807 * 64-bit mode is simpler.4808 */4809 if (IEM_IS_64BIT_CODE(pVCpu))4810 {4811 if (iSegReg >= X86_SREG_FS)4812 {4813 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4814 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);4815 GCPtrMem += pSel->u64Base;4816 }4817 4818 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))4819 return GCPtrMem;4820 }4821 /*4822 * 16-bit and 32-bit segmentation.4823 */4824 else4825 {4826 Assert(GCPtrMem <= UINT32_MAX);4827 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4828 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);4829 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE4830 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);4831 if ( fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE) /* data, expand up */4832 /** @todo explore exactly how the CS stuff works in real mode. See also4833 * http://www.rcollins.org/Productivity/DescriptorCache.html and4834 * http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */4835 || (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. */ /** @todo testcase! */4836 {4837 /* expand up */4838 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4839 if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit4840 && GCPtrLast32 >= (uint32_t)GCPtrMem))4841 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4842 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4843 }4844 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */4845 {4846 /* expand down - the uppger boundary is defined by the B bit, not G. */4847 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4848 if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit4849 && (pSel->Attr.n.u1DefBig || GCPtrLast32 <= UINT32_C(0xffff))4850 && GCPtrLast32 >= (uint32_t)GCPtrMem))4851 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4852 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4853 }4854 else4855 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4856 }4857 iemRaiseGeneralProtectionFault0Jmp(pVCpu);4858 }4859 4860 #endif /* IEM_WITH_SETJMP */4861 4862 /**4863 * Fakes a long mode stack selector for SS = 0.4864 *4865 * @param pDescSs Where to return the fake stack descriptor.4866 * @param uDpl The DPL we want.4867 */4868 DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT4869 {4870 pDescSs->Long.au64[0] = 0;4871 pDescSs->Long.au64[1] = 0;4872 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;4873 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */4874 pDescSs->Long.Gen.u2Dpl = uDpl;4875 pDescSs->Long.Gen.u1Present = 1;4876 pDescSs->Long.Gen.u1Long = 1;4877 }4878 4879 4880 /*4881 * Unmap helpers.4882 */4883 4884 #ifdef IEM_WITH_SETJMP4885 4886 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4887 {4888 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4889 if (RT_LIKELY(bMapInfo == 0))4890 return;4891 # endif4892 iemMemCommitAndUnmapRwSafeJmp(pVCpu, bMapInfo);4893 }4894 4895 4896 DECL_INLINE_THROW(void) iemMemCommitAndUnmapAtJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4897 {4898 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4899 if (RT_LIKELY(bMapInfo == 0))4900 return;4901 # endif4902 iemMemCommitAndUnmapAtSafeJmp(pVCpu, bMapInfo);4903 }4904 4905 4906 DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4907 {4908 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4909 if (RT_LIKELY(bMapInfo == 0))4910 return;4911 # endif4912 iemMemCommitAndUnmapWoSafeJmp(pVCpu, bMapInfo);4913 }4914 4915 4916 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4917 {4918 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4919 if (RT_LIKELY(bMapInfo == 0))4920 return;4921 # endif4922 iemMemCommitAndUnmapRoSafeJmp(pVCpu, bMapInfo);4923 }4924 4925 DECLINLINE(void) iemMemRollbackAndUnmapWo(PVMCPUCC pVCpu, uint8_t bMapInfo) RT_NOEXCEPT4926 {4927 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4928 if (RT_LIKELY(bMapInfo == 0))4929 return;4930 # endif4931 iemMemRollbackAndUnmapWoSafe(pVCpu, bMapInfo);4932 }4933 4934 #endif /* IEM_WITH_SETJMP */4935 4936 4937 /*4938 * Instantiate R/W inline templates.4939 */4940 4941 /** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK4942 * Used to check if an unaligned access is if within the page and won't4943 * trigger an \#AC.4944 *4945 * This can also be used to deal with misaligned accesses on platforms that are4946 * senstive to such if desires.4947 */4948 #if 14949 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \4950 ( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \4951 && !((a_pVCpu)->iem.s.fExec & IEM_F_X86_AC) )4952 #else4953 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 04954 #endif4955 4956 #define TMPL_MEM_WITH_ATOMIC_MAPPING4957 4958 #define TMPL_MEM_TYPE uint8_t4959 #define TMPL_MEM_TYPE_ALIGN 04960 #define TMPL_MEM_TYPE_SIZE 14961 #define TMPL_MEM_FN_SUFF U84962 #define TMPL_MEM_FMT_TYPE "%#04x"4963 #define TMPL_MEM_FMT_DESC "byte"4964 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4965 4966 #define TMPL_MEM_WITH_STACK4967 4968 #define TMPL_MEM_TYPE uint16_t4969 #define TMPL_MEM_TYPE_ALIGN 14970 #define TMPL_MEM_TYPE_SIZE 24971 #define TMPL_MEM_FN_SUFF U164972 #define TMPL_MEM_FMT_TYPE "%#06x"4973 #define TMPL_MEM_FMT_DESC "word"4974 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4975 4976 #define TMPL_WITH_PUSH_SREG4977 #define TMPL_MEM_TYPE uint32_t4978 #define TMPL_MEM_TYPE_ALIGN 34979 #define TMPL_MEM_TYPE_SIZE 44980 #define TMPL_MEM_FN_SUFF U324981 #define TMPL_MEM_FMT_TYPE "%#010x"4982 #define TMPL_MEM_FMT_DESC "dword"4983 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4984 #undef TMPL_WITH_PUSH_SREG4985 4986 #define TMPL_MEM_TYPE uint64_t4987 #define TMPL_MEM_TYPE_ALIGN 74988 #define TMPL_MEM_TYPE_SIZE 84989 #define TMPL_MEM_FN_SUFF U644990 #define TMPL_MEM_FMT_TYPE "%#018RX64"4991 #define TMPL_MEM_FMT_DESC "qword"4992 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4993 4994 #undef TMPL_MEM_WITH_STACK4995 #undef TMPL_MEM_WITH_ATOMIC_MAPPING4996 4997 #define TMPL_MEM_NO_MAPPING /* currently sticky */4998 4999 #define TMPL_MEM_NO_STORE5000 #define TMPL_MEM_TYPE uint32_t5001 #define TMPL_MEM_TYPE_ALIGN 05002 #define TMPL_MEM_TYPE_SIZE 45003 #define TMPL_MEM_FN_SUFF U32NoAc5004 #define TMPL_MEM_FMT_TYPE "%#010x"5005 #define TMPL_MEM_FMT_DESC "dword"5006 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5007 5008 #define TMPL_MEM_NO_STORE5009 #define TMPL_MEM_TYPE uint64_t5010 #define TMPL_MEM_TYPE_ALIGN 05011 #define TMPL_MEM_TYPE_SIZE 85012 #define TMPL_MEM_FN_SUFF U64NoAc5013 #define TMPL_MEM_FMT_TYPE "%#018RX64"5014 #define TMPL_MEM_FMT_DESC "qword"5015 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5016 5017 #define TMPL_MEM_NO_STORE5018 #define TMPL_MEM_TYPE uint64_t5019 #define TMPL_MEM_TYPE_ALIGN 155020 #define TMPL_MEM_TYPE_SIZE 85021 #define TMPL_MEM_FN_SUFF U64AlignedU1285022 #define TMPL_MEM_FMT_TYPE "%#018RX64"5023 #define TMPL_MEM_FMT_DESC "qword"5024 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5025 5026 #undef TMPL_MEM_NO_MAPPING5027 5028 #define TMPL_MEM_TYPE RTFLOAT80U5029 #define TMPL_MEM_TYPE_ALIGN 75030 #define TMPL_MEM_TYPE_SIZE 105031 #define TMPL_MEM_FN_SUFF R805032 #define TMPL_MEM_FMT_TYPE "%.10Rhxs"5033 #define TMPL_MEM_FMT_DESC "tword"5034 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5035 5036 #define TMPL_MEM_TYPE RTPBCD80U5037 #define TMPL_MEM_TYPE_ALIGN 7 /** @todo RTPBCD80U alignment testcase */5038 #define TMPL_MEM_TYPE_SIZE 105039 #define TMPL_MEM_FN_SUFF D805040 #define TMPL_MEM_FMT_TYPE "%.10Rhxs"5041 #define TMPL_MEM_FMT_DESC "tword"5042 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5043 5044 #define TMPL_MEM_WITH_ATOMIC_MAPPING5045 #define TMPL_MEM_TYPE RTUINT128U5046 #define TMPL_MEM_TYPE_ALIGN 155047 #define TMPL_MEM_TYPE_SIZE 165048 #define TMPL_MEM_FN_SUFF U1285049 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5050 #define TMPL_MEM_FMT_DESC "dqword"5051 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5052 #undef TMPL_MEM_WITH_ATOMIC_MAPPING5053 5054 #define TMPL_MEM_NO_MAPPING5055 #define TMPL_MEM_TYPE RTUINT128U5056 #define TMPL_MEM_TYPE_ALIGN 05057 #define TMPL_MEM_TYPE_SIZE 165058 #define TMPL_MEM_FN_SUFF U128NoAc5059 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5060 #define TMPL_MEM_FMT_DESC "dqword"5061 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5062 #undef TMPL_MEM_NO_MAPPING5063 5064 5065 /* Every template relying on unaligned accesses inside a page not being okay should go below. */5066 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK5067 #define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 05068 5069 #define TMPL_MEM_NO_MAPPING5070 #define TMPL_MEM_TYPE RTUINT128U5071 #define TMPL_MEM_TYPE_ALIGN 155072 #define TMPL_MEM_TYPE_SIZE 165073 #define TMPL_MEM_FN_SUFF U128AlignedSse5074 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5075 #define TMPL_MEM_FMT_DESC "dqword"5076 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5077 #undef TMPL_MEM_NO_MAPPING5078 5079 #define TMPL_MEM_NO_MAPPING5080 #define TMPL_MEM_TYPE RTUINT256U5081 #define TMPL_MEM_TYPE_ALIGN 05082 #define TMPL_MEM_TYPE_SIZE 325083 #define TMPL_MEM_FN_SUFF U256NoAc5084 #define TMPL_MEM_FMT_TYPE "%.32Rhxs"5085 #define TMPL_MEM_FMT_DESC "qqword"5086 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5087 #undef TMPL_MEM_NO_MAPPING5088 5089 #define TMPL_MEM_NO_MAPPING5090 #define TMPL_MEM_TYPE RTUINT256U5091 #define TMPL_MEM_TYPE_ALIGN 315092 #define TMPL_MEM_TYPE_SIZE 325093 #define TMPL_MEM_FN_SUFF U256AlignedAvx5094 #define TMPL_MEM_FMT_TYPE "%.32Rhxs"5095 #define TMPL_MEM_FMT_DESC "qqword"5096 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5097 #undef TMPL_MEM_NO_MAPPING5098 5099 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK5100 5101 /** @} */5102 5103 5104 3281 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX 5105 3282 … … 5159 3336 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ 5160 3337 5161 #if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) 5162 /** 5163 * Adds an entry to the TLB trace buffer. 5164 * 5165 * @note Don't use directly, only via the IEMTLBTRACE_XXX macros. 5166 */ 5167 DECLINLINE(void) iemTlbTrace(PVMCPU pVCpu, IEMTLBTRACETYPE enmType, uint64_t u64Param, uint64_t u64Param2 = 0, 5168 uint8_t bParam = 0, uint32_t u32Param = 0/*, uint16_t u16Param = 0 */) 5169 { 5170 uint32_t const fMask = RT_BIT_32(pVCpu->iem.s.cTlbTraceEntriesShift) - 1; 5171 PIEMTLBTRACEENTRY const pEntry = &pVCpu->iem.s.paTlbTraceEntries[pVCpu->iem.s.idxTlbTraceEntry++ & fMask]; 5172 pEntry->u64Param = u64Param; 5173 pEntry->u64Param2 = u64Param2; 5174 pEntry->u16Param = 0; //u16Param; 5175 pEntry->u32Param = u32Param; 5176 pEntry->bParam = bParam; 5177 pEntry->enmType = enmType; 5178 pEntry->rip = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base; 5179 } 5180 #endif 5181 5182 #endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */ 3338 #endif /* !VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInline_x86_h */ -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMInlineDecode-x86.h
r108246 r108260 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - Inlined Functions.3 * IEM - Interpreted Execution Manager - Inlined Decoding related Functions, x86 target. 4 4 */ 5 5 … … 26 26 */ 27 27 28 #ifndef VMM_INCLUDED_SRC_ include_IEMInline_h29 #define VMM_INCLUDED_SRC_ include_IEMInline_h28 #ifndef VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineDecode_x86_h 29 #define VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineDecode_x86_h 30 30 #ifndef RT_WITHOUT_PRAGMA_ONCE 31 31 # pragma once … … 35 35 36 36 37 /**38 * Makes status code addjustments (pass up from I/O and access handler)39 * as well as maintaining statistics.40 *41 * @returns Strict VBox status code to pass up.42 * @param pVCpu The cross context virtual CPU structure of the calling thread.43 * @param rcStrict The status from executing an instruction.44 */45 DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT46 {47 if (rcStrict != VINF_SUCCESS)48 {49 /* Deal with the cases that should be treated as VINF_SUCCESS first. */50 if ( rcStrict == VINF_IEM_YIELD_PENDING_FF51 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */52 || rcStrict == VINF_VMX_VMEXIT53 #endif54 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM55 || rcStrict == VINF_SVM_VMEXIT56 #endif57 )58 {59 rcStrict = pVCpu->iem.s.rcPassUp;60 if (RT_LIKELY(rcStrict == VINF_SUCCESS))61 { /* likely */ }62 else63 pVCpu->iem.s.cRetPassUpStatus++;64 }65 else if (RT_SUCCESS(rcStrict))66 {67 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)68 || rcStrict == VINF_IOM_R3_IOPORT_READ69 || rcStrict == VINF_IOM_R3_IOPORT_WRITE70 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE71 || rcStrict == VINF_IOM_R3_MMIO_READ72 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE73 || rcStrict == VINF_IOM_R3_MMIO_WRITE74 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE75 || rcStrict == VINF_CPUM_R3_MSR_READ76 || rcStrict == VINF_CPUM_R3_MSR_WRITE77 || rcStrict == VINF_EM_RAW_EMULATE_INSTR78 || rcStrict == VINF_EM_RAW_TO_R379 || rcStrict == VINF_EM_TRIPLE_FAULT80 || rcStrict == VINF_EM_EMULATE_SPLIT_LOCK81 || rcStrict == VINF_GIM_R3_HYPERCALL82 /* raw-mode / virt handlers only: */83 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT84 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT85 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT86 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT87 || rcStrict == VINF_SELM_SYNC_GDT88 || rcStrict == VINF_CSAM_PENDING_ACTION89 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE90 /* nested hw.virt codes: */91 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE92 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR93 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));94 /** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */95 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;96 if (rcPassUp == VINF_SUCCESS)97 pVCpu->iem.s.cRetInfStatuses++;98 else if ( rcPassUp < VINF_EM_FIRST99 || rcPassUp > VINF_EM_LAST100 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))101 {102 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));103 pVCpu->iem.s.cRetPassUpStatus++;104 rcStrict = rcPassUp;105 }106 else107 {108 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));109 pVCpu->iem.s.cRetInfStatuses++;110 }111 }112 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)113 pVCpu->iem.s.cRetAspectNotImplemented++;114 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)115 pVCpu->iem.s.cRetInstrNotImplemented++;116 else117 pVCpu->iem.s.cRetErrStatuses++;118 }119 else120 {121 rcStrict = pVCpu->iem.s.rcPassUp;122 if (rcStrict != VINF_SUCCESS)123 pVCpu->iem.s.cRetPassUpStatus++;124 }125 126 /* Just clear it here as well. */127 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;128 129 return rcStrict;130 }131 132 133 /**134 * Sets the pass up status.135 *136 * @returns VINF_SUCCESS.137 * @param pVCpu The cross context virtual CPU structure of the138 * calling thread.139 * @param rcPassUp The pass up status. Must be informational.140 * VINF_SUCCESS is not allowed.141 */142 DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT143 {144 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);145 146 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;147 if (rcOldPassUp == VINF_SUCCESS)148 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);149 /* If both are EM scheduling codes, use EM priority rules. */150 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST151 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)152 {153 if (rcPassUp < rcOldPassUp)154 {155 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));156 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);157 }158 else159 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));160 }161 /* Override EM scheduling with specific status code. */162 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)163 {164 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));165 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);166 }167 /* Don't override specific status code, first come first served. */168 else169 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));170 return VINF_SUCCESS;171 }172 173 174 /**175 * Calculates the IEM_F_X86_AC flags.176 *177 * @returns IEM_F_X86_AC or zero178 * @param pVCpu The cross context virtual CPU structure of the179 * calling thread.180 */181 DECL_FORCE_INLINE(uint32_t) iemCalcExecAcFlag(PVMCPUCC pVCpu) RT_NOEXCEPT182 {183 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS);184 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));185 186 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC187 || (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_AM | X86_CR0_PE)) != (X86_CR0_AM | X86_CR0_PE)188 || ( !pVCpu->cpum.GstCtx.eflags.Bits.u1VM189 && pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl != 3))190 return 0;191 return IEM_F_X86_AC;192 }193 194 195 /**196 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.197 *198 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will199 * reject expand down data segments and conforming code segments.200 *201 * ASSUMES that the CPU is in 32-bit mode.202 *203 * @note Will return zero when if any of the segment register state is marked204 * external, this must be factored into assertions checking fExec205 * consistency.206 *207 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.208 * @param pVCpu The cross context virtual CPU structure of the209 * calling thread.210 * @sa iemCalc32BitFlatIndicatorEsDs211 */212 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT213 {214 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);215 return ( ( pVCpu->cpum.GstCtx.es.Attr.u216 | pVCpu->cpum.GstCtx.cs.Attr.u217 | pVCpu->cpum.GstCtx.ss.Attr.u218 | pVCpu->cpum.GstCtx.ds.Attr.u)219 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))220 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)221 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)222 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1)223 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1)224 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))225 == 0226 && ( pVCpu->cpum.GstCtx.es.u64Base227 | pVCpu->cpum.GstCtx.cs.u64Base228 | pVCpu->cpum.GstCtx.ss.u64Base229 | pVCpu->cpum.GstCtx.ds.u64Base)230 == 0231 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))232 ? IEM_F_MODE_X86_32BIT_FLAT : 0;233 }234 235 236 /**237 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are238 * flat already.239 *240 * This is used by sysenter.241 *242 * @note Will return zero when if any of the segment register state is marked243 * external, this must be factored into assertions checking fExec244 * consistency.245 *246 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.247 * @param pVCpu The cross context virtual CPU structure of the248 * calling thread.249 * @sa iemCalc32BitFlatIndicator250 */251 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT252 {253 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);254 return ( ( pVCpu->cpum.GstCtx.es.Attr.u255 | pVCpu->cpum.GstCtx.ds.Attr.u)256 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))257 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)258 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)259 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))260 == 0261 && ( pVCpu->cpum.GstCtx.es.u64Base262 | pVCpu->cpum.GstCtx.ds.u64Base)263 == 0264 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))265 ? IEM_F_MODE_X86_32BIT_FLAT : 0;266 }267 268 269 /**270 * Calculates the IEM_F_MODE_XXX, CPL and AC flags.271 *272 * @returns IEM_F_MODE_XXX, IEM_F_X86_CPL_MASK and IEM_F_X86_AC.273 * @param pVCpu The cross context virtual CPU structure of the274 * calling thread.275 */276 DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT277 {278 /*279 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx280 * here to try get this done as efficiently as possible.281 */282 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS);283 284 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)285 {286 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)287 {288 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));289 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);290 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC291 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)292 || fExec != (3U << IEM_F_X86_CPL_SHIFT))293 { /* likely */ }294 else295 fExec |= IEM_F_X86_AC;296 297 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)298 {299 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));300 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu);301 }302 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long303 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))304 fExec |= IEM_F_MODE_X86_64BIT;305 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)306 fExec |= IEM_F_MODE_X86_16BIT_PROT;307 else308 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386;309 return fExec;310 }311 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC312 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM))313 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);314 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT) | IEM_F_X86_AC;315 }316 317 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */318 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))319 {320 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)321 return IEM_F_MODE_X86_16BIT;322 return IEM_F_MODE_X86_16BIT_PRE_386;323 }324 325 /* 32-bit unreal mode. */326 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu);327 }328 329 330 /**331 * Calculates the AMD-V and VT-x related context flags.332 *333 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and334 * IEM_F_X86_CTX_VMX.335 * @param pVCpu The cross context virtual CPU structure of the336 * calling thread.337 */338 DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT339 {340 /*341 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled342 * and CPUMIsGuestInNestedHwvirtMode to some extent.343 */344 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);345 346 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);347 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)348 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);349 if (RT_LIKELY(!fTmp))350 return 0; /* likely */351 352 if (fTmp & X86_CR4_VMXE)353 {354 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);355 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)356 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST;357 return IEM_F_X86_CTX_VMX;358 }359 360 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);361 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)362 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST;363 return IEM_F_X86_CTX_SVM;364 }365 366 #ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */367 368 /**369 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.370 *371 * @returns IEM_F_BRK_PENDING_XXX or zero.372 * @param pVCpu The cross context virtual CPU structure of the373 * calling thread.374 */375 DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT376 {377 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);378 379 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)380 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))381 return 0;382 return iemCalcExecDbgFlagsSlow(pVCpu);383 }384 385 /**386 * Calculates the the IEM_F_XXX flags.387 *388 * @returns IEM_F_XXX combination match the current CPU state.389 * @param pVCpu The cross context virtual CPU structure of the390 * calling thread.391 */392 DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT393 {394 return iemCalcExecModeAndCplFlags(pVCpu)395 | iemCalcExecHwVirtFlags(pVCpu)396 /* SMM is not yet implemented */397 | iemCalcExecDbgFlags(pVCpu)398 ;399 }400 401 402 /**403 * Re-calculates the MODE and CPL parts of IEMCPU::fExec.404 *405 * @param pVCpu The cross context virtual CPU structure of the406 * calling thread.407 */408 DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplAndAcFlags(PVMCPUCC pVCpu)409 {410 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK | IEM_F_X86_AC))411 | iemCalcExecModeAndCplFlags(pVCpu);412 }413 414 415 /**416 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.417 *418 * @param pVCpu The cross context virtual CPU structure of the419 * calling thread.420 */421 DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)422 {423 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)424 | iemCalcExecDbgFlags(pVCpu);425 }426 427 #endif /* VBOX_INCLUDED_vmm_dbgf_h */428 429 430 37 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 431 432 # if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */433 434 /**435 * Initializes the execution state.436 *437 * @param pVCpu The cross context virtual CPU structure of the438 * calling thread.439 * @param fExecOpts Optional execution flags:440 * - IEM_F_BYPASS_HANDLERS441 * - IEM_F_X86_DISREGARD_LOCK442 *443 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal444 * side-effects in strict builds.445 */446 DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT447 {448 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);449 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));450 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));451 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));452 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));453 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));454 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));455 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));456 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));457 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));458 459 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;460 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts;461 pVCpu->iem.s.cActiveMappings = 0;462 pVCpu->iem.s.iNextMapping = 0;463 464 # ifdef VBOX_STRICT465 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;466 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;467 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;468 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;469 pVCpu->iem.s.fPrefixes = 0xfeedbeef;470 pVCpu->iem.s.uRexReg = 127;471 pVCpu->iem.s.uRexB = 127;472 pVCpu->iem.s.offModRm = 127;473 pVCpu->iem.s.uRexIndex = 127;474 pVCpu->iem.s.iEffSeg = 127;475 pVCpu->iem.s.idxPrefix = 127;476 pVCpu->iem.s.uVex3rdReg = 127;477 pVCpu->iem.s.uVexLength = 127;478 pVCpu->iem.s.fEvexStuff = 127;479 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;480 # ifdef IEM_WITH_CODE_TLB481 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;482 pVCpu->iem.s.pbInstrBuf = NULL;483 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;484 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;485 pVCpu->iem.s.offCurInstrStart = INT16_MAX;486 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);487 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF488 pVCpu->iem.s.offOpcode = 127;489 # endif490 # else491 pVCpu->iem.s.offOpcode = 127;492 pVCpu->iem.s.cbOpcode = 127;493 # endif494 # endif /* VBOX_STRICT */495 }496 497 498 # if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)499 /**500 * Performs a minimal reinitialization of the execution state.501 *502 * This is intended to be used by VM-exits, SMM, LOADALL and other similar503 * 'world-switch' types operations on the CPU. Currently only nested504 * hardware-virtualization uses it.505 *506 * @param pVCpu The cross context virtual CPU structure of the calling EMT.507 * @param cbInstr The instruction length (for flushing).508 */509 DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT510 {511 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);512 iemOpcodeFlushHeavy(pVCpu, cbInstr);513 }514 # endif515 516 # endif /* VBOX_INCLUDED_vmm_dbgf_h || DOXYGEN_RUNNING */517 518 /**519 * Counterpart to #iemInitExec that undoes evil strict-build stuff.520 *521 * @param pVCpu The cross context virtual CPU structure of the522 * calling thread.523 */524 DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT525 {526 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */527 # ifdef VBOX_STRICT528 # ifdef IEM_WITH_CODE_TLB529 NOREF(pVCpu);530 # else531 pVCpu->iem.s.cbOpcode = 0;532 # endif533 # else534 NOREF(pVCpu);535 # endif536 }537 538 539 /**540 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.541 *542 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.543 *544 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.545 * @param pVCpu The cross context virtual CPU structure of the calling thread.546 * @param rcStrict The status code to fiddle.547 */548 DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT549 {550 iemUninitExec(pVCpu);551 return iemExecStatusCodeFiddling(pVCpu, rcStrict);552 }553 554 555 /**556 * Macro used by the IEMExec* method to check the given instruction length.557 *558 * Will return on failure!559 *560 * @param a_cbInstr The given instruction length.561 * @param a_cbMin The minimum length.562 */563 # define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \564 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \565 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)566 567 38 568 39 # ifndef IEM_WITH_SETJMP … … 1541 1012 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */ 1542 1013 1543 1544 /** @name Misc Worker Functions.1545 * @{1546 */1547 1548 /**1549 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or1550 * not (kind of obsolete now).1551 *1552 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1553 */1554 #define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )1555 1556 /**1557 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).1558 *1559 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1560 * @param a_fEfl The new EFLAGS.1561 */1562 #define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)1563 1564 1565 /**1566 * Loads a NULL data selector into a selector register, both the hidden and1567 * visible parts, in protected mode.1568 *1569 * @param pVCpu The cross context virtual CPU structure of the calling thread.1570 * @param pSReg Pointer to the segment register.1571 * @param uRpl The RPL.1572 */1573 DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT1574 {1575 /** @todo Testcase: write a testcase checking what happends when loading a NULL1576 * data selector in protected mode. */1577 pSReg->Sel = uRpl;1578 pSReg->ValidSel = uRpl;1579 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;1580 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))1581 {1582 /* VT-x (Intel 3960x) observed doing something like this. */1583 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);1584 pSReg->u32Limit = UINT32_MAX;1585 pSReg->u64Base = 0;1586 }1587 else1588 {1589 pSReg->Attr.u = X86DESCATTR_UNUSABLE;1590 pSReg->u32Limit = 0;1591 pSReg->u64Base = 0;1592 }1593 }1594 1595 /** @} */1596 1597 1598 /*1599 *1600 * Helpers routines.1601 * Helpers routines.1602 * Helpers routines.1603 *1604 */1605 1014 1606 1015 #ifndef IEM_WITH_OPAQUE_DECODER_STATE … … 1681 1090 1682 1091 1683 1684 /** @name Register Access. 1685 * @{ 1686 */ 1687 1688 /** 1689 * Gets a reference (pointer) to the specified hidden segment register. 1690 * 1691 * @returns Hidden register reference. 1692 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1693 * @param iSegReg The segment register. 1694 */ 1695 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT 1696 { 1697 Assert(iSegReg < X86_SREG_COUNT); 1698 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 1699 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; 1700 1701 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); 1702 return pSReg; 1703 } 1704 1705 1706 /** 1707 * Ensures that the given hidden segment register is up to date. 1708 * 1709 * @returns Hidden register reference. 1710 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1711 * @param pSReg The segment register. 1712 */ 1713 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT 1714 { 1715 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); 1716 NOREF(pVCpu); 1717 return pSReg; 1718 } 1719 1720 1721 /** 1722 * Gets a reference (pointer) to the specified segment register (the selector 1723 * value). 1724 * 1725 * @returns Pointer to the selector variable. 1726 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1727 * @param iSegReg The segment register. 1728 */ 1729 DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT 1730 { 1731 Assert(iSegReg < X86_SREG_COUNT); 1732 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 1733 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel; 1734 } 1735 1736 1737 /** 1738 * Fetches the selector value of a segment register. 1739 * 1740 * @returns The selector value. 1741 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1742 * @param iSegReg The segment register. 1743 */ 1744 DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT 1745 { 1746 Assert(iSegReg < X86_SREG_COUNT); 1747 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 1748 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel; 1749 } 1750 1751 1752 /** 1753 * Fetches the base address value of a segment register. 1754 * 1755 * @returns The selector value. 1756 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1757 * @param iSegReg The segment register. 1758 */ 1759 DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT 1760 { 1761 Assert(iSegReg < X86_SREG_COUNT); 1762 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 1763 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base; 1764 } 1765 1766 1767 /** 1768 * Gets a reference (pointer) to the specified general purpose register. 1769 * 1770 * @returns Register reference. 1771 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1772 * @param iReg The general purpose register. 1773 */ 1774 DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1775 { 1776 Assert(iReg < 16); 1777 return &pVCpu->cpum.GstCtx.aGRegs[iReg]; 1778 } 1779 1780 1781 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 1782 /** 1783 * Gets a reference (pointer) to the specified 8-bit general purpose register. 1784 * 1785 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here. 1786 * 1787 * @returns Register reference. 1788 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1789 * @param iReg The register. 1790 */ 1791 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1792 { 1793 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REX | IEM_OP_PRF_VEX))) 1794 { 1795 Assert(iReg < 16); 1796 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8; 1797 } 1798 /* high 8-bit register. */ 1799 Assert(iReg < 8); 1800 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi; 1801 } 1802 #endif 1803 1804 1805 /** 1806 * Gets a reference (pointer) to the specified 8-bit general purpose register, 1807 * alternative version with extended (20) register index. 1808 * 1809 * @returns Register reference. 1810 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1811 * @param iRegEx The register. The 16 first are regular ones, 1812 * whereas 16 thru 19 maps to AH, CH, DH and BH. 1813 */ 1814 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT 1815 { 1816 /** @todo This could be done by double indexing on little endian hosts: 1817 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */ 1818 if (iRegEx < 16) 1819 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8; 1820 1821 /* high 8-bit register. */ 1822 Assert(iRegEx < 20); 1823 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi; 1824 } 1825 1826 1827 /** 1828 * Gets a reference (pointer) to the specified 16-bit general purpose register. 1829 * 1830 * @returns Register reference. 1831 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1832 * @param iReg The register. 1833 */ 1834 DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1835 { 1836 Assert(iReg < 16); 1837 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16; 1838 } 1839 1840 1841 /** 1842 * Gets a reference (pointer) to the specified 32-bit general purpose register. 1843 * 1844 * @returns Register reference. 1845 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1846 * @param iReg The register. 1847 */ 1848 DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1849 { 1850 Assert(iReg < 16); 1851 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32; 1852 } 1853 1854 1855 /** 1856 * Gets a reference (pointer) to the specified signed 32-bit general purpose register. 1857 * 1858 * @returns Register reference. 1859 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1860 * @param iReg The register. 1861 */ 1862 DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1863 { 1864 Assert(iReg < 16); 1865 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32; 1866 } 1867 1868 1869 /** 1870 * Gets a reference (pointer) to the specified 64-bit general purpose register. 1871 * 1872 * @returns Register reference. 1873 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1874 * @param iReg The register. 1875 */ 1876 DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1877 { 1878 Assert(iReg < 64); 1879 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64; 1880 } 1881 1882 1883 /** 1884 * Gets a reference (pointer) to the specified signed 64-bit general purpose register. 1885 * 1886 * @returns Register reference. 1887 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1888 * @param iReg The register. 1889 */ 1890 DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1891 { 1892 Assert(iReg < 16); 1893 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64; 1894 } 1895 1896 1897 /** 1898 * Gets a reference (pointer) to the specified segment register's base address. 1899 * 1900 * @returns Segment register base address reference. 1901 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1902 * @param iSegReg The segment selector. 1903 */ 1904 DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT 1905 { 1906 Assert(iSegReg < X86_SREG_COUNT); 1907 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 1908 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base; 1909 } 1910 1911 1912 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 1913 /** 1914 * Fetches the value of a 8-bit general purpose register. 1915 * 1916 * @returns The register value. 1917 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1918 * @param iReg The register. 1919 */ 1920 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1921 { 1922 return *iemGRegRefU8(pVCpu, iReg); 1923 } 1924 #endif 1925 1926 1927 /** 1928 * Fetches the value of a 8-bit general purpose register, alternative version 1929 * with extended (20) register index. 1930 1931 * @returns The register value. 1932 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1933 * @param iRegEx The register. The 16 first are regular ones, 1934 * whereas 16 thru 19 maps to AH, CH, DH and BH. 1935 */ 1936 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT 1937 { 1938 return *iemGRegRefU8Ex(pVCpu, iRegEx); 1939 } 1940 1941 1942 /** 1943 * Fetches the value of a 16-bit general purpose register. 1944 * 1945 * @returns The register value. 1946 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1947 * @param iReg The register. 1948 */ 1949 DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1950 { 1951 Assert(iReg < 16); 1952 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16; 1953 } 1954 1955 1956 /** 1957 * Fetches the value of a 32-bit general purpose register. 1958 * 1959 * @returns The register value. 1960 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1961 * @param iReg The register. 1962 */ 1963 DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1964 { 1965 Assert(iReg < 16); 1966 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32; 1967 } 1968 1969 1970 /** 1971 * Fetches the value of a 64-bit general purpose register. 1972 * 1973 * @returns The register value. 1974 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1975 * @param iReg The register. 1976 */ 1977 DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT 1978 { 1979 Assert(iReg < 16); 1980 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64; 1981 } 1982 1983 1984 /** 1985 * Stores a 16-bit value to a general purpose register. 1986 * 1987 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1988 * @param iReg The register. 1989 * @param uValue The value to store. 1990 */ 1991 DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT 1992 { 1993 Assert(iReg < 16); 1994 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue; 1995 } 1996 1997 1998 /** 1999 * Stores a 32-bit value to a general purpose register, implicitly clearing high 2000 * values. 2001 * 2002 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2003 * @param iReg The register. 2004 * @param uValue The value to store. 2005 */ 2006 DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT 2007 { 2008 Assert(iReg < 16); 2009 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue; 2010 } 2011 2012 2013 /** 2014 * Stores a 64-bit value to a general purpose register. 2015 * 2016 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2017 * @param iReg The register. 2018 * @param uValue The value to store. 2019 */ 2020 DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT 2021 { 2022 Assert(iReg < 16); 2023 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue; 2024 } 2025 2026 2027 /** 2028 * Get the address of the top of the stack. 2029 * 2030 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2031 */ 2032 DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT 2033 { 2034 if (IEM_IS_64BIT_CODE(pVCpu)) 2035 return pVCpu->cpum.GstCtx.rsp; 2036 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 2037 return pVCpu->cpum.GstCtx.esp; 2038 return pVCpu->cpum.GstCtx.sp; 2039 } 2040 2041 2042 /** 2043 * Updates the RIP/EIP/IP to point to the next instruction. 2044 * 2045 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2046 * @param cbInstr The number of bytes to add. 2047 */ 2048 DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT 2049 { 2050 /* 2051 * Advance RIP. 2052 * 2053 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit, 2054 * while in all other modes except LM64 the updates are 32-bit. This means 2055 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e. 2056 * 4GB and 64KB rollovers, and decide whether anything needs masking. 2057 * 2058 * See PC wrap around tests in bs3-cpu-weird-1. 2059 */ 2060 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip; 2061 uint64_t const uRipNext = uRipPrev + cbInstr; 2062 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16))) 2063 || IEM_IS_64BIT_CODE(pVCpu))) 2064 pVCpu->cpum.GstCtx.rip = uRipNext; 2065 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) 2066 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext; 2067 else 2068 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext; 2069 } 2070 2071 2072 /** 2073 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the 2074 * following EFLAGS bits are set: 2075 * - X86_EFL_RF - clear it. 2076 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them. 2077 * - X86_EFL_TF - generate single step \#DB trap. 2078 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not 2079 * instruction). 2080 * 2081 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}, 2082 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint 2083 * takes priority over both NMIs and hardware interrupts. So, neither is 2084 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are 2085 * either unsupported will be triggered on-top of any \#DB raised here.) 2086 * 2087 * The RF flag only needs to be cleared here as it only suppresses instruction 2088 * breakpoints which are not raised here (happens synchronously during 2089 * instruction fetching). 2090 * 2091 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its 2092 * status has no bearing on whether \#DB exceptions are raised. 2093 * 2094 * @note This must *NOT* be called by the two instructions setting the 2095 * CPUMCTX_INHIBIT_SHADOW_SS flag. 2096 * 2097 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events} 2098 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching 2099 * Stacks} 2100 */ 2101 template<uint32_t const a_fTF = X86_EFL_TF> 2102 static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT 2103 { 2104 /* 2105 * Normally we're just here to clear RF and/or interrupt shadow bits. 2106 */ 2107 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0)) 2108 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW); 2109 else 2110 { 2111 /* 2112 * Raise a #DB or/and DBGF event. 2113 */ 2114 VBOXSTRICTRC rcStrict; 2115 if (pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK)) 2116 { 2117 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); 2118 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; 2119 if (pVCpu->cpum.GstCtx.eflags.uBoth & a_fTF) 2120 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS; 2121 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT) 2122 >> CPUMCTX_DBG_HIT_DRX_SHIFT; 2123 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n", 2124 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6], 2125 pVCpu->cpum.GstCtx.rflags.uBoth)); 2126 2127 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK); 2128 rcStrict = iemRaiseDebugException(pVCpu); 2129 2130 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */ 2131 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict)) 2132 { 2133 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT; 2134 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict))); 2135 } 2136 } 2137 else 2138 { 2139 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK); 2140 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT; 2141 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict))); 2142 } 2143 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK; 2144 Assert(rcStrict != VINF_SUCCESS); 2145 return rcStrict; 2146 } 2147 return rcNormal; 2148 } 2149 2150 2151 /** 2152 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending. 2153 * 2154 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2155 * @param rcNormal VINF_SUCCESS to continue TB. 2156 * VINF_IEM_REEXEC_BREAK to force TB exit when 2157 * taking the wrong conditional branhc. 2158 */ 2159 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT 2160 { 2161 /* 2162 * We assume that most of the time nothing actually needs doing here. 2163 */ 2164 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX); 2165 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth 2166 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) )) 2167 return rcNormal; 2168 return iemFinishInstructionWithFlagsSet(pVCpu, rcNormal); 2169 } 2170 2171 2172 /** 2173 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF 2174 * and CPUMCTX_INHIBIT_SHADOW. 2175 * 2176 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2177 * @param cbInstr The number of bytes to add. 2178 */ 2179 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT 2180 { 2181 iemRegAddToRip(pVCpu, cbInstr); 2182 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 2183 } 2184 2185 2186 /** 2187 * Updates the RIP to point to the next instruction and clears EFLAGS.RF 2188 * and CPUMCTX_INHIBIT_SHADOW. 2189 * 2190 * Only called from 64-bit code. 2191 * 2192 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2193 * @param cbInstr The number of bytes to add. 2194 * @param rcNormal VINF_SUCCESS to continue TB. 2195 * VINF_IEM_REEXEC_BREAK to force TB exit when 2196 * taking the wrong conditional branhc. 2197 */ 2198 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2199 { 2200 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr; 2201 return iemRegFinishClearingRF(pVCpu, rcNormal); 2202 } 2203 2204 2205 /** 2206 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and 2207 * CPUMCTX_INHIBIT_SHADOW. 2208 * 2209 * This is never from 64-bit code. 2210 * 2211 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2212 * @param cbInstr The number of bytes to add. 2213 * @param rcNormal VINF_SUCCESS to continue TB. 2214 * VINF_IEM_REEXEC_BREAK to force TB exit when 2215 * taking the wrong conditional branhc. 2216 */ 2217 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2218 { 2219 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr); 2220 return iemRegFinishClearingRF(pVCpu, rcNormal); 2221 } 2222 2223 2224 /** 2225 * Updates the IP to point to the next instruction and clears EFLAGS.RF and 2226 * CPUMCTX_INHIBIT_SHADOW. 2227 * 2228 * This is only ever used from 16-bit code on a pre-386 CPU. 2229 * 2230 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2231 * @param cbInstr The number of bytes to add. 2232 * @param rcNormal VINF_SUCCESS to continue TB. 2233 * VINF_IEM_REEXEC_BREAK to force TB exit when 2234 * taking the wrong conditional branhc. 2235 */ 2236 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2237 { 2238 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr); 2239 return iemRegFinishClearingRF(pVCpu, rcNormal); 2240 } 2241 2242 2243 /** 2244 * Tail method for a finish function that does't clear flags or raise \#DB. 2245 * 2246 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2247 * @param rcNormal VINF_SUCCESS to continue TB. 2248 * VINF_IEM_REEXEC_BREAK to force TB exit when 2249 * taking the wrong conditional branhc. 2250 */ 2251 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT 2252 { 2253 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX); 2254 Assert(!( pVCpu->cpum.GstCtx.eflags.uBoth 2255 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ); 2256 RT_NOREF(pVCpu); 2257 return rcNormal; 2258 } 2259 2260 2261 /** 2262 * Updates the RIP to point to the next instruction, but does not need to clear 2263 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags. 2264 * 2265 * Only called from 64-bit code. 2266 * 2267 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2268 * @param cbInstr The number of bytes to add. 2269 * @param rcNormal VINF_SUCCESS to continue TB. 2270 * VINF_IEM_REEXEC_BREAK to force TB exit when 2271 * taking the wrong conditional branhc. 2272 */ 2273 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2274 { 2275 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr; 2276 return iemRegFinishNoFlags(pVCpu, rcNormal); 2277 } 2278 2279 2280 /** 2281 * Updates the EIP to point to the next instruction, but does not need to clear 2282 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags. 2283 * 2284 * This is never from 64-bit code. 2285 * 2286 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2287 * @param cbInstr The number of bytes to add. 2288 * @param rcNormal VINF_SUCCESS to continue TB. 2289 * VINF_IEM_REEXEC_BREAK to force TB exit when 2290 * taking the wrong conditional branhc. 2291 */ 2292 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2293 { 2294 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr); 2295 return iemRegFinishNoFlags(pVCpu, rcNormal); 2296 } 2297 2298 2299 /** 2300 * Updates the IP to point to the next instruction, but does not need to clear 2301 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags. 2302 * 2303 * This is only ever used from 16-bit code on a pre-386 CPU. 2304 * 2305 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2306 * @param cbInstr The number of bytes to add. 2307 * @param rcNormal VINF_SUCCESS to continue TB. 2308 * VINF_IEM_REEXEC_BREAK to force TB exit when 2309 * taking the wrong conditional branhc. 2310 * 2311 */ 2312 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT 2313 { 2314 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr); 2315 return iemRegFinishNoFlags(pVCpu, rcNormal); 2316 } 2317 2318 2319 /** 2320 * Adds a 8-bit signed jump offset to RIP from 64-bit code. 2321 * 2322 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2323 * segment limit. 2324 * 2325 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2326 * @param cbInstr Instruction size. 2327 * @param offNextInstr The offset of the next instruction. 2328 * @param enmEffOpSize Effective operand size. 2329 * @param rcNormal VINF_SUCCESS to continue TB. 2330 * VINF_IEM_REEXEC_BREAK to force TB exit when 2331 * taking the wrong conditional branhc. 2332 */ 2333 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2334 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2335 { 2336 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2337 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT); 2338 2339 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2340 if (enmEffOpSize == IEMMODE_16BIT) 2341 uNewRip &= UINT16_MAX; 2342 2343 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 2344 pVCpu->cpum.GstCtx.rip = uNewRip; 2345 else 2346 return iemRaiseGeneralProtectionFault0(pVCpu); 2347 2348 #ifndef IEM_WITH_CODE_TLB 2349 iemOpcodeFlushLight(pVCpu, cbInstr); 2350 #endif 2351 2352 /* 2353 * Clear RF and finish the instruction (maybe raise #DB). 2354 */ 2355 return iemRegFinishClearingRF(pVCpu, rcNormal); 2356 } 2357 2358 2359 /** 2360 * Adds a 8-bit signed jump offset to RIP from 64-bit code when the caller is 2361 * sure it stays within the same page. 2362 * 2363 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2364 * segment limit. 2365 * 2366 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2367 * @param cbInstr Instruction size. 2368 * @param offNextInstr The offset of the next instruction. 2369 * @param enmEffOpSize Effective operand size. 2370 * @param rcNormal VINF_SUCCESS to continue TB. 2371 * VINF_IEM_REEXEC_BREAK to force TB exit when 2372 * taking the wrong conditional branhc. 2373 */ 2374 DECL_FORCE_INLINE(VBOXSTRICTRC) 2375 iemRegRip64RelativeJumpS8IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2376 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2377 { 2378 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2379 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize); 2380 2381 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2382 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT)); 2383 pVCpu->cpum.GstCtx.rip = uNewRip; 2384 2385 #ifndef IEM_WITH_CODE_TLB 2386 iemOpcodeFlushLight(pVCpu, cbInstr); 2387 #endif 2388 2389 /* 2390 * Clear RF and finish the instruction (maybe raise #DB). 2391 */ 2392 return iemRegFinishClearingRF(pVCpu, rcNormal); 2393 } 2394 2395 2396 /** 2397 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit 2398 * code (never 64-bit). 2399 * 2400 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2401 * segment limit. 2402 * 2403 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2404 * @param cbInstr Instruction size. 2405 * @param offNextInstr The offset of the next instruction. 2406 * @param enmEffOpSize Effective operand size. 2407 * @param rcNormal VINF_SUCCESS to continue TB. 2408 * VINF_IEM_REEXEC_BREAK to force TB exit when 2409 * taking the wrong conditional branhc. 2410 */ 2411 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2412 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2413 { 2414 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2415 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); 2416 2417 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr; 2418 if (enmEffOpSize == IEMMODE_16BIT) 2419 uNewEip &= UINT16_MAX; 2420 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2421 pVCpu->cpum.GstCtx.rip = uNewEip; 2422 else 2423 return iemRaiseGeneralProtectionFault0(pVCpu); 2424 2425 #ifndef IEM_WITH_CODE_TLB 2426 iemOpcodeFlushLight(pVCpu, cbInstr); 2427 #endif 2428 2429 /* 2430 * Clear RF and finish the instruction (maybe raise #DB). 2431 */ 2432 return iemRegFinishClearingRF(pVCpu, rcNormal); 2433 } 2434 2435 2436 /** 2437 * Adds a 8-bit signed jump offset to EIP, on 386 or later from FLAT 32-bit code 2438 * (never 64-bit). 2439 * 2440 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2441 * segment limit. 2442 * 2443 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2444 * @param cbInstr Instruction size. 2445 * @param offNextInstr The offset of the next instruction. 2446 * @param enmEffOpSize Effective operand size. 2447 * @param rcNormal VINF_SUCCESS to continue TB. 2448 * VINF_IEM_REEXEC_BREAK to force TB exit when 2449 * taking the wrong conditional branhc. 2450 */ 2451 DECL_FORCE_INLINE(VBOXSTRICTRC) 2452 iemRegEip32RelativeJumpS8FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2453 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2454 { 2455 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2456 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); 2457 2458 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr; 2459 if (enmEffOpSize == IEMMODE_16BIT) 2460 uNewEip &= UINT16_MAX; 2461 pVCpu->cpum.GstCtx.rip = uNewEip; 2462 2463 #ifndef IEM_WITH_CODE_TLB 2464 iemOpcodeFlushLight(pVCpu, cbInstr); 2465 #endif 2466 2467 /* 2468 * Clear RF and finish the instruction (maybe raise #DB). 2469 */ 2470 return iemRegFinishClearingRF(pVCpu, rcNormal); 2471 } 2472 2473 2474 /** 2475 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU. 2476 * 2477 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2478 * segment limit. 2479 * 2480 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2481 * @param cbInstr Instruction size. 2482 * @param offNextInstr The offset of the next instruction. 2483 * @param rcNormal VINF_SUCCESS to continue TB. 2484 * VINF_IEM_REEXEC_BREAK to force TB exit when 2485 * taking the wrong conditional branhc. 2486 */ 2487 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2488 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT 2489 { 2490 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2491 2492 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr; 2493 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2494 pVCpu->cpum.GstCtx.rip = uNewIp; 2495 else 2496 return iemRaiseGeneralProtectionFault0(pVCpu); 2497 2498 #ifndef IEM_WITH_CODE_TLB 2499 iemOpcodeFlushLight(pVCpu, cbInstr); 2500 #endif 2501 2502 /* 2503 * Clear RF and finish the instruction (maybe raise #DB). 2504 */ 2505 return iemRegFinishClearingRF(pVCpu, rcNormal); 2506 } 2507 2508 2509 /** 2510 * Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or 2511 * clearing of flags. 2512 * 2513 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2514 * segment limit. 2515 * 2516 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2517 * @param cbInstr Instruction size. 2518 * @param offNextInstr The offset of the next instruction. 2519 * @param enmEffOpSize Effective operand size. 2520 * @param rcNormal VINF_SUCCESS to continue TB. 2521 * VINF_IEM_REEXEC_BREAK to force TB exit when 2522 * taking the wrong conditional branhc. 2523 */ 2524 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2525 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2526 { 2527 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2528 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT); 2529 2530 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2531 if (enmEffOpSize == IEMMODE_16BIT) 2532 uNewRip &= UINT16_MAX; 2533 2534 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 2535 pVCpu->cpum.GstCtx.rip = uNewRip; 2536 else 2537 return iemRaiseGeneralProtectionFault0(pVCpu); 2538 2539 #ifndef IEM_WITH_CODE_TLB 2540 iemOpcodeFlushLight(pVCpu, cbInstr); 2541 #endif 2542 return iemRegFinishNoFlags(pVCpu, rcNormal); 2543 } 2544 2545 2546 /** 2547 * Adds a 8-bit signed jump offset to RIP from 64-bit code when caller is sure 2548 * it stays within the same page, no checking or clearing of flags. 2549 * 2550 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2551 * segment limit. 2552 * 2553 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2554 * @param cbInstr Instruction size. 2555 * @param offNextInstr The offset of the next instruction. 2556 * @param enmEffOpSize Effective operand size. 2557 * @param rcNormal VINF_SUCCESS to continue TB. 2558 * VINF_IEM_REEXEC_BREAK to force TB exit when 2559 * taking the wrong conditional branhc. 2560 */ 2561 DECL_FORCE_INLINE(VBOXSTRICTRC) 2562 iemRegRip64RelativeJumpS8IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2563 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2564 { 2565 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2566 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize); 2567 2568 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2569 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT)); 2570 pVCpu->cpum.GstCtx.rip = uNewRip; 2571 2572 #ifndef IEM_WITH_CODE_TLB 2573 iemOpcodeFlushLight(pVCpu, cbInstr); 2574 #endif 2575 return iemRegFinishNoFlags(pVCpu, rcNormal); 2576 } 2577 2578 2579 /** 2580 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit 2581 * code (never 64-bit), no checking or clearing of flags. 2582 * 2583 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2584 * segment limit. 2585 * 2586 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2587 * @param cbInstr Instruction size. 2588 * @param offNextInstr The offset of the next instruction. 2589 * @param enmEffOpSize Effective operand size. 2590 * @param rcNormal VINF_SUCCESS to continue TB. 2591 * VINF_IEM_REEXEC_BREAK to force TB exit when 2592 * taking the wrong conditional branhc. 2593 */ 2594 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2595 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2596 { 2597 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2598 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); 2599 2600 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr; 2601 if (enmEffOpSize == IEMMODE_16BIT) 2602 uNewEip &= UINT16_MAX; 2603 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2604 pVCpu->cpum.GstCtx.rip = uNewEip; 2605 else 2606 return iemRaiseGeneralProtectionFault0(pVCpu); 2607 2608 #ifndef IEM_WITH_CODE_TLB 2609 iemOpcodeFlushLight(pVCpu, cbInstr); 2610 #endif 2611 return iemRegFinishNoFlags(pVCpu, rcNormal); 2612 } 2613 2614 2615 /** 2616 * Adds a 8-bit signed jump offset to EIP, on 386 or later from flat 32-bit code 2617 * (never 64-bit), no checking or clearing of flags. 2618 * 2619 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2620 * segment limit. 2621 * 2622 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2623 * @param cbInstr Instruction size. 2624 * @param offNextInstr The offset of the next instruction. 2625 * @param enmEffOpSize Effective operand size. 2626 * @param rcNormal VINF_SUCCESS to continue TB. 2627 * VINF_IEM_REEXEC_BREAK to force TB exit when 2628 * taking the wrong conditional branhc. 2629 */ 2630 DECL_FORCE_INLINE(VBOXSTRICTRC) 2631 iemRegEip32RelativeJumpS8FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, 2632 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT 2633 { 2634 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2635 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); 2636 2637 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr; 2638 if (enmEffOpSize == IEMMODE_16BIT) 2639 uNewEip &= UINT16_MAX; 2640 pVCpu->cpum.GstCtx.rip = uNewEip; 2641 2642 #ifndef IEM_WITH_CODE_TLB 2643 iemOpcodeFlushLight(pVCpu, cbInstr); 2644 #endif 2645 return iemRegFinishNoFlags(pVCpu, rcNormal); 2646 } 2647 2648 2649 /** 2650 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or 2651 * clearing of flags. 2652 * 2653 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2654 * segment limit. 2655 * 2656 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2657 * @param cbInstr Instruction size. 2658 * @param offNextInstr The offset of the next instruction. 2659 * @param rcNormal VINF_SUCCESS to continue TB. 2660 * VINF_IEM_REEXEC_BREAK to force TB exit when 2661 * taking the wrong conditional branhc. 2662 */ 2663 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 2664 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT 2665 { 2666 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2667 2668 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr; 2669 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2670 pVCpu->cpum.GstCtx.rip = uNewIp; 2671 else 2672 return iemRaiseGeneralProtectionFault0(pVCpu); 2673 2674 #ifndef IEM_WITH_CODE_TLB 2675 iemOpcodeFlushLight(pVCpu, cbInstr); 2676 #endif 2677 return iemRegFinishNoFlags(pVCpu, rcNormal); 2678 } 2679 2680 2681 /** 2682 * Adds a 16-bit signed jump offset to RIP from 64-bit code. 2683 * 2684 * @returns Strict VBox status code. 2685 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2686 * @param cbInstr Instruction size. 2687 * @param offNextInstr The offset of the next instruction. 2688 * @param rcNormal VINF_SUCCESS to continue TB. 2689 * VINF_IEM_REEXEC_BREAK to force TB exit when 2690 * taking the wrong conditional branhc. 2691 */ 2692 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2693 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2694 { 2695 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2696 2697 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr); 2698 2699 #ifndef IEM_WITH_CODE_TLB 2700 iemOpcodeFlushLight(pVCpu, cbInstr); 2701 #endif 2702 2703 /* 2704 * Clear RF and finish the instruction (maybe raise #DB). 2705 */ 2706 return iemRegFinishClearingRF(pVCpu, rcNormal); 2707 } 2708 2709 2710 /** 2711 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code. 2712 * 2713 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2714 * segment limit. 2715 * 2716 * @returns Strict VBox status code. 2717 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2718 * @param cbInstr Instruction size. 2719 * @param offNextInstr The offset of the next instruction. 2720 * @param rcNormal VINF_SUCCESS to continue TB. 2721 * VINF_IEM_REEXEC_BREAK to force TB exit when 2722 * taking the wrong conditional branhc. 2723 * 2724 * @note This is also used by 16-bit code in pre-386 mode, as the code is 2725 * identical. 2726 */ 2727 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2728 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2729 { 2730 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2731 2732 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr; 2733 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2734 pVCpu->cpum.GstCtx.rip = uNewIp; 2735 else 2736 return iemRaiseGeneralProtectionFault0(pVCpu); 2737 2738 #ifndef IEM_WITH_CODE_TLB 2739 iemOpcodeFlushLight(pVCpu, cbInstr); 2740 #endif 2741 2742 /* 2743 * Clear RF and finish the instruction (maybe raise #DB). 2744 */ 2745 return iemRegFinishClearingRF(pVCpu, rcNormal); 2746 } 2747 2748 2749 /** 2750 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code. 2751 * 2752 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2753 * segment limit. 2754 * 2755 * @returns Strict VBox status code. 2756 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2757 * @param cbInstr Instruction size. 2758 * @param offNextInstr The offset of the next instruction. 2759 * @param rcNormal VINF_SUCCESS to continue TB. 2760 * VINF_IEM_REEXEC_BREAK to force TB exit when 2761 * taking the wrong conditional branhc. 2762 * 2763 * @note This is also used by 16-bit code in pre-386 mode, as the code is 2764 * identical. 2765 */ 2766 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2767 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2768 { 2769 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2770 2771 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr; 2772 pVCpu->cpum.GstCtx.rip = uNewIp; 2773 2774 #ifndef IEM_WITH_CODE_TLB 2775 iemOpcodeFlushLight(pVCpu, cbInstr); 2776 #endif 2777 2778 /* 2779 * Clear RF and finish the instruction (maybe raise #DB). 2780 */ 2781 return iemRegFinishClearingRF(pVCpu, rcNormal); 2782 } 2783 2784 2785 /** 2786 * Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or 2787 * clearing of flags. 2788 * 2789 * @returns Strict VBox status code. 2790 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2791 * @param cbInstr Instruction size. 2792 * @param offNextInstr The offset of the next instruction. 2793 * @param rcNormal VINF_SUCCESS to continue TB. 2794 * VINF_IEM_REEXEC_BREAK to force TB exit when 2795 * taking the wrong conditional branhc. 2796 */ 2797 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 2798 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2799 { 2800 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2801 2802 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr); 2803 2804 #ifndef IEM_WITH_CODE_TLB 2805 iemOpcodeFlushLight(pVCpu, cbInstr); 2806 #endif 2807 return iemRegFinishNoFlags(pVCpu, rcNormal); 2808 } 2809 2810 2811 /** 2812 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code, 2813 * no checking or clearing of flags. 2814 * 2815 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2816 * segment limit. 2817 * 2818 * @returns Strict VBox status code. 2819 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2820 * @param cbInstr Instruction size. 2821 * @param offNextInstr The offset of the next instruction. 2822 * @param rcNormal VINF_SUCCESS to continue TB. 2823 * VINF_IEM_REEXEC_BREAK to force TB exit when 2824 * taking the wrong conditional branhc. 2825 * 2826 * @note This is also used by 16-bit code in pre-386 mode, as the code is 2827 * identical. 2828 */ 2829 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 2830 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2831 { 2832 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2833 2834 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr; 2835 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2836 pVCpu->cpum.GstCtx.rip = uNewIp; 2837 else 2838 return iemRaiseGeneralProtectionFault0(pVCpu); 2839 2840 #ifndef IEM_WITH_CODE_TLB 2841 iemOpcodeFlushLight(pVCpu, cbInstr); 2842 #endif 2843 return iemRegFinishNoFlags(pVCpu, rcNormal); 2844 } 2845 2846 2847 /** 2848 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code, no checking or 2849 * clearing of flags. 2850 * 2851 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2852 * segment limit. 2853 * 2854 * @returns Strict VBox status code. 2855 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2856 * @param cbInstr Instruction size. 2857 * @param offNextInstr The offset of the next instruction. 2858 * @param rcNormal VINF_SUCCESS to continue TB. 2859 * VINF_IEM_REEXEC_BREAK to force TB exit when 2860 * taking the wrong conditional branhc. 2861 * 2862 * @note This is also used by 16-bit code in pre-386 mode, as the code is 2863 * identical. 2864 */ 2865 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 2866 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT 2867 { 2868 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2869 2870 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr; 2871 pVCpu->cpum.GstCtx.rip = uNewIp; 2872 2873 #ifndef IEM_WITH_CODE_TLB 2874 iemOpcodeFlushLight(pVCpu, cbInstr); 2875 #endif 2876 return iemRegFinishNoFlags(pVCpu, rcNormal); 2877 } 2878 2879 2880 /** 2881 * Adds a 32-bit signed jump offset to RIP from 64-bit code. 2882 * 2883 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2884 * segment limit. 2885 * 2886 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the 2887 * only alternative for relative jumps in 64-bit code and that is already 2888 * handled in the decoder stage. 2889 * 2890 * @returns Strict VBox status code. 2891 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2892 * @param cbInstr Instruction size. 2893 * @param offNextInstr The offset of the next instruction. 2894 * @param rcNormal VINF_SUCCESS to continue TB. 2895 * VINF_IEM_REEXEC_BREAK to force TB exit when 2896 * taking the wrong conditional branhc. 2897 */ 2898 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2899 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 2900 { 2901 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2902 2903 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2904 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 2905 pVCpu->cpum.GstCtx.rip = uNewRip; 2906 else 2907 return iemRaiseGeneralProtectionFault0(pVCpu); 2908 2909 #ifndef IEM_WITH_CODE_TLB 2910 iemOpcodeFlushLight(pVCpu, cbInstr); 2911 #endif 2912 2913 /* 2914 * Clear RF and finish the instruction (maybe raise #DB). 2915 */ 2916 return iemRegFinishClearingRF(pVCpu, rcNormal); 2917 } 2918 2919 2920 /** 2921 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is 2922 * sure the target is in the same page. 2923 * 2924 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2925 * segment limit. 2926 * 2927 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the 2928 * only alternative for relative jumps in 64-bit code and that is already 2929 * handled in the decoder stage. 2930 * 2931 * @returns Strict VBox status code. 2932 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2933 * @param cbInstr Instruction size. 2934 * @param offNextInstr The offset of the next instruction. 2935 * @param rcNormal VINF_SUCCESS to continue TB. 2936 * VINF_IEM_REEXEC_BREAK to force TB exit when 2937 * taking the wrong conditional branhc. 2938 */ 2939 DECL_FORCE_INLINE(VBOXSTRICTRC) 2940 iemRegRip64RelativeJumpS32IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2941 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 2942 { 2943 Assert(IEM_IS_64BIT_CODE(pVCpu)); 2944 2945 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 2946 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT)); 2947 pVCpu->cpum.GstCtx.rip = uNewRip; 2948 2949 #ifndef IEM_WITH_CODE_TLB 2950 iemOpcodeFlushLight(pVCpu, cbInstr); 2951 #endif 2952 2953 /* 2954 * Clear RF and finish the instruction (maybe raise #DB). 2955 */ 2956 return iemRegFinishClearingRF(pVCpu, rcNormal); 2957 } 2958 2959 2960 /** 2961 * Adds a 32-bit signed jump offset to RIP from 64-bit code. 2962 * 2963 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 2964 * segment limit. 2965 * 2966 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the 2967 * only alternative for relative jumps in 32-bit code and that is already 2968 * handled in the decoder stage. 2969 * 2970 * @returns Strict VBox status code. 2971 * @param pVCpu The cross context virtual CPU structure of the calling thread. 2972 * @param cbInstr Instruction size. 2973 * @param offNextInstr The offset of the next instruction. 2974 * @param rcNormal VINF_SUCCESS to continue TB. 2975 * VINF_IEM_REEXEC_BREAK to force TB exit when 2976 * taking the wrong conditional branhc. 2977 */ 2978 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 2979 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 2980 { 2981 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 2982 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 2983 2984 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr; 2985 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 2986 pVCpu->cpum.GstCtx.rip = uNewEip; 2987 else 2988 return iemRaiseGeneralProtectionFault0(pVCpu); 2989 2990 #ifndef IEM_WITH_CODE_TLB 2991 iemOpcodeFlushLight(pVCpu, cbInstr); 2992 #endif 2993 2994 /* 2995 * Clear RF and finish the instruction (maybe raise #DB). 2996 */ 2997 return iemRegFinishClearingRF(pVCpu, rcNormal); 2998 } 2999 3000 3001 /** 3002 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code. 3003 * 3004 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3005 * segment limit. 3006 * 3007 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the 3008 * only alternative for relative jumps in 32-bit code and that is already 3009 * handled in the decoder stage. 3010 * 3011 * @returns Strict VBox status code. 3012 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3013 * @param cbInstr Instruction size. 3014 * @param offNextInstr The offset of the next instruction. 3015 * @param rcNormal VINF_SUCCESS to continue TB. 3016 * VINF_IEM_REEXEC_BREAK to force TB exit when 3017 * taking the wrong conditional branhc. 3018 */ 3019 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, 3020 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 3021 { 3022 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3023 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 3024 3025 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr; 3026 pVCpu->cpum.GstCtx.rip = uNewEip; 3027 3028 #ifndef IEM_WITH_CODE_TLB 3029 iemOpcodeFlushLight(pVCpu, cbInstr); 3030 #endif 3031 3032 /* 3033 * Clear RF and finish the instruction (maybe raise #DB). 3034 */ 3035 return iemRegFinishClearingRF(pVCpu, rcNormal); 3036 } 3037 3038 3039 3040 /** 3041 * Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or 3042 * clearing of flags. 3043 * 3044 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3045 * segment limit. 3046 * 3047 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the 3048 * only alternative for relative jumps in 64-bit code and that is already 3049 * handled in the decoder stage. 3050 * 3051 * @returns Strict VBox status code. 3052 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3053 * @param cbInstr Instruction size. 3054 * @param offNextInstr The offset of the next instruction. 3055 * @param rcNormal VINF_SUCCESS to continue TB. 3056 * VINF_IEM_REEXEC_BREAK to force TB exit when 3057 * taking the wrong conditional branhc. 3058 */ 3059 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 3060 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 3061 { 3062 Assert(IEM_IS_64BIT_CODE(pVCpu)); 3063 3064 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 3065 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 3066 pVCpu->cpum.GstCtx.rip = uNewRip; 3067 else 3068 return iemRaiseGeneralProtectionFault0(pVCpu); 3069 3070 #ifndef IEM_WITH_CODE_TLB 3071 iemOpcodeFlushLight(pVCpu, cbInstr); 3072 #endif 3073 return iemRegFinishNoFlags(pVCpu, rcNormal); 3074 } 3075 3076 3077 /** 3078 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is 3079 * sure it stays within the same page, no checking or clearing of flags. 3080 * 3081 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3082 * segment limit. 3083 * 3084 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the 3085 * only alternative for relative jumps in 64-bit code and that is already 3086 * handled in the decoder stage. 3087 * 3088 * @returns Strict VBox status code. 3089 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3090 * @param cbInstr Instruction size. 3091 * @param offNextInstr The offset of the next instruction. 3092 * @param rcNormal VINF_SUCCESS to continue TB. 3093 * VINF_IEM_REEXEC_BREAK to force TB exit when 3094 * taking the wrong conditional branhc. 3095 */ 3096 DECL_FORCE_INLINE(VBOXSTRICTRC) 3097 iemRegRip64RelativeJumpS32IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 3098 { 3099 Assert(IEM_IS_64BIT_CODE(pVCpu)); 3100 3101 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; 3102 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT)); 3103 pVCpu->cpum.GstCtx.rip = uNewRip; 3104 3105 #ifndef IEM_WITH_CODE_TLB 3106 iemOpcodeFlushLight(pVCpu, cbInstr); 3107 #endif 3108 return iemRegFinishNoFlags(pVCpu, rcNormal); 3109 } 3110 3111 3112 /** 3113 * Adds a 32-bit signed jump offset to RIP from 32-bit code, no checking or 3114 * clearing of flags. 3115 * 3116 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3117 * segment limit. 3118 * 3119 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the 3120 * only alternative for relative jumps in 32-bit code and that is already 3121 * handled in the decoder stage. 3122 * 3123 * @returns Strict VBox status code. 3124 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3125 * @param cbInstr Instruction size. 3126 * @param offNextInstr The offset of the next instruction. 3127 * @param rcNormal VINF_SUCCESS to continue TB. 3128 * VINF_IEM_REEXEC_BREAK to force TB exit when 3129 * taking the wrong conditional branhc. 3130 */ 3131 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 3132 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 3133 { 3134 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3135 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 3136 3137 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr; 3138 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 3139 pVCpu->cpum.GstCtx.rip = uNewEip; 3140 else 3141 return iemRaiseGeneralProtectionFault0(pVCpu); 3142 3143 #ifndef IEM_WITH_CODE_TLB 3144 iemOpcodeFlushLight(pVCpu, cbInstr); 3145 #endif 3146 return iemRegFinishNoFlags(pVCpu, rcNormal); 3147 } 3148 3149 3150 /** 3151 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code, no checking or 3152 * clearing of flags. 3153 * 3154 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3155 * segment limit. 3156 * 3157 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the 3158 * only alternative for relative jumps in 32-bit code and that is already 3159 * handled in the decoder stage. 3160 * 3161 * @returns Strict VBox status code. 3162 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3163 * @param cbInstr Instruction size. 3164 * @param offNextInstr The offset of the next instruction. 3165 * @param rcNormal VINF_SUCCESS to continue TB. 3166 * VINF_IEM_REEXEC_BREAK to force TB exit when 3167 * taking the wrong conditional branhc. 3168 */ 3169 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, 3170 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT 3171 { 3172 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3173 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 3174 3175 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr; 3176 pVCpu->cpum.GstCtx.rip = uNewEip; 3177 3178 #ifndef IEM_WITH_CODE_TLB 3179 iemOpcodeFlushLight(pVCpu, cbInstr); 3180 #endif 3181 return iemRegFinishNoFlags(pVCpu, rcNormal); 3182 } 3183 3184 3185 /** 3186 * Extended version of iemFinishInstructionWithFlagsSet that goes with 3187 * iemRegAddToRipAndFinishingClearingRfEx. 3188 * 3189 * See iemFinishInstructionWithFlagsSet() for details. 3190 */ 3191 static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT 3192 { 3193 /* 3194 * Raise a #DB. 3195 */ 3196 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); 3197 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; 3198 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS 3199 | ( (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT) 3200 >> CPUMCTX_DBG_HIT_DRX_SHIFT); 3201 /** @todo Do we set all pending \#DB events, or just one? */ 3202 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n", 3203 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6], 3204 pVCpu->cpum.GstCtx.rflags.uBoth)); 3205 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK); 3206 return iemRaiseDebugException(pVCpu); 3207 } 3208 3209 3210 /** 3211 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and 3212 * others potentially updating EFLAGS.TF. 3213 * 3214 * The single step event must be generated using the TF value at the start of 3215 * the instruction, not the new value set by it. 3216 * 3217 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3218 * @param cbInstr The number of bytes to add. 3219 * @param fEflOld The EFLAGS at the start of the instruction 3220 * execution. 3221 */ 3222 DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT 3223 { 3224 iemRegAddToRip(pVCpu, cbInstr); 3225 if (!(fEflOld & X86_EFL_TF)) 3226 { 3227 /* Specialized iemRegFinishClearingRF edition here that doesn't check X86_EFL_TF. */ 3228 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX); 3229 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth 3230 & (X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) )) 3231 return VINF_SUCCESS; 3232 return iemFinishInstructionWithFlagsSet<0 /*a_fTF*/>(pVCpu, VINF_SUCCESS); /* TF=0, so ignore it. */ 3233 } 3234 return iemFinishInstructionWithTfSet(pVCpu); 3235 } 3236 3237 3238 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 3239 /** 3240 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF. 3241 * 3242 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3243 */ 3244 DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT 3245 { 3246 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu)); 3247 } 3248 #endif 3249 3250 3251 #ifdef IEM_WITH_CODE_TLB 3252 3253 /** 3254 * Performs a near jump to the specified address, no checking or clearing of 3255 * flags 3256 * 3257 * May raise a \#GP(0) if the new IP outside the code segment limit. 3258 * 3259 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3260 * @param uNewIp The new IP value. 3261 */ 3262 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT 3263 { 3264 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit 3265 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */)) 3266 pVCpu->cpum.GstCtx.rip = uNewIp; 3267 else 3268 return iemRaiseGeneralProtectionFault0(pVCpu); 3269 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3270 } 3271 3272 3273 /** 3274 * Performs a near jump to the specified address, no checking or clearing of 3275 * flags 3276 * 3277 * May raise a \#GP(0) if the new RIP is outside the code segment limit. 3278 * 3279 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3280 * @param uNewEip The new EIP value. 3281 */ 3282 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT 3283 { 3284 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 3285 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3286 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 3287 pVCpu->cpum.GstCtx.rip = uNewEip; 3288 else 3289 return iemRaiseGeneralProtectionFault0(pVCpu); 3290 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3291 } 3292 3293 3294 /** 3295 * Performs a near jump to the specified address, no checking or clearing of 3296 * flags. 3297 * 3298 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3299 * segment limit. 3300 * 3301 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3302 * @param uNewRip The new RIP value. 3303 */ 3304 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT 3305 { 3306 Assert(IEM_IS_64BIT_CODE(pVCpu)); 3307 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 3308 pVCpu->cpum.GstCtx.rip = uNewRip; 3309 else 3310 return iemRaiseGeneralProtectionFault0(pVCpu); 3311 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3312 } 3313 3314 #endif /* IEM_WITH_CODE_TLB */ 3315 3316 /** 3317 * Performs a near jump to the specified address. 3318 * 3319 * May raise a \#GP(0) if the new IP outside the code segment limit. 3320 * 3321 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3322 * @param uNewIp The new IP value. 3323 * @param cbInstr The instruction length, for flushing in the non-TLB case. 3324 */ 3325 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp, uint8_t cbInstr) RT_NOEXCEPT 3326 { 3327 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit 3328 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */)) 3329 pVCpu->cpum.GstCtx.rip = uNewIp; 3330 else 3331 return iemRaiseGeneralProtectionFault0(pVCpu); 3332 #ifndef IEM_WITH_CODE_TLB 3333 iemOpcodeFlushLight(pVCpu, cbInstr); 3334 #else 3335 RT_NOREF_PV(cbInstr); 3336 #endif 3337 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3338 } 3339 3340 3341 /** 3342 * Performs a near jump to the specified address. 3343 * 3344 * May raise a \#GP(0) if the new RIP is outside the code segment limit. 3345 * 3346 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3347 * @param uNewEip The new EIP value. 3348 * @param cbInstr The instruction length, for flushing in the non-TLB case. 3349 */ 3350 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip, uint8_t cbInstr) RT_NOEXCEPT 3351 { 3352 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); 3353 Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3354 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) 3355 pVCpu->cpum.GstCtx.rip = uNewEip; 3356 else 3357 return iemRaiseGeneralProtectionFault0(pVCpu); 3358 #ifndef IEM_WITH_CODE_TLB 3359 iemOpcodeFlushLight(pVCpu, cbInstr); 3360 #else 3361 RT_NOREF_PV(cbInstr); 3362 #endif 3363 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3364 } 3365 3366 3367 /** 3368 * Performs a near jump to the specified address. 3369 * 3370 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code 3371 * segment limit. 3372 * 3373 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3374 * @param uNewRip The new RIP value. 3375 * @param cbInstr The instruction length, for flushing in the non-TLB case. 3376 */ 3377 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip, uint8_t cbInstr) RT_NOEXCEPT 3378 { 3379 Assert(IEM_IS_64BIT_CODE(pVCpu)); 3380 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) 3381 pVCpu->cpum.GstCtx.rip = uNewRip; 3382 else 3383 return iemRaiseGeneralProtectionFault0(pVCpu); 3384 #ifndef IEM_WITH_CODE_TLB 3385 iemOpcodeFlushLight(pVCpu, cbInstr); 3386 #else 3387 RT_NOREF_PV(cbInstr); 3388 #endif 3389 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3390 } 3391 3392 3393 /** 3394 * Implements a 16-bit relative call, no checking or clearing of 3395 * flags. 3396 * 3397 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3398 * @param cbInstr The instruction length. 3399 * @param offDisp The 16-bit displacement. 3400 */ 3401 DECL_FORCE_INLINE(VBOXSTRICTRC) 3402 iemRegRipRelativeCallS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT 3403 { 3404 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr; 3405 uint16_t const uNewIp = uOldIp + offDisp; 3406 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit 3407 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */) 3408 { /* likely */ } 3409 else 3410 return iemRaiseGeneralProtectionFault0(pVCpu); 3411 3412 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp); 3413 if (rcStrict == VINF_SUCCESS) 3414 { /* likely */ } 3415 else 3416 return rcStrict; 3417 3418 pVCpu->cpum.GstCtx.rip = uNewIp; 3419 #ifndef IEM_WITH_CODE_TLB 3420 iemOpcodeFlushLight(pVCpu, cbInstr); 3421 #endif 3422 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3423 } 3424 3425 3426 /** 3427 * Implements a 16-bit relative call. 3428 * 3429 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3430 * @param cbInstr The instruction length. 3431 * @param offDisp The 16-bit displacement. 3432 */ 3433 DECL_FORCE_INLINE(VBOXSTRICTRC) 3434 iemRegRipRelativeCallS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT 3435 { 3436 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr; 3437 uint16_t const uNewIp = uOldIp + offDisp; 3438 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit 3439 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */) 3440 { /* likely */ } 3441 else 3442 return iemRaiseGeneralProtectionFault0(pVCpu); 3443 3444 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp); 3445 if (rcStrict == VINF_SUCCESS) 3446 { /* likely */ } 3447 else 3448 return rcStrict; 3449 3450 pVCpu->cpum.GstCtx.rip = uNewIp; 3451 #ifndef IEM_WITH_CODE_TLB 3452 iemOpcodeFlushLight(pVCpu, cbInstr); 3453 #endif 3454 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3455 } 3456 3457 3458 /** 3459 * Implements a 32-bit relative call, no checking or clearing of flags. 3460 * 3461 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3462 * @param cbInstr The instruction length. 3463 * @param offDisp The 32-bit displacement. 3464 */ 3465 DECL_FORCE_INLINE(VBOXSTRICTRC) 3466 iemRegEip32RelativeCallS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT 3467 { 3468 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3469 3470 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr; 3471 uint32_t const uNewRip = uOldRip + offDisp; 3472 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3473 { /* likely */ } 3474 else 3475 return iemRaiseGeneralProtectionFault0(pVCpu); 3476 3477 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip); 3478 if (rcStrict == VINF_SUCCESS) 3479 { /* likely */ } 3480 else 3481 return rcStrict; 3482 3483 pVCpu->cpum.GstCtx.rip = uNewRip; 3484 #ifndef IEM_WITH_CODE_TLB 3485 iemOpcodeFlushLight(pVCpu, cbInstr); 3486 #endif 3487 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3488 } 3489 3490 3491 /** 3492 * Implements a 32-bit relative call. 3493 * 3494 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3495 * @param cbInstr The instruction length. 3496 * @param offDisp The 32-bit displacement. 3497 */ 3498 DECL_FORCE_INLINE(VBOXSTRICTRC) 3499 iemRegEip32RelativeCallS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT 3500 { 3501 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu)); 3502 3503 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr; 3504 uint32_t const uNewRip = uOldRip + offDisp; 3505 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3506 { /* likely */ } 3507 else 3508 return iemRaiseGeneralProtectionFault0(pVCpu); 3509 3510 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip); 3511 if (rcStrict == VINF_SUCCESS) 3512 { /* likely */ } 3513 else 3514 return rcStrict; 3515 3516 pVCpu->cpum.GstCtx.rip = uNewRip; 3517 #ifndef IEM_WITH_CODE_TLB 3518 iemOpcodeFlushLight(pVCpu, cbInstr); 3519 #endif 3520 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3521 } 3522 3523 3524 /** 3525 * Implements a 64-bit relative call, no checking or clearing of flags. 3526 * 3527 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3528 * @param cbInstr The instruction length. 3529 * @param offDisp The 64-bit displacement. 3530 */ 3531 DECL_FORCE_INLINE(VBOXSTRICTRC) 3532 iemRegRip64RelativeCallS64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT 3533 { 3534 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr; 3535 uint64_t const uNewRip = uOldRip + (int64_t)offDisp; 3536 if (IEM_IS_CANONICAL(uNewRip)) 3537 { /* likely */ } 3538 else 3539 return iemRaiseNotCanonical(pVCpu); 3540 3541 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip); 3542 if (rcStrict == VINF_SUCCESS) 3543 { /* likely */ } 3544 else 3545 return rcStrict; 3546 3547 pVCpu->cpum.GstCtx.rip = uNewRip; 3548 #ifndef IEM_WITH_CODE_TLB 3549 iemOpcodeFlushLight(pVCpu, cbInstr); 3550 #endif 3551 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3552 } 3553 3554 3555 /** 3556 * Implements a 64-bit relative call. 3557 * 3558 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3559 * @param cbInstr The instruction length. 3560 * @param offDisp The 64-bit displacement. 3561 */ 3562 DECL_FORCE_INLINE(VBOXSTRICTRC) 3563 iemRegRip64RelativeCallS64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT 3564 { 3565 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr; 3566 uint64_t const uNewRip = uOldRip + (int64_t)offDisp; 3567 if (IEM_IS_CANONICAL(uNewRip)) 3568 { /* likely */ } 3569 else 3570 return iemRaiseNotCanonical(pVCpu); 3571 3572 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip); 3573 if (rcStrict == VINF_SUCCESS) 3574 { /* likely */ } 3575 else 3576 return rcStrict; 3577 3578 pVCpu->cpum.GstCtx.rip = uNewRip; 3579 #ifndef IEM_WITH_CODE_TLB 3580 iemOpcodeFlushLight(pVCpu, cbInstr); 3581 #endif 3582 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3583 } 3584 3585 3586 /** 3587 * Implements an 16-bit indirect call, no checking or clearing of 3588 * flags. 3589 * 3590 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3591 * @param cbInstr The instruction length. 3592 * @param uNewRip The new RIP value. 3593 */ 3594 DECL_FORCE_INLINE(VBOXSTRICTRC) 3595 iemRegIp16IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT 3596 { 3597 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr; 3598 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3599 { /* likely */ } 3600 else 3601 return iemRaiseGeneralProtectionFault0(pVCpu); 3602 3603 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip); 3604 if (rcStrict == VINF_SUCCESS) 3605 { /* likely */ } 3606 else 3607 return rcStrict; 3608 3609 pVCpu->cpum.GstCtx.rip = uNewRip; 3610 #ifndef IEM_WITH_CODE_TLB 3611 iemOpcodeFlushLight(pVCpu, cbInstr); 3612 #endif 3613 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3614 } 3615 3616 3617 /** 3618 * Implements an 16-bit indirect call, no checking or clearing of 3619 * flags. 3620 * 3621 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3622 * @param cbInstr The instruction length. 3623 * @param uNewRip The new RIP value. 3624 */ 3625 DECL_FORCE_INLINE(VBOXSTRICTRC) 3626 iemRegEip32IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT 3627 { 3628 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr; 3629 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3630 { /* likely */ } 3631 else 3632 return iemRaiseGeneralProtectionFault0(pVCpu); 3633 3634 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip); 3635 if (rcStrict == VINF_SUCCESS) 3636 { /* likely */ } 3637 else 3638 return rcStrict; 3639 3640 pVCpu->cpum.GstCtx.rip = uNewRip; 3641 #ifndef IEM_WITH_CODE_TLB 3642 iemOpcodeFlushLight(pVCpu, cbInstr); 3643 #endif 3644 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3645 } 3646 3647 3648 /** 3649 * Implements an 16-bit indirect call. 3650 * 3651 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3652 * @param cbInstr The instruction length. 3653 * @param uNewRip The new RIP value. 3654 */ 3655 DECL_FORCE_INLINE(VBOXSTRICTRC) 3656 iemRegIp16IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT 3657 { 3658 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr; 3659 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3660 { /* likely */ } 3661 else 3662 return iemRaiseGeneralProtectionFault0(pVCpu); 3663 3664 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip); 3665 if (rcStrict == VINF_SUCCESS) 3666 { /* likely */ } 3667 else 3668 return rcStrict; 3669 3670 pVCpu->cpum.GstCtx.rip = uNewRip; 3671 #ifndef IEM_WITH_CODE_TLB 3672 iemOpcodeFlushLight(pVCpu, cbInstr); 3673 #endif 3674 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3675 } 3676 3677 3678 /** 3679 * Implements an 16-bit indirect call. 3680 * 3681 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3682 * @param cbInstr The instruction length. 3683 * @param uNewRip The new RIP value. 3684 */ 3685 DECL_FORCE_INLINE(VBOXSTRICTRC) 3686 iemRegEip32IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT 3687 { 3688 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr; 3689 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3690 { /* likely */ } 3691 else 3692 return iemRaiseGeneralProtectionFault0(pVCpu); 3693 3694 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip); 3695 if (rcStrict == VINF_SUCCESS) 3696 { /* likely */ } 3697 else 3698 return rcStrict; 3699 3700 pVCpu->cpum.GstCtx.rip = uNewRip; 3701 #ifndef IEM_WITH_CODE_TLB 3702 iemOpcodeFlushLight(pVCpu, cbInstr); 3703 #endif 3704 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3705 } 3706 3707 3708 /** 3709 * Implements an 32-bit indirect call, no checking or clearing of 3710 * flags. 3711 * 3712 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3713 * @param cbInstr The instruction length. 3714 * @param uNewRip The new RIP value. 3715 */ 3716 DECL_FORCE_INLINE(VBOXSTRICTRC) 3717 iemRegEip32IndirectCallU32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT 3718 { 3719 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr; 3720 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3721 { /* likely */ } 3722 else 3723 return iemRaiseGeneralProtectionFault0(pVCpu); 3724 3725 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip); 3726 if (rcStrict == VINF_SUCCESS) 3727 { /* likely */ } 3728 else 3729 return rcStrict; 3730 3731 pVCpu->cpum.GstCtx.rip = uNewRip; 3732 #ifndef IEM_WITH_CODE_TLB 3733 iemOpcodeFlushLight(pVCpu, cbInstr); 3734 #endif 3735 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3736 } 3737 3738 3739 /** 3740 * Implements an 32-bit indirect call. 3741 * 3742 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3743 * @param cbInstr The instruction length. 3744 * @param uNewRip The new RIP value. 3745 */ 3746 DECL_FORCE_INLINE(VBOXSTRICTRC) 3747 iemRegEip32IndirectCallU32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT 3748 { 3749 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr; 3750 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit) 3751 { /* likely */ } 3752 else 3753 return iemRaiseGeneralProtectionFault0(pVCpu); 3754 3755 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip); 3756 if (rcStrict == VINF_SUCCESS) 3757 { /* likely */ } 3758 else 3759 return rcStrict; 3760 3761 pVCpu->cpum.GstCtx.rip = uNewRip; 3762 #ifndef IEM_WITH_CODE_TLB 3763 iemOpcodeFlushLight(pVCpu, cbInstr); 3764 #endif 3765 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3766 } 3767 3768 3769 /** 3770 * Implements an 64-bit indirect call, no checking or clearing of 3771 * flags. 3772 * 3773 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3774 * @param cbInstr The instruction length. 3775 * @param uNewRip The new RIP value. 3776 */ 3777 DECL_FORCE_INLINE(VBOXSTRICTRC) 3778 iemRegRip64IndirectCallU64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT 3779 { 3780 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr; 3781 if (IEM_IS_CANONICAL(uNewRip)) 3782 { /* likely */ } 3783 else 3784 return iemRaiseGeneralProtectionFault0(pVCpu); 3785 3786 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip); 3787 if (rcStrict == VINF_SUCCESS) 3788 { /* likely */ } 3789 else 3790 return rcStrict; 3791 3792 pVCpu->cpum.GstCtx.rip = uNewRip; 3793 #ifndef IEM_WITH_CODE_TLB 3794 iemOpcodeFlushLight(pVCpu, cbInstr); 3795 #endif 3796 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 3797 } 3798 3799 3800 /** 3801 * Implements an 64-bit indirect call. 3802 * 3803 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3804 * @param cbInstr The instruction length. 3805 * @param uNewRip The new RIP value. 3806 */ 3807 DECL_FORCE_INLINE(VBOXSTRICTRC) 3808 iemRegRip64IndirectCallU64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT 3809 { 3810 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr; 3811 if (IEM_IS_CANONICAL(uNewRip)) 3812 { /* likely */ } 3813 else 3814 return iemRaiseGeneralProtectionFault0(pVCpu); 3815 3816 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip); 3817 if (rcStrict == VINF_SUCCESS) 3818 { /* likely */ } 3819 else 3820 return rcStrict; 3821 3822 pVCpu->cpum.GstCtx.rip = uNewRip; 3823 #ifndef IEM_WITH_CODE_TLB 3824 iemOpcodeFlushLight(pVCpu, cbInstr); 3825 #endif 3826 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 3827 } 3828 3829 3830 3831 /** 3832 * Adds to the stack pointer. 3833 * 3834 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3835 * @param cbToAdd The number of bytes to add (8-bit!). 3836 */ 3837 DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT 3838 { 3839 if (IEM_IS_64BIT_CODE(pVCpu)) 3840 pVCpu->cpum.GstCtx.rsp += cbToAdd; 3841 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3842 pVCpu->cpum.GstCtx.esp += cbToAdd; 3843 else 3844 pVCpu->cpum.GstCtx.sp += cbToAdd; 3845 } 3846 3847 3848 /** 3849 * Subtracts from the stack pointer. 3850 * 3851 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3852 * @param cbToSub The number of bytes to subtract (8-bit!). 3853 */ 3854 DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT 3855 { 3856 if (IEM_IS_64BIT_CODE(pVCpu)) 3857 pVCpu->cpum.GstCtx.rsp -= cbToSub; 3858 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3859 pVCpu->cpum.GstCtx.esp -= cbToSub; 3860 else 3861 pVCpu->cpum.GstCtx.sp -= cbToSub; 3862 } 3863 3864 3865 /** 3866 * Adds to the temporary stack pointer. 3867 * 3868 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3869 * @param pTmpRsp The temporary SP/ESP/RSP to update. 3870 * @param cbToAdd The number of bytes to add (16-bit). 3871 */ 3872 DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT 3873 { 3874 if (IEM_IS_64BIT_CODE(pVCpu)) 3875 pTmpRsp->u += cbToAdd; 3876 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3877 pTmpRsp->DWords.dw0 += cbToAdd; 3878 else 3879 pTmpRsp->Words.w0 += cbToAdd; 3880 } 3881 3882 3883 /** 3884 * Subtracts from the temporary stack pointer. 3885 * 3886 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3887 * @param pTmpRsp The temporary SP/ESP/RSP to update. 3888 * @param cbToSub The number of bytes to subtract. 3889 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is 3890 * expecting that. 3891 */ 3892 DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT 3893 { 3894 if (IEM_IS_64BIT_CODE(pVCpu)) 3895 pTmpRsp->u -= cbToSub; 3896 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3897 pTmpRsp->DWords.dw0 -= cbToSub; 3898 else 3899 pTmpRsp->Words.w0 -= cbToSub; 3900 } 3901 3902 3903 /** 3904 * Calculates the effective stack address for a push of the specified size as 3905 * well as the new RSP value (upper bits may be masked). 3906 * 3907 * @returns Effective stack addressf for the push. 3908 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3909 * @param cbItem The size of the stack item to pop. 3910 * @param puNewRsp Where to return the new RSP value. 3911 */ 3912 DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT 3913 { 3914 RTUINT64U uTmpRsp; 3915 RTGCPTR GCPtrTop; 3916 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp; 3917 3918 if (IEM_IS_64BIT_CODE(pVCpu)) 3919 GCPtrTop = uTmpRsp.u -= cbItem; 3920 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3921 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem; 3922 else 3923 GCPtrTop = uTmpRsp.Words.w0 -= cbItem; 3924 *puNewRsp = uTmpRsp.u; 3925 return GCPtrTop; 3926 } 3927 3928 3929 /** 3930 * Gets the current stack pointer and calculates the value after a pop of the 3931 * specified size. 3932 * 3933 * @returns Current stack pointer. 3934 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3935 * @param cbItem The size of the stack item to pop. 3936 * @param puNewRsp Where to return the new RSP value. 3937 */ 3938 DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT 3939 { 3940 RTUINT64U uTmpRsp; 3941 RTGCPTR GCPtrTop; 3942 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp; 3943 3944 if (IEM_IS_64BIT_CODE(pVCpu)) 3945 { 3946 GCPtrTop = uTmpRsp.u; 3947 uTmpRsp.u += cbItem; 3948 } 3949 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3950 { 3951 GCPtrTop = uTmpRsp.DWords.dw0; 3952 uTmpRsp.DWords.dw0 += cbItem; 3953 } 3954 else 3955 { 3956 GCPtrTop = uTmpRsp.Words.w0; 3957 uTmpRsp.Words.w0 += cbItem; 3958 } 3959 *puNewRsp = uTmpRsp.u; 3960 return GCPtrTop; 3961 } 3962 3963 3964 /** 3965 * Calculates the effective stack address for a push of the specified size as 3966 * well as the new temporary RSP value (upper bits may be masked). 3967 * 3968 * @returns Effective stack addressf for the push. 3969 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3970 * @param pTmpRsp The temporary stack pointer. This is updated. 3971 * @param cbItem The size of the stack item to pop. 3972 */ 3973 DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT 3974 { 3975 RTGCPTR GCPtrTop; 3976 3977 if (IEM_IS_64BIT_CODE(pVCpu)) 3978 GCPtrTop = pTmpRsp->u -= cbItem; 3979 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 3980 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem; 3981 else 3982 GCPtrTop = pTmpRsp->Words.w0 -= cbItem; 3983 return GCPtrTop; 3984 } 3985 3986 3987 /** 3988 * Gets the effective stack address for a pop of the specified size and 3989 * calculates and updates the temporary RSP. 3990 * 3991 * @returns Current stack pointer. 3992 * @param pVCpu The cross context virtual CPU structure of the calling thread. 3993 * @param pTmpRsp The temporary stack pointer. This is updated. 3994 * @param cbItem The size of the stack item to pop. 3995 */ 3996 DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT 3997 { 3998 RTGCPTR GCPtrTop; 3999 if (IEM_IS_64BIT_CODE(pVCpu)) 4000 { 4001 GCPtrTop = pTmpRsp->u; 4002 pTmpRsp->u += cbItem; 4003 } 4004 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) 4005 { 4006 GCPtrTop = pTmpRsp->DWords.dw0; 4007 pTmpRsp->DWords.dw0 += cbItem; 4008 } 4009 else 4010 { 4011 GCPtrTop = pTmpRsp->Words.w0; 4012 pTmpRsp->Words.w0 += cbItem; 4013 } 4014 return GCPtrTop; 4015 } 4016 4017 4018 /** Common body for iemRegRipNearReturnAndFinishClearingRF() 4019 * and iemRegRipNearReturnAndFinishNoFlags(). */ 4020 template<bool a_fWithFlags> 4021 DECL_FORCE_INLINE(VBOXSTRICTRC) 4022 iemRegRipNearReturnCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT 4023 { 4024 /* Fetch the new RIP from the stack. */ 4025 VBOXSTRICTRC rcStrict; 4026 RTUINT64U NewRip; 4027 RTUINT64U NewRsp; 4028 NewRsp.u = pVCpu->cpum.GstCtx.rsp; 4029 switch (enmEffOpSize) 4030 { 4031 case IEMMODE_16BIT: 4032 NewRip.u = 0; 4033 rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp); 4034 break; 4035 case IEMMODE_32BIT: 4036 NewRip.u = 0; 4037 rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp); 4038 break; 4039 case IEMMODE_64BIT: 4040 rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp); 4041 break; 4042 IEM_NOT_REACHED_DEFAULT_CASE_RET(); 4043 } 4044 if (rcStrict != VINF_SUCCESS) 4045 return rcStrict; 4046 4047 /* Check the new ew RIP before loading it. */ 4048 /** @todo Should test this as the intel+amd pseudo code doesn't mention half 4049 * of it. The canonical test is performed here and for call. */ 4050 if (enmEffOpSize != IEMMODE_64BIT) 4051 { 4052 if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit)) 4053 { /* likely */ } 4054 else 4055 { 4056 Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit)); 4057 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); 4058 } 4059 } 4060 else 4061 { 4062 if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u))) 4063 { /* likely */ } 4064 else 4065 { 4066 Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u)); 4067 return iemRaiseNotCanonical(pVCpu); 4068 } 4069 } 4070 4071 /* Apply cbPop */ 4072 if (cbPop) 4073 iemRegAddToRspEx(pVCpu, &NewRsp, cbPop); 4074 4075 /* Commit it. */ 4076 pVCpu->cpum.GstCtx.rip = NewRip.u; 4077 pVCpu->cpum.GstCtx.rsp = NewRsp.u; 4078 4079 /* Flush the prefetch buffer. */ 4080 #ifndef IEM_WITH_CODE_TLB 4081 iemOpcodeFlushLight(pVCpu, cbInstr); 4082 #endif 4083 RT_NOREF(cbInstr); 4084 4085 4086 if (a_fWithFlags) 4087 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS); 4088 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS); 4089 } 4090 4091 4092 /** 4093 * Implements retn and retn imm16. 4094 * 4095 * @param pVCpu The cross context virtual CPU structure of the 4096 * calling thread. 4097 * @param cbInstr The current instruction length. 4098 * @param enmEffOpSize The effective operand size. This is constant. 4099 * @param cbPop The amount of arguments to pop from the stack 4100 * (bytes). This can be constant (zero). 4101 */ 4102 DECL_FORCE_INLINE(VBOXSTRICTRC) 4103 iemRegRipNearReturnAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT 4104 { 4105 return iemRegRipNearReturnCommon<true /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize); 4106 } 4107 4108 4109 /** 4110 * Implements retn and retn imm16, no checking or clearing of 4111 * flags. 4112 * 4113 * @param pVCpu The cross context virtual CPU structure of the 4114 * calling thread. 4115 * @param cbInstr The current instruction length. 4116 * @param enmEffOpSize The effective operand size. This is constant. 4117 * @param cbPop The amount of arguments to pop from the stack 4118 * (bytes). This can be constant (zero). 4119 */ 4120 DECL_FORCE_INLINE(VBOXSTRICTRC) 4121 iemRegRipNearReturnAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT 4122 { 4123 return iemRegRipNearReturnCommon<false /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize); 4124 } 4125 4126 /** @} */ 4127 4128 4129 /** @name FPU access and helpers. 4130 * 4131 * @{ 4132 */ 4133 4134 4135 /** 4136 * Hook for preparing to use the host FPU. 4137 * 4138 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4139 * 4140 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4141 */ 4142 DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT 4143 { 4144 #ifdef IN_RING3 4145 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); 4146 #else 4147 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu); 4148 #endif 4149 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4150 } 4151 4152 4153 /** 4154 * Hook for preparing to use the host FPU for SSE. 4155 * 4156 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4157 * 4158 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4159 */ 4160 DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT 4161 { 4162 iemFpuPrepareUsage(pVCpu); 4163 } 4164 4165 4166 /** 4167 * Hook for preparing to use the host FPU for AVX. 4168 * 4169 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4170 * 4171 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4172 */ 4173 DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT 4174 { 4175 iemFpuPrepareUsage(pVCpu); 4176 } 4177 4178 4179 /** 4180 * Hook for actualizing the guest FPU state before the interpreter reads it. 4181 * 4182 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4183 * 4184 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4185 */ 4186 DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT 4187 { 4188 #ifdef IN_RING3 4189 NOREF(pVCpu); 4190 #else 4191 CPUMRZFpuStateActualizeForRead(pVCpu); 4192 #endif 4193 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4194 } 4195 4196 4197 /** 4198 * Hook for actualizing the guest FPU state before the interpreter changes it. 4199 * 4200 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4201 * 4202 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4203 */ 4204 DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT 4205 { 4206 #ifdef IN_RING3 4207 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); 4208 #else 4209 CPUMRZFpuStateActualizeForChange(pVCpu); 4210 #endif 4211 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4212 } 4213 4214 4215 /** 4216 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read 4217 * only. 4218 * 4219 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4220 * 4221 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4222 */ 4223 DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT 4224 { 4225 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM) 4226 NOREF(pVCpu); 4227 #else 4228 CPUMRZFpuStateActualizeSseForRead(pVCpu); 4229 #endif 4230 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4231 } 4232 4233 4234 /** 4235 * Hook for actualizing the guest XMM0..15 and MXCSR register state for 4236 * read+write. 4237 * 4238 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4239 * 4240 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4241 */ 4242 DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT 4243 { 4244 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM) 4245 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); 4246 #else 4247 CPUMRZFpuStateActualizeForChange(pVCpu); 4248 #endif 4249 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4250 4251 /* Make sure any changes are loaded the next time around. */ 4252 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE; 4253 } 4254 4255 4256 /** 4257 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read 4258 * only. 4259 * 4260 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4261 * 4262 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4263 */ 4264 DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT 4265 { 4266 #ifdef IN_RING3 4267 NOREF(pVCpu); 4268 #else 4269 CPUMRZFpuStateActualizeAvxForRead(pVCpu); 4270 #endif 4271 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4272 } 4273 4274 4275 /** 4276 * Hook for actualizing the guest YMM0..15 and MXCSR register state for 4277 * read+write. 4278 * 4279 * This is necessary in ring-0 and raw-mode context (nop in ring-3). 4280 * 4281 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4282 */ 4283 DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT 4284 { 4285 #ifdef IN_RING3 4286 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); 4287 #else 4288 CPUMRZFpuStateActualizeForChange(pVCpu); 4289 #endif 4290 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); 4291 4292 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */ 4293 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE; 4294 } 4295 4296 4297 /** 4298 * Stores a QNaN value into a FPU register. 4299 * 4300 * @param pReg Pointer to the register. 4301 */ 4302 DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT 4303 { 4304 pReg->au32[0] = UINT32_C(0x00000000); 4305 pReg->au32[1] = UINT32_C(0xc0000000); 4306 pReg->au16[4] = UINT16_C(0xffff); 4307 } 4308 4309 4310 /** 4311 * Updates the FOP, FPU.CS and FPUIP registers, extended version. 4312 * 4313 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4314 * @param pFpuCtx The FPU context. 4315 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode). 4316 */ 4317 DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT 4318 { 4319 Assert(uFpuOpcode != UINT16_MAX); 4320 pFpuCtx->FOP = uFpuOpcode; 4321 /** @todo x87.CS and FPUIP needs to be kept seperately. */ 4322 if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) 4323 { 4324 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled 4325 * happens in real mode here based on the fnsave and fnstenv images. */ 4326 pFpuCtx->CS = 0; 4327 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4); 4328 } 4329 else if (!IEM_IS_LONG_MODE(pVCpu)) 4330 { 4331 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel; 4332 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip; 4333 } 4334 else 4335 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip; 4336 } 4337 4338 4339 /** 4340 * Marks the specified stack register as free (for FFREE). 4341 * 4342 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4343 * @param iStReg The register to free. 4344 */ 4345 DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT 4346 { 4347 Assert(iStReg < 8); 4348 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4349 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; 4350 pFpuCtx->FTW &= ~RT_BIT(iReg); 4351 } 4352 4353 4354 /** 4355 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it. 4356 * 4357 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4358 */ 4359 DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT 4360 { 4361 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4362 uint16_t uFsw = pFpuCtx->FSW; 4363 uint16_t uTop = uFsw & X86_FSW_TOP_MASK; 4364 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK; 4365 uFsw &= ~X86_FSW_TOP_MASK; 4366 uFsw |= uTop; 4367 pFpuCtx->FSW = uFsw; 4368 } 4369 4370 4371 /** 4372 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything. 4373 * 4374 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4375 */ 4376 DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT 4377 { 4378 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4379 uint16_t uFsw = pFpuCtx->FSW; 4380 uint16_t uTop = uFsw & X86_FSW_TOP_MASK; 4381 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK; 4382 uFsw &= ~X86_FSW_TOP_MASK; 4383 uFsw |= uTop; 4384 pFpuCtx->FSW = uFsw; 4385 } 4386 4387 4388 4389 4390 DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT 4391 { 4392 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4393 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; 4394 if (pFpuCtx->FTW & RT_BIT(iReg)) 4395 return VINF_SUCCESS; 4396 return VERR_NOT_FOUND; 4397 } 4398 4399 4400 DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT 4401 { 4402 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4403 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; 4404 if (pFpuCtx->FTW & RT_BIT(iReg)) 4405 { 4406 *ppRef = &pFpuCtx->aRegs[iStReg].r80; 4407 return VINF_SUCCESS; 4408 } 4409 return VERR_NOT_FOUND; 4410 } 4411 4412 4413 DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, 4414 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT 4415 { 4416 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4417 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); 4418 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK; 4419 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK; 4420 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1))) 4421 { 4422 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80; 4423 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80; 4424 return VINF_SUCCESS; 4425 } 4426 return VERR_NOT_FOUND; 4427 } 4428 4429 4430 DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT 4431 { 4432 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; 4433 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); 4434 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK; 4435 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK; 4436 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1))) 4437 { 4438 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80; 4439 return VINF_SUCCESS; 4440 } 4441 return VERR_NOT_FOUND; 4442 } 4443 4444 4445 /** 4446 * Rotates the stack registers when setting new TOS. 4447 * 4448 * @param pFpuCtx The FPU context. 4449 * @param iNewTop New TOS value. 4450 * @remarks We only do this to speed up fxsave/fxrstor which 4451 * arrange the FP registers in stack order. 4452 * MUST be done before writing the new TOS (FSW). 4453 */ 4454 DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT 4455 { 4456 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW); 4457 RTFLOAT80U ar80Temp[8]; 4458 4459 if (iOldTop == iNewTop) 4460 return; 4461 4462 /* Unscrew the stack and get it into 'native' order. */ 4463 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80; 4464 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80; 4465 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80; 4466 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80; 4467 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80; 4468 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80; 4469 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80; 4470 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80; 4471 4472 /* Now rotate the stack to the new position. */ 4473 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK]; 4474 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK]; 4475 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK]; 4476 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK]; 4477 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK]; 4478 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK]; 4479 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK]; 4480 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK]; 4481 } 4482 4483 4484 /** 4485 * Updates the FPU exception status after FCW is changed. 4486 * 4487 * @param pFpuCtx The FPU context. 4488 */ 4489 DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT 4490 { 4491 uint16_t u16Fsw = pFpuCtx->FSW; 4492 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK)) 4493 u16Fsw |= X86_FSW_ES | X86_FSW_B; 4494 else 4495 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B); 4496 pFpuCtx->FSW = u16Fsw; 4497 } 4498 4499 4500 /** 4501 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE. 4502 * 4503 * @returns The full FTW. 4504 * @param pFpuCtx The FPU context. 4505 */ 4506 DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT 4507 { 4508 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW; 4509 uint16_t u16Ftw = 0; 4510 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); 4511 for (unsigned iSt = 0; iSt < 8; iSt++) 4512 { 4513 unsigned const iReg = (iSt + iTop) & 7; 4514 if (!(u8Ftw & RT_BIT(iReg))) 4515 u16Ftw |= 3 << (iReg * 2); /* empty */ 4516 else 4517 { 4518 uint16_t uTag; 4519 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80; 4520 if (pr80Reg->s.uExponent == 0x7fff) 4521 uTag = 2; /* Exponent is all 1's => Special. */ 4522 else if (pr80Reg->s.uExponent == 0x0000) 4523 { 4524 if (pr80Reg->s.uMantissa == 0x0000) 4525 uTag = 1; /* All bits are zero => Zero. */ 4526 else 4527 uTag = 2; /* Must be special. */ 4528 } 4529 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */ 4530 uTag = 0; /* Valid. */ 4531 else 4532 uTag = 2; /* Must be special. */ 4533 4534 u16Ftw |= uTag << (iReg * 2); 4535 } 4536 } 4537 4538 return u16Ftw; 4539 } 4540 4541 4542 /** 4543 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR). 4544 * 4545 * @returns The compressed FTW. 4546 * @param u16FullFtw The full FTW to convert. 4547 */ 4548 DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT 4549 { 4550 uint8_t u8Ftw = 0; 4551 for (unsigned i = 0; i < 8; i++) 4552 { 4553 if ((u16FullFtw & 3) != 3 /*empty*/) 4554 u8Ftw |= RT_BIT(i); 4555 u16FullFtw >>= 2; 4556 } 4557 4558 return u8Ftw; 4559 } 4560 4561 /** @} */ 4562 4563 4564 /** @name Memory access. 4565 * 4566 * @{ 4567 */ 4568 4569 4570 /** 4571 * Checks whether alignment checks are enabled or not. 4572 * 4573 * @returns true if enabled, false if not. 4574 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4575 */ 4576 DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT 4577 { 4578 #if 0 4579 AssertCompile(X86_CR0_AM == X86_EFL_AC); 4580 return IEM_GET_CPL(pVCpu) == 3 4581 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM); 4582 #else 4583 return RT_BOOL(pVCpu->iem.s.fExec & IEM_F_X86_AC); 4584 #endif 4585 } 4586 4587 /** 4588 * Checks if the given segment can be written to, raise the appropriate 4589 * exception if not. 4590 * 4591 * @returns VBox strict status code. 4592 * 4593 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4594 * @param pHid Pointer to the hidden register. 4595 * @param iSegReg The register number. 4596 * @param pu64BaseAddr Where to return the base address to use for the 4597 * segment. (In 64-bit code it may differ from the 4598 * base in the hidden segment.) 4599 */ 4600 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, 4601 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT 4602 { 4603 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4604 4605 if (IEM_IS_64BIT_CODE(pVCpu)) 4606 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base; 4607 else 4608 { 4609 if (!pHid->Attr.n.u1Present) 4610 { 4611 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg); 4612 AssertRelease(uSel == 0); 4613 LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg)); 4614 return iemRaiseGeneralProtectionFault0(pVCpu); 4615 } 4616 4617 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE) 4618 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) ) 4619 && !IEM_IS_64BIT_CODE(pVCpu) ) 4620 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W); 4621 *pu64BaseAddr = pHid->u64Base; 4622 } 4623 return VINF_SUCCESS; 4624 } 4625 4626 4627 /** 4628 * Checks if the given segment can be read from, raise the appropriate 4629 * exception if not. 4630 * 4631 * @returns VBox strict status code. 4632 * 4633 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4634 * @param pHid Pointer to the hidden register. 4635 * @param iSegReg The register number. 4636 * @param pu64BaseAddr Where to return the base address to use for the 4637 * segment. (In 64-bit code it may differ from the 4638 * base in the hidden segment.) 4639 */ 4640 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, 4641 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT 4642 { 4643 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4644 4645 if (IEM_IS_64BIT_CODE(pVCpu)) 4646 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base; 4647 else 4648 { 4649 if (!pHid->Attr.n.u1Present) 4650 { 4651 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg); 4652 AssertRelease(uSel == 0); 4653 LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg)); 4654 return iemRaiseGeneralProtectionFault0(pVCpu); 4655 } 4656 4657 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE) 4658 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R); 4659 *pu64BaseAddr = pHid->u64Base; 4660 } 4661 return VINF_SUCCESS; 4662 } 4663 4664 4665 /** 4666 * Maps a physical page. 4667 * 4668 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr). 4669 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4670 * @param GCPhysMem The physical address. 4671 * @param fAccess The intended access. 4672 * @param ppvMem Where to return the mapping address. 4673 * @param pLock The PGM lock. 4674 */ 4675 DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, 4676 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT 4677 { 4678 #ifdef IEM_LOG_MEMORY_WRITES 4679 if (fAccess & IEM_ACCESS_TYPE_WRITE) 4680 return VERR_PGM_PHYS_TLB_CATCH_ALL; 4681 #endif 4682 4683 /** @todo This API may require some improving later. A private deal with PGM 4684 * regarding locking and unlocking needs to be struct. A couple of TLBs 4685 * living in PGM, but with publicly accessible inlined access methods 4686 * could perhaps be an even better solution. */ 4687 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu, 4688 GCPhysMem, 4689 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE), 4690 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS), 4691 ppvMem, 4692 pLock); 4693 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/ 4694 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc)); 4695 4696 return rc; 4697 } 4698 4699 4700 /** 4701 * Unmap a page previously mapped by iemMemPageMap. 4702 * 4703 * @param pVCpu The cross context virtual CPU structure of the calling thread. 4704 * @param GCPhysMem The physical address. 4705 * @param fAccess The intended access. 4706 * @param pvMem What iemMemPageMap returned. 4707 * @param pLock The PGM lock. 4708 */ 4709 DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, 4710 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT 4711 { 4712 NOREF(pVCpu); 4713 NOREF(GCPhysMem); 4714 NOREF(fAccess); 4715 NOREF(pvMem); 4716 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock); 4717 } 4718 4719 #ifdef IEM_WITH_SETJMP 4720 4721 /** @todo slim this down */ 4722 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, 4723 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP 4724 { 4725 Assert(cbMem >= 1); 4726 Assert(iSegReg < X86_SREG_COUNT); 4727 4728 /* 4729 * 64-bit mode is simpler. 4730 */ 4731 if (IEM_IS_64BIT_CODE(pVCpu)) 4732 { 4733 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX) 4734 { 4735 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4736 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg); 4737 GCPtrMem += pSel->u64Base; 4738 } 4739 4740 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1))) 4741 return GCPtrMem; 4742 iemRaiseGeneralProtectionFault0Jmp(pVCpu); 4743 } 4744 /* 4745 * 16-bit and 32-bit segmentation. 4746 */ 4747 else if (iSegReg != UINT8_MAX) 4748 { 4749 /** @todo Does this apply to segments with 4G-1 limit? */ 4750 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1; 4751 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem)) 4752 { 4753 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4754 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg); 4755 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE 4756 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */ 4757 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */ 4758 | X86_SEL_TYPE_CODE)) 4759 { 4760 case X86DESCATTR_P: /* readonly data, expand up */ 4761 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */ 4762 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */ 4763 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */ 4764 /* expand up */ 4765 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit)) 4766 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; 4767 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n", 4768 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit)); 4769 break; 4770 4771 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */ 4772 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */ 4773 /* expand down */ 4774 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit 4775 && ( pSel->Attr.n.u1DefBig 4776 || GCPtrLast32 <= UINT32_C(0xffff)) )) 4777 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; 4778 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n", 4779 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX)); 4780 break; 4781 4782 default: 4783 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u)); 4784 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R); 4785 break; 4786 } 4787 } 4788 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32)); 4789 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R); 4790 } 4791 /* 4792 * 32-bit flat address. 4793 */ 4794 else 4795 return GCPtrMem; 4796 } 4797 4798 4799 /** @todo slim this down */ 4800 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, 4801 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP 4802 { 4803 Assert(cbMem >= 1); 4804 Assert(iSegReg < X86_SREG_COUNT); 4805 4806 /* 4807 * 64-bit mode is simpler. 4808 */ 4809 if (IEM_IS_64BIT_CODE(pVCpu)) 4810 { 4811 if (iSegReg >= X86_SREG_FS) 4812 { 4813 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4814 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg); 4815 GCPtrMem += pSel->u64Base; 4816 } 4817 4818 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1))) 4819 return GCPtrMem; 4820 } 4821 /* 4822 * 16-bit and 32-bit segmentation. 4823 */ 4824 else 4825 { 4826 Assert(GCPtrMem <= UINT32_MAX); 4827 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); 4828 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg); 4829 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE 4830 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN); 4831 if ( fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE) /* data, expand up */ 4832 /** @todo explore exactly how the CS stuff works in real mode. See also 4833 * http://www.rcollins.org/Productivity/DescriptorCache.html and 4834 * http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */ 4835 || (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. */ /** @todo testcase! */ 4836 { 4837 /* expand up */ 4838 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1; 4839 if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit 4840 && GCPtrLast32 >= (uint32_t)GCPtrMem)) 4841 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; 4842 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W); 4843 } 4844 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */ 4845 { 4846 /* expand down - the uppger boundary is defined by the B bit, not G. */ 4847 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1; 4848 if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit 4849 && (pSel->Attr.n.u1DefBig || GCPtrLast32 <= UINT32_C(0xffff)) 4850 && GCPtrLast32 >= (uint32_t)GCPtrMem)) 4851 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; 4852 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W); 4853 } 4854 else 4855 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W); 4856 } 4857 iemRaiseGeneralProtectionFault0Jmp(pVCpu); 4858 } 4859 4860 #endif /* IEM_WITH_SETJMP */ 4861 4862 /** 4863 * Fakes a long mode stack selector for SS = 0. 4864 * 4865 * @param pDescSs Where to return the fake stack descriptor. 4866 * @param uDpl The DPL we want. 4867 */ 4868 DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT 4869 { 4870 pDescSs->Long.au64[0] = 0; 4871 pDescSs->Long.au64[1] = 0; 4872 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC; 4873 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */ 4874 pDescSs->Long.Gen.u2Dpl = uDpl; 4875 pDescSs->Long.Gen.u1Present = 1; 4876 pDescSs->Long.Gen.u1Long = 1; 4877 } 4878 4879 4880 /* 4881 * Unmap helpers. 4882 */ 4883 4884 #ifdef IEM_WITH_SETJMP 4885 4886 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP 4887 { 4888 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) 4889 if (RT_LIKELY(bMapInfo == 0)) 4890 return; 4891 # endif 4892 iemMemCommitAndUnmapRwSafeJmp(pVCpu, bMapInfo); 4893 } 4894 4895 4896 DECL_INLINE_THROW(void) iemMemCommitAndUnmapAtJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP 4897 { 4898 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) 4899 if (RT_LIKELY(bMapInfo == 0)) 4900 return; 4901 # endif 4902 iemMemCommitAndUnmapAtSafeJmp(pVCpu, bMapInfo); 4903 } 4904 4905 4906 DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP 4907 { 4908 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) 4909 if (RT_LIKELY(bMapInfo == 0)) 4910 return; 4911 # endif 4912 iemMemCommitAndUnmapWoSafeJmp(pVCpu, bMapInfo); 4913 } 4914 4915 4916 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP 4917 { 4918 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) 4919 if (RT_LIKELY(bMapInfo == 0)) 4920 return; 4921 # endif 4922 iemMemCommitAndUnmapRoSafeJmp(pVCpu, bMapInfo); 4923 } 4924 4925 DECLINLINE(void) iemMemRollbackAndUnmapWo(PVMCPUCC pVCpu, uint8_t bMapInfo) RT_NOEXCEPT 4926 { 4927 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) 4928 if (RT_LIKELY(bMapInfo == 0)) 4929 return; 4930 # endif 4931 iemMemRollbackAndUnmapWoSafe(pVCpu, bMapInfo); 4932 } 4933 4934 #endif /* IEM_WITH_SETJMP */ 4935 4936 4937 /* 4938 * Instantiate R/W inline templates. 4939 */ 4940 4941 /** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK 4942 * Used to check if an unaligned access is if within the page and won't 4943 * trigger an \#AC. 4944 * 4945 * This can also be used to deal with misaligned accesses on platforms that are 4946 * senstive to such if desires. 4947 */ 4948 #if 1 4949 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \ 4950 ( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \ 4951 && !((a_pVCpu)->iem.s.fExec & IEM_F_X86_AC) ) 4952 #else 4953 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0 4954 #endif 4955 4956 #define TMPL_MEM_WITH_ATOMIC_MAPPING 4957 4958 #define TMPL_MEM_TYPE uint8_t 4959 #define TMPL_MEM_TYPE_ALIGN 0 4960 #define TMPL_MEM_TYPE_SIZE 1 4961 #define TMPL_MEM_FN_SUFF U8 4962 #define TMPL_MEM_FMT_TYPE "%#04x" 4963 #define TMPL_MEM_FMT_DESC "byte" 4964 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 4965 4966 #define TMPL_MEM_WITH_STACK 4967 4968 #define TMPL_MEM_TYPE uint16_t 4969 #define TMPL_MEM_TYPE_ALIGN 1 4970 #define TMPL_MEM_TYPE_SIZE 2 4971 #define TMPL_MEM_FN_SUFF U16 4972 #define TMPL_MEM_FMT_TYPE "%#06x" 4973 #define TMPL_MEM_FMT_DESC "word" 4974 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 4975 4976 #define TMPL_WITH_PUSH_SREG 4977 #define TMPL_MEM_TYPE uint32_t 4978 #define TMPL_MEM_TYPE_ALIGN 3 4979 #define TMPL_MEM_TYPE_SIZE 4 4980 #define TMPL_MEM_FN_SUFF U32 4981 #define TMPL_MEM_FMT_TYPE "%#010x" 4982 #define TMPL_MEM_FMT_DESC "dword" 4983 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 4984 #undef TMPL_WITH_PUSH_SREG 4985 4986 #define TMPL_MEM_TYPE uint64_t 4987 #define TMPL_MEM_TYPE_ALIGN 7 4988 #define TMPL_MEM_TYPE_SIZE 8 4989 #define TMPL_MEM_FN_SUFF U64 4990 #define TMPL_MEM_FMT_TYPE "%#018RX64" 4991 #define TMPL_MEM_FMT_DESC "qword" 4992 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 4993 4994 #undef TMPL_MEM_WITH_STACK 4995 #undef TMPL_MEM_WITH_ATOMIC_MAPPING 4996 4997 #define TMPL_MEM_NO_MAPPING /* currently sticky */ 4998 4999 #define TMPL_MEM_NO_STORE 5000 #define TMPL_MEM_TYPE uint32_t 5001 #define TMPL_MEM_TYPE_ALIGN 0 5002 #define TMPL_MEM_TYPE_SIZE 4 5003 #define TMPL_MEM_FN_SUFF U32NoAc 5004 #define TMPL_MEM_FMT_TYPE "%#010x" 5005 #define TMPL_MEM_FMT_DESC "dword" 5006 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5007 5008 #define TMPL_MEM_NO_STORE 5009 #define TMPL_MEM_TYPE uint64_t 5010 #define TMPL_MEM_TYPE_ALIGN 0 5011 #define TMPL_MEM_TYPE_SIZE 8 5012 #define TMPL_MEM_FN_SUFF U64NoAc 5013 #define TMPL_MEM_FMT_TYPE "%#018RX64" 5014 #define TMPL_MEM_FMT_DESC "qword" 5015 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5016 5017 #define TMPL_MEM_NO_STORE 5018 #define TMPL_MEM_TYPE uint64_t 5019 #define TMPL_MEM_TYPE_ALIGN 15 5020 #define TMPL_MEM_TYPE_SIZE 8 5021 #define TMPL_MEM_FN_SUFF U64AlignedU128 5022 #define TMPL_MEM_FMT_TYPE "%#018RX64" 5023 #define TMPL_MEM_FMT_DESC "qword" 5024 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5025 5026 #undef TMPL_MEM_NO_MAPPING 5027 5028 #define TMPL_MEM_TYPE RTFLOAT80U 5029 #define TMPL_MEM_TYPE_ALIGN 7 5030 #define TMPL_MEM_TYPE_SIZE 10 5031 #define TMPL_MEM_FN_SUFF R80 5032 #define TMPL_MEM_FMT_TYPE "%.10Rhxs" 5033 #define TMPL_MEM_FMT_DESC "tword" 5034 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5035 5036 #define TMPL_MEM_TYPE RTPBCD80U 5037 #define TMPL_MEM_TYPE_ALIGN 7 /** @todo RTPBCD80U alignment testcase */ 5038 #define TMPL_MEM_TYPE_SIZE 10 5039 #define TMPL_MEM_FN_SUFF D80 5040 #define TMPL_MEM_FMT_TYPE "%.10Rhxs" 5041 #define TMPL_MEM_FMT_DESC "tword" 5042 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5043 5044 #define TMPL_MEM_WITH_ATOMIC_MAPPING 5045 #define TMPL_MEM_TYPE RTUINT128U 5046 #define TMPL_MEM_TYPE_ALIGN 15 5047 #define TMPL_MEM_TYPE_SIZE 16 5048 #define TMPL_MEM_FN_SUFF U128 5049 #define TMPL_MEM_FMT_TYPE "%.16Rhxs" 5050 #define TMPL_MEM_FMT_DESC "dqword" 5051 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5052 #undef TMPL_MEM_WITH_ATOMIC_MAPPING 5053 5054 #define TMPL_MEM_NO_MAPPING 5055 #define TMPL_MEM_TYPE RTUINT128U 5056 #define TMPL_MEM_TYPE_ALIGN 0 5057 #define TMPL_MEM_TYPE_SIZE 16 5058 #define TMPL_MEM_FN_SUFF U128NoAc 5059 #define TMPL_MEM_FMT_TYPE "%.16Rhxs" 5060 #define TMPL_MEM_FMT_DESC "dqword" 5061 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5062 #undef TMPL_MEM_NO_MAPPING 5063 5064 5065 /* Every template relying on unaligned accesses inside a page not being okay should go below. */ 5066 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK 5067 #define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0 5068 5069 #define TMPL_MEM_NO_MAPPING 5070 #define TMPL_MEM_TYPE RTUINT128U 5071 #define TMPL_MEM_TYPE_ALIGN 15 5072 #define TMPL_MEM_TYPE_SIZE 16 5073 #define TMPL_MEM_FN_SUFF U128AlignedSse 5074 #define TMPL_MEM_FMT_TYPE "%.16Rhxs" 5075 #define TMPL_MEM_FMT_DESC "dqword" 5076 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5077 #undef TMPL_MEM_NO_MAPPING 5078 5079 #define TMPL_MEM_NO_MAPPING 5080 #define TMPL_MEM_TYPE RTUINT256U 5081 #define TMPL_MEM_TYPE_ALIGN 0 5082 #define TMPL_MEM_TYPE_SIZE 32 5083 #define TMPL_MEM_FN_SUFF U256NoAc 5084 #define TMPL_MEM_FMT_TYPE "%.32Rhxs" 5085 #define TMPL_MEM_FMT_DESC "qqword" 5086 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5087 #undef TMPL_MEM_NO_MAPPING 5088 5089 #define TMPL_MEM_NO_MAPPING 5090 #define TMPL_MEM_TYPE RTUINT256U 5091 #define TMPL_MEM_TYPE_ALIGN 31 5092 #define TMPL_MEM_TYPE_SIZE 32 5093 #define TMPL_MEM_FN_SUFF U256AlignedAvx 5094 #define TMPL_MEM_FMT_TYPE "%.32Rhxs" 5095 #define TMPL_MEM_FMT_DESC "qqword" 5096 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h" 5097 #undef TMPL_MEM_NO_MAPPING 5098 5099 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK 5100 5101 /** @} */ 5102 5103 5104 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX 5105 5106 /** 5107 * Gets CR0 fixed-0 bits in VMX operation. 5108 * 5109 * We do this rather than fetching what we report to the guest (in 5110 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same 5111 * values regardless of whether unrestricted-guest feature is available on the CPU. 5112 * 5113 * @returns CR0 fixed-0 bits. 5114 * @param pVCpu The cross context virtual CPU structure. 5115 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode 5116 * must be returned. When @c false, the CR0 fixed-0 5117 * bits for VMX root mode is returned. 5118 * 5119 */ 5120 DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT 5121 { 5122 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); 5123 5124 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs; 5125 if ( fVmxNonRootMode 5126 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)) 5127 return VMX_V_CR0_FIXED0_UX; 5128 return VMX_V_CR0_FIXED0; 5129 } 5130 5131 5132 # ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */ 5133 /** 5134 * Sets virtual-APIC write emulation as pending. 5135 * 5136 * @param pVCpu The cross context virtual CPU structure. 5137 * @param offApic The offset in the virtual-APIC page that was written. 5138 */ 5139 DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT 5140 { 5141 Assert(offApic < XAPIC_OFF_END + 4); 5142 5143 /* 5144 * Record the currently updated APIC offset, as we need this later for figuring 5145 * out whether to perform TPR, EOI or self-IPI virtualization as well as well 5146 * as for supplying the exit qualification when causing an APIC-write VM-exit. 5147 */ 5148 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic; 5149 5150 /* 5151 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI 5152 * virtualization or APIC-write emulation). 5153 */ 5154 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) 5155 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE); 5156 } 5157 # endif /* XAPIC_OFF_END */ 5158 5159 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ 5160 5161 #if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) 5162 /** 5163 * Adds an entry to the TLB trace buffer. 5164 * 5165 * @note Don't use directly, only via the IEMTLBTRACE_XXX macros. 5166 */ 5167 DECLINLINE(void) iemTlbTrace(PVMCPU pVCpu, IEMTLBTRACETYPE enmType, uint64_t u64Param, uint64_t u64Param2 = 0, 5168 uint8_t bParam = 0, uint32_t u32Param = 0/*, uint16_t u16Param = 0 */) 5169 { 5170 uint32_t const fMask = RT_BIT_32(pVCpu->iem.s.cTlbTraceEntriesShift) - 1; 5171 PIEMTLBTRACEENTRY const pEntry = &pVCpu->iem.s.paTlbTraceEntries[pVCpu->iem.s.idxTlbTraceEntry++ & fMask]; 5172 pEntry->u64Param = u64Param; 5173 pEntry->u64Param2 = u64Param2; 5174 pEntry->u16Param = 0; //u16Param; 5175 pEntry->u32Param = u32Param; 5176 pEntry->bParam = bParam; 5177 pEntry->enmType = enmType; 5178 pEntry->rip = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base; 5179 } 5180 #endif 5181 5182 #endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */ 1092 #endif /* !VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineDecode_x86_h */ -
trunk/src/VBox/VMM/VMMAll/target-x86/IEMInlineMem-x86.h
r108255 r108260 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - Inlined Functions.3 * IEM - Interpreted Execution Manager - Inlined Memory Functions, x86 target. 4 4 */ 5 5 … … 26 26 */ 27 27 28 #ifndef VMM_INCLUDED_SRC_ include_IEMInline_h29 #define VMM_INCLUDED_SRC_ include_IEMInline_h28 #ifndef VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineMem_x86_h 29 #define VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineMem_x86_h 30 30 #ifndef RT_WITHOUT_PRAGMA_ONCE 31 31 # pragma once 32 32 #endif 33 33 34 #include <VBox/err.h> 35 36 37 /** 38 * Makes status code addjustments (pass up from I/O and access handler) 39 * as well as maintaining statistics. 40 * 41 * @returns Strict VBox status code to pass up. 42 * @param pVCpu The cross context virtual CPU structure of the calling thread. 43 * @param rcStrict The status from executing an instruction. 44 */ 45 DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT 46 { 47 if (rcStrict != VINF_SUCCESS) 48 { 49 /* Deal with the cases that should be treated as VINF_SUCCESS first. */ 50 if ( rcStrict == VINF_IEM_YIELD_PENDING_FF 51 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */ 52 || rcStrict == VINF_VMX_VMEXIT 53 #endif 54 #ifdef VBOX_WITH_NESTED_HWVIRT_SVM 55 || rcStrict == VINF_SVM_VMEXIT 56 #endif 57 ) 58 { 59 rcStrict = pVCpu->iem.s.rcPassUp; 60 if (RT_LIKELY(rcStrict == VINF_SUCCESS)) 61 { /* likely */ } 62 else 63 pVCpu->iem.s.cRetPassUpStatus++; 64 } 65 else if (RT_SUCCESS(rcStrict)) 66 { 67 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST) 68 || rcStrict == VINF_IOM_R3_IOPORT_READ 69 || rcStrict == VINF_IOM_R3_IOPORT_WRITE 70 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE 71 || rcStrict == VINF_IOM_R3_MMIO_READ 72 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE 73 || rcStrict == VINF_IOM_R3_MMIO_WRITE 74 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE 75 || rcStrict == VINF_CPUM_R3_MSR_READ 76 || rcStrict == VINF_CPUM_R3_MSR_WRITE 77 || rcStrict == VINF_EM_RAW_EMULATE_INSTR 78 || rcStrict == VINF_EM_RAW_TO_R3 79 || rcStrict == VINF_EM_TRIPLE_FAULT 80 || rcStrict == VINF_EM_EMULATE_SPLIT_LOCK 81 || rcStrict == VINF_GIM_R3_HYPERCALL 82 /* raw-mode / virt handlers only: */ 83 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT 84 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT 85 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT 86 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT 87 || rcStrict == VINF_SELM_SYNC_GDT 88 || rcStrict == VINF_CSAM_PENDING_ACTION 89 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE 90 /* nested hw.virt codes: */ 91 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE 92 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR 93 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); 94 /** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */ 95 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp; 96 if (rcPassUp == VINF_SUCCESS) 97 pVCpu->iem.s.cRetInfStatuses++; 98 else if ( rcPassUp < VINF_EM_FIRST 99 || rcPassUp > VINF_EM_LAST 100 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict)) 101 { 102 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict))); 103 pVCpu->iem.s.cRetPassUpStatus++; 104 rcStrict = rcPassUp; 105 } 106 else 107 { 108 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict))); 109 pVCpu->iem.s.cRetInfStatuses++; 110 } 111 } 112 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED) 113 pVCpu->iem.s.cRetAspectNotImplemented++; 114 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED) 115 pVCpu->iem.s.cRetInstrNotImplemented++; 116 else 117 pVCpu->iem.s.cRetErrStatuses++; 118 } 119 else 120 { 121 rcStrict = pVCpu->iem.s.rcPassUp; 122 if (rcStrict != VINF_SUCCESS) 123 pVCpu->iem.s.cRetPassUpStatus++; 124 } 125 126 /* Just clear it here as well. */ 127 pVCpu->iem.s.rcPassUp = VINF_SUCCESS; 128 129 return rcStrict; 130 } 131 132 133 /** 134 * Sets the pass up status. 135 * 136 * @returns VINF_SUCCESS. 137 * @param pVCpu The cross context virtual CPU structure of the 138 * calling thread. 139 * @param rcPassUp The pass up status. Must be informational. 140 * VINF_SUCCESS is not allowed. 141 */ 142 DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT 143 { 144 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS); 145 146 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp; 147 if (rcOldPassUp == VINF_SUCCESS) 148 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); 149 /* If both are EM scheduling codes, use EM priority rules. */ 150 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST 151 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST) 152 { 153 if (rcPassUp < rcOldPassUp) 154 { 155 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); 156 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); 157 } 158 else 159 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); 160 } 161 /* Override EM scheduling with specific status code. */ 162 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST) 163 { 164 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); 165 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); 166 } 167 /* Don't override specific status code, first come first served. */ 168 else 169 LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); 170 return VINF_SUCCESS; 171 } 172 173 174 /** 175 * Calculates the IEM_F_X86_AC flags. 176 * 177 * @returns IEM_F_X86_AC or zero 178 * @param pVCpu The cross context virtual CPU structure of the 179 * calling thread. 180 */ 181 DECL_FORCE_INLINE(uint32_t) iemCalcExecAcFlag(PVMCPUCC pVCpu) RT_NOEXCEPT 182 { 183 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS); 184 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); 185 186 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC 187 || (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_AM | X86_CR0_PE)) != (X86_CR0_AM | X86_CR0_PE) 188 || ( !pVCpu->cpum.GstCtx.eflags.Bits.u1VM 189 && pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl != 3)) 190 return 0; 191 return IEM_F_X86_AC; 192 } 193 194 195 /** 196 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag. 197 * 198 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will 199 * reject expand down data segments and conforming code segments. 200 * 201 * ASSUMES that the CPU is in 32-bit mode. 202 * 203 * @note Will return zero when if any of the segment register state is marked 204 * external, this must be factored into assertions checking fExec 205 * consistency. 206 * 207 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero. 208 * @param pVCpu The cross context virtual CPU structure of the 209 * calling thread. 210 * @sa iemCalc32BitFlatIndicatorEsDs 211 */ 212 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT 213 { 214 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF); 215 return ( ( pVCpu->cpum.GstCtx.es.Attr.u 216 | pVCpu->cpum.GstCtx.cs.Attr.u 217 | pVCpu->cpum.GstCtx.ss.Attr.u 218 | pVCpu->cpum.GstCtx.ds.Attr.u) 219 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE)) 220 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P) 221 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1) 222 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1) 223 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1) 224 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1)) 225 == 0 226 && ( pVCpu->cpum.GstCtx.es.u64Base 227 | pVCpu->cpum.GstCtx.cs.u64Base 228 | pVCpu->cpum.GstCtx.ss.u64Base 229 | pVCpu->cpum.GstCtx.ds.u64Base) 230 == 0 231 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES)) 232 ? IEM_F_MODE_X86_32BIT_FLAT : 0; 233 } 234 235 236 /** 237 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are 238 * flat already. 239 * 240 * This is used by sysenter. 241 * 242 * @note Will return zero when if any of the segment register state is marked 243 * external, this must be factored into assertions checking fExec 244 * consistency. 245 * 246 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero. 247 * @param pVCpu The cross context virtual CPU structure of the 248 * calling thread. 249 * @sa iemCalc32BitFlatIndicator 250 */ 251 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT 252 { 253 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF); 254 return ( ( pVCpu->cpum.GstCtx.es.Attr.u 255 | pVCpu->cpum.GstCtx.ds.Attr.u) 256 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE)) 257 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P) 258 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1) 259 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1)) 260 == 0 261 && ( pVCpu->cpum.GstCtx.es.u64Base 262 | pVCpu->cpum.GstCtx.ds.u64Base) 263 == 0 264 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES)) 265 ? IEM_F_MODE_X86_32BIT_FLAT : 0; 266 } 267 268 269 /** 270 * Calculates the IEM_F_MODE_XXX, CPL and AC flags. 271 * 272 * @returns IEM_F_MODE_XXX, IEM_F_X86_CPL_MASK and IEM_F_X86_AC. 273 * @param pVCpu The cross context virtual CPU structure of the 274 * calling thread. 275 */ 276 DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT 277 { 278 /* 279 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx 280 * here to try get this done as efficiently as possible. 281 */ 282 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS); 283 284 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) 285 { 286 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM) 287 { 288 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); 289 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT); 290 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC 291 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM) 292 || fExec != (3U << IEM_F_X86_CPL_SHIFT)) 293 { /* likely */ } 294 else 295 fExec |= IEM_F_X86_AC; 296 297 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) 298 { 299 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)); 300 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu); 301 } 302 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long 303 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)) 304 fExec |= IEM_F_MODE_X86_64BIT; 305 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) 306 fExec |= IEM_F_MODE_X86_16BIT_PROT; 307 else 308 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386; 309 return fExec; 310 } 311 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC 312 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)) 313 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT); 314 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT) | IEM_F_X86_AC; 315 } 316 317 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */ 318 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)) 319 { 320 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) 321 return IEM_F_MODE_X86_16BIT; 322 return IEM_F_MODE_X86_16BIT_PRE_386; 323 } 324 325 /* 32-bit unreal mode. */ 326 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu); 327 } 328 329 330 /** 331 * Calculates the AMD-V and VT-x related context flags. 332 * 333 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and 334 * IEM_F_X86_CTX_VMX. 335 * @param pVCpu The cross context virtual CPU structure of the 336 * calling thread. 337 */ 338 DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT 339 { 340 /* 341 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled 342 * and CPUMIsGuestInNestedHwvirtMode to some extent. 343 */ 344 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); 345 346 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME); 347 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE) 348 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME); 349 if (RT_LIKELY(!fTmp)) 350 return 0; /* likely */ 351 352 if (fTmp & X86_CR4_VMXE) 353 { 354 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX); 355 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode) 356 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST; 357 return IEM_F_X86_CTX_VMX; 358 } 359 360 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM); 361 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN) 362 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST; 363 return IEM_F_X86_CTX_SVM; 364 } 365 366 #ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */ 367 368 /** 369 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags. 370 * 371 * @returns IEM_F_BRK_PENDING_XXX or zero. 372 * @param pVCpu The cross context virtual CPU structure of the 373 * calling thread. 374 */ 375 DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT 376 { 377 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7); 378 379 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK) 380 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0)) 381 return 0; 382 return iemCalcExecDbgFlagsSlow(pVCpu); 383 } 384 385 /** 386 * Calculates the the IEM_F_XXX flags. 387 * 388 * @returns IEM_F_XXX combination match the current CPU state. 389 * @param pVCpu The cross context virtual CPU structure of the 390 * calling thread. 391 */ 392 DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT 393 { 394 return iemCalcExecModeAndCplFlags(pVCpu) 395 | iemCalcExecHwVirtFlags(pVCpu) 396 /* SMM is not yet implemented */ 397 | iemCalcExecDbgFlags(pVCpu) 398 ; 399 } 400 401 402 /** 403 * Re-calculates the MODE and CPL parts of IEMCPU::fExec. 404 * 405 * @param pVCpu The cross context virtual CPU structure of the 406 * calling thread. 407 */ 408 DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplAndAcFlags(PVMCPUCC pVCpu) 409 { 410 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK | IEM_F_X86_AC)) 411 | iemCalcExecModeAndCplFlags(pVCpu); 412 } 413 414 415 /** 416 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec. 417 * 418 * @param pVCpu The cross context virtual CPU structure of the 419 * calling thread. 420 */ 421 DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu) 422 { 423 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK) 424 | iemCalcExecDbgFlags(pVCpu); 425 } 426 427 #endif /* VBOX_INCLUDED_vmm_dbgf_h */ 428 429 430 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 431 432 # if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */ 433 434 /** 435 * Initializes the execution state. 436 * 437 * @param pVCpu The cross context virtual CPU structure of the 438 * calling thread. 439 * @param fExecOpts Optional execution flags: 440 * - IEM_F_BYPASS_HANDLERS 441 * - IEM_F_X86_DISREGARD_LOCK 442 * 443 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal 444 * side-effects in strict builds. 445 */ 446 DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT 447 { 448 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK); 449 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); 450 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); 451 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); 452 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es)); 453 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds)); 454 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs)); 455 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs)); 456 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr)); 457 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr)); 458 459 pVCpu->iem.s.rcPassUp = VINF_SUCCESS; 460 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts; 461 pVCpu->iem.s.cActiveMappings = 0; 462 pVCpu->iem.s.iNextMapping = 0; 463 464 # ifdef VBOX_STRICT 465 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe; 466 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe; 467 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe; 468 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe; 469 pVCpu->iem.s.fPrefixes = 0xfeedbeef; 470 pVCpu->iem.s.uRexReg = 127; 471 pVCpu->iem.s.uRexB = 127; 472 pVCpu->iem.s.offModRm = 127; 473 pVCpu->iem.s.uRexIndex = 127; 474 pVCpu->iem.s.iEffSeg = 127; 475 pVCpu->iem.s.idxPrefix = 127; 476 pVCpu->iem.s.uVex3rdReg = 127; 477 pVCpu->iem.s.uVexLength = 127; 478 pVCpu->iem.s.fEvexStuff = 127; 479 pVCpu->iem.s.uFpuOpcode = UINT16_MAX; 480 # ifdef IEM_WITH_CODE_TLB 481 pVCpu->iem.s.offInstrNextByte = UINT16_MAX; 482 pVCpu->iem.s.pbInstrBuf = NULL; 483 pVCpu->iem.s.cbInstrBuf = UINT16_MAX; 484 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX; 485 pVCpu->iem.s.offCurInstrStart = INT16_MAX; 486 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff); 487 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 488 pVCpu->iem.s.offOpcode = 127; 489 # endif 490 # else 491 pVCpu->iem.s.offOpcode = 127; 492 pVCpu->iem.s.cbOpcode = 127; 493 # endif 494 # endif /* VBOX_STRICT */ 495 } 496 497 498 # if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) 499 /** 500 * Performs a minimal reinitialization of the execution state. 501 * 502 * This is intended to be used by VM-exits, SMM, LOADALL and other similar 503 * 'world-switch' types operations on the CPU. Currently only nested 504 * hardware-virtualization uses it. 505 * 506 * @param pVCpu The cross context virtual CPU structure of the calling EMT. 507 * @param cbInstr The instruction length (for flushing). 508 */ 509 DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT 510 { 511 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); 512 iemOpcodeFlushHeavy(pVCpu, cbInstr); 513 } 514 # endif 515 516 # endif /* VBOX_INCLUDED_vmm_dbgf_h || DOXYGEN_RUNNING */ 517 518 /** 519 * Counterpart to #iemInitExec that undoes evil strict-build stuff. 520 * 521 * @param pVCpu The cross context virtual CPU structure of the 522 * calling thread. 523 */ 524 DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT 525 { 526 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */ 527 # ifdef VBOX_STRICT 528 # ifdef IEM_WITH_CODE_TLB 529 NOREF(pVCpu); 530 # else 531 pVCpu->iem.s.cbOpcode = 0; 532 # endif 533 # else 534 NOREF(pVCpu); 535 # endif 536 } 537 538 539 /** 540 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter. 541 * 542 * Only calling iemRCRawMaybeReenter in raw-mode, obviously. 543 * 544 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller. 545 * @param pVCpu The cross context virtual CPU structure of the calling thread. 546 * @param rcStrict The status code to fiddle. 547 */ 548 DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT 549 { 550 iemUninitExec(pVCpu); 551 return iemExecStatusCodeFiddling(pVCpu, rcStrict); 552 } 553 554 555 /** 556 * Macro used by the IEMExec* method to check the given instruction length. 557 * 558 * Will return on failure! 559 * 560 * @param a_cbInstr The given instruction length. 561 * @param a_cbMin The minimum length. 562 */ 563 # define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \ 564 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \ 565 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH) 566 567 568 # ifndef IEM_WITH_SETJMP 569 570 /** 571 * Fetches the first opcode byte. 572 * 573 * @returns Strict VBox status code. 574 * @param pVCpu The cross context virtual CPU structure of the 575 * calling thread. 576 * @param pu8 Where to return the opcode byte. 577 */ 578 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT 579 { 580 /* 581 * Check for hardware instruction breakpoints. 582 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs. 583 */ 584 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR))) 585 { /* likely */ } 586 else 587 { 588 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu, 589 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, 590 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS) 591 || IEM_IS_GUEST_CPU_AMD(pVCpu)); 592 if (RT_LIKELY(rcStrict == VINF_SUCCESS)) 593 { /* likely */ } 594 else 595 { 596 *pu8 = 0xff; /* shut up gcc. sigh */ 597 if (rcStrict == VINF_EM_RAW_GUEST_TRAP) 598 return iemRaiseDebugException(pVCpu); 599 return rcStrict; 600 } 601 } 602 603 /* 604 * Fetch the first opcode byte. 605 */ 606 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 607 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) 608 { 609 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; 610 *pu8 = pVCpu->iem.s.abOpcode[offOpcode]; 611 return VINF_SUCCESS; 612 } 613 return iemOpcodeGetNextU8Slow(pVCpu, pu8); 614 } 615 616 # else /* IEM_WITH_SETJMP */ 617 618 /** 619 * Fetches the first opcode byte, longjmp on error. 620 * 621 * @returns The opcode byte. 622 * @param pVCpu The cross context virtual CPU structure of the calling thread. 623 */ 624 DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP 625 { 626 /* 627 * Check for hardware instruction breakpoints. 628 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs. 629 */ 630 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR))) 631 { /* likely */ } 632 else 633 { 634 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu, 635 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, 636 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS) 637 || IEM_IS_GUEST_CPU_AMD(pVCpu)); 638 if (RT_LIKELY(rcStrict == VINF_SUCCESS)) 639 { /* likely */ } 640 else 641 { 642 if (rcStrict == VINF_EM_RAW_GUEST_TRAP) 643 rcStrict = iemRaiseDebugException(pVCpu); 644 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); 645 } 646 } 647 648 /* 649 * Fetch the first opcode byte. 650 */ 651 # ifdef IEM_WITH_CODE_TLB 652 uint8_t bRet; 653 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; 654 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; 655 if (RT_LIKELY( pbBuf != NULL 656 && offBuf < pVCpu->iem.s.cbInstrBuf)) 657 { 658 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1; 659 bRet = pbBuf[offBuf]; 660 } 661 else 662 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu); 663 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 664 Assert(pVCpu->iem.s.offOpcode == 0); 665 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet; 666 # endif 667 return bRet; 668 669 # else /* !IEM_WITH_CODE_TLB */ 670 uintptr_t offOpcode = pVCpu->iem.s.offOpcode; 671 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) 672 { 673 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; 674 return pVCpu->iem.s.abOpcode[offOpcode]; 675 } 676 return iemOpcodeGetNextU8SlowJmp(pVCpu); 677 # endif 678 } 679 680 # endif /* IEM_WITH_SETJMP */ 681 682 /** 683 * Fetches the first opcode byte, returns/throws automatically on failure. 684 * 685 * @param a_pu8 Where to return the opcode byte. 686 * @remark Implicitly references pVCpu. 687 */ 688 # ifndef IEM_WITH_SETJMP 689 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) \ 690 do \ 691 { \ 692 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \ 693 if (rcStrict2 == VINF_SUCCESS) \ 694 { /* likely */ } \ 695 else \ 696 return rcStrict2; \ 697 } while (0) 698 # else 699 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu)) 700 # endif /* IEM_WITH_SETJMP */ 701 702 703 # ifndef IEM_WITH_SETJMP 704 705 /** 706 * Fetches the next opcode byte. 707 * 708 * @returns Strict VBox status code. 709 * @param pVCpu The cross context virtual CPU structure of the 710 * calling thread. 711 * @param pu8 Where to return the opcode byte. 712 */ 713 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT 714 { 715 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 716 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) 717 { 718 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; 719 *pu8 = pVCpu->iem.s.abOpcode[offOpcode]; 720 return VINF_SUCCESS; 721 } 722 return iemOpcodeGetNextU8Slow(pVCpu, pu8); 723 } 724 725 # else /* IEM_WITH_SETJMP */ 726 727 /** 728 * Fetches the next opcode byte, longjmp on error. 729 * 730 * @returns The opcode byte. 731 * @param pVCpu The cross context virtual CPU structure of the calling thread. 732 */ 733 DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP 734 { 735 # ifdef IEM_WITH_CODE_TLB 736 uint8_t bRet; 737 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; 738 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; 739 if (RT_LIKELY( pbBuf != NULL 740 && offBuf < pVCpu->iem.s.cbInstrBuf)) 741 { 742 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1; 743 bRet = pbBuf[offBuf]; 744 } 745 else 746 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu); 747 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 748 Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode)); 749 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet; 750 # endif 751 return bRet; 752 753 # else /* !IEM_WITH_CODE_TLB */ 754 uintptr_t offOpcode = pVCpu->iem.s.offOpcode; 755 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) 756 { 757 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; 758 return pVCpu->iem.s.abOpcode[offOpcode]; 759 } 760 return iemOpcodeGetNextU8SlowJmp(pVCpu); 761 # endif 762 } 763 764 # endif /* IEM_WITH_SETJMP */ 765 766 /** 767 * Fetches the next opcode byte, returns automatically on failure. 768 * 769 * @param a_pu8 Where to return the opcode byte. 770 * @remark Implicitly references pVCpu. 771 */ 772 # ifndef IEM_WITH_SETJMP 773 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) \ 774 do \ 775 { \ 776 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \ 777 if (rcStrict2 == VINF_SUCCESS) \ 778 { /* likely */ } \ 779 else \ 780 return rcStrict2; \ 781 } while (0) 782 # else 783 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu)) 784 # endif /* IEM_WITH_SETJMP */ 785 786 787 # ifndef IEM_WITH_SETJMP 788 /** 789 * Fetches the next signed byte from the opcode stream. 790 * 791 * @returns Strict VBox status code. 792 * @param pVCpu The cross context virtual CPU structure of the calling thread. 793 * @param pi8 Where to return the signed byte. 794 */ 795 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT 796 { 797 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8); 798 } 799 # endif /* !IEM_WITH_SETJMP */ 800 801 802 /** 803 * Fetches the next signed byte from the opcode stream, returning automatically 804 * on failure. 805 * 806 * @param a_pi8 Where to return the signed byte. 807 * @remark Implicitly references pVCpu. 808 */ 809 # ifndef IEM_WITH_SETJMP 810 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) \ 811 do \ 812 { \ 813 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \ 814 if (rcStrict2 != VINF_SUCCESS) \ 815 return rcStrict2; \ 816 } while (0) 817 # else /* IEM_WITH_SETJMP */ 818 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) 819 820 # endif /* IEM_WITH_SETJMP */ 821 822 823 # ifndef IEM_WITH_SETJMP 824 /** 825 * Fetches the next signed byte from the opcode stream, extending it to 826 * unsigned 16-bit. 827 * 828 * @returns Strict VBox status code. 829 * @param pVCpu The cross context virtual CPU structure of the calling thread. 830 * @param pu16 Where to return the unsigned word. 831 */ 832 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT 833 { 834 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 835 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) 836 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16); 837 838 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode]; 839 pVCpu->iem.s.offOpcode = offOpcode + 1; 840 return VINF_SUCCESS; 841 } 842 # endif /* !IEM_WITH_SETJMP */ 843 844 /** 845 * Fetches the next signed byte from the opcode stream and sign-extending it to 846 * a word, returning automatically on failure. 847 * 848 * @param a_pu16 Where to return the word. 849 * @remark Implicitly references pVCpu. 850 */ 851 # ifndef IEM_WITH_SETJMP 852 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \ 853 do \ 854 { \ 855 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \ 856 if (rcStrict2 != VINF_SUCCESS) \ 857 return rcStrict2; \ 858 } while (0) 859 # else 860 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) 861 # endif 862 863 # ifndef IEM_WITH_SETJMP 864 /** 865 * Fetches the next signed byte from the opcode stream, extending it to 866 * unsigned 32-bit. 867 * 868 * @returns Strict VBox status code. 869 * @param pVCpu The cross context virtual CPU structure of the calling thread. 870 * @param pu32 Where to return the unsigned dword. 871 */ 872 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT 873 { 874 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 875 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) 876 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32); 877 878 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode]; 879 pVCpu->iem.s.offOpcode = offOpcode + 1; 880 return VINF_SUCCESS; 881 } 882 # endif /* !IEM_WITH_SETJMP */ 883 884 /** 885 * Fetches the next signed byte from the opcode stream and sign-extending it to 886 * a word, returning automatically on failure. 887 * 888 * @param a_pu32 Where to return the word. 889 * @remark Implicitly references pVCpu. 890 */ 891 # ifndef IEM_WITH_SETJMP 892 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \ 893 do \ 894 { \ 895 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \ 896 if (rcStrict2 != VINF_SUCCESS) \ 897 return rcStrict2; \ 898 } while (0) 899 # else 900 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) 901 # endif 902 903 904 # ifndef IEM_WITH_SETJMP 905 /** 906 * Fetches the next signed byte from the opcode stream, extending it to 907 * unsigned 64-bit. 908 * 909 * @returns Strict VBox status code. 910 * @param pVCpu The cross context virtual CPU structure of the calling thread. 911 * @param pu64 Where to return the unsigned qword. 912 */ 913 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT 914 { 915 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 916 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) 917 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64); 918 919 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode]; 920 pVCpu->iem.s.offOpcode = offOpcode + 1; 921 return VINF_SUCCESS; 922 } 923 # endif /* !IEM_WITH_SETJMP */ 924 925 /** 926 * Fetches the next signed byte from the opcode stream and sign-extending it to 927 * a word, returning automatically on failure. 928 * 929 * @param a_pu64 Where to return the word. 930 * @remark Implicitly references pVCpu. 931 */ 932 # ifndef IEM_WITH_SETJMP 933 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \ 934 do \ 935 { \ 936 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \ 937 if (rcStrict2 != VINF_SUCCESS) \ 938 return rcStrict2; \ 939 } while (0) 940 # else 941 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) 942 # endif 943 944 945 # ifndef IEM_WITH_SETJMP 946 947 /** 948 * Fetches the next opcode word. 949 * 950 * @returns Strict VBox status code. 951 * @param pVCpu The cross context virtual CPU structure of the calling thread. 952 * @param pu16 Where to return the opcode word. 953 */ 954 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT 955 { 956 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 957 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode)) 958 { 959 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2; 960 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 961 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 962 # else 963 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); 964 # endif 965 return VINF_SUCCESS; 966 } 967 return iemOpcodeGetNextU16Slow(pVCpu, pu16); 968 } 969 970 # else /* IEM_WITH_SETJMP */ 971 972 /** 973 * Fetches the next opcode word, longjmp on error. 974 * 975 * @returns The opcode word. 976 * @param pVCpu The cross context virtual CPU structure of the calling thread. 977 */ 978 DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP 979 { 980 # ifdef IEM_WITH_CODE_TLB 981 uint16_t u16Ret; 982 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; 983 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; 984 if (RT_LIKELY( pbBuf != NULL 985 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf)) 986 { 987 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2; 988 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 989 u16Ret = *(uint16_t const *)&pbBuf[offBuf]; 990 # else 991 u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]); 992 # endif 993 } 994 else 995 u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu); 996 997 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 998 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 999 Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode)); 1000 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1001 *(uint16_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret; 1002 # else 1003 pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret); 1004 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret); 1005 # endif 1006 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2; 1007 # endif 1008 1009 return u16Ret; 1010 1011 # else /* !IEM_WITH_CODE_TLB */ 1012 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1013 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode)) 1014 { 1015 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2; 1016 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1017 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 1018 # else 1019 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); 1020 # endif 1021 } 1022 return iemOpcodeGetNextU16SlowJmp(pVCpu); 1023 # endif /* !IEM_WITH_CODE_TLB */ 1024 } 1025 1026 # endif /* IEM_WITH_SETJMP */ 1027 1028 /** 1029 * Fetches the next opcode word, returns automatically on failure. 1030 * 1031 * @param a_pu16 Where to return the opcode word. 1032 * @remark Implicitly references pVCpu. 1033 */ 1034 # ifndef IEM_WITH_SETJMP 1035 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) \ 1036 do \ 1037 { \ 1038 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \ 1039 if (rcStrict2 != VINF_SUCCESS) \ 1040 return rcStrict2; \ 1041 } while (0) 1042 # else 1043 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu)) 1044 # endif 1045 1046 # ifndef IEM_WITH_SETJMP 1047 /** 1048 * Fetches the next opcode word, zero extending it to a double word. 1049 * 1050 * @returns Strict VBox status code. 1051 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1052 * @param pu32 Where to return the opcode double word. 1053 */ 1054 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT 1055 { 1056 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 1057 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode)) 1058 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32); 1059 1060 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); 1061 pVCpu->iem.s.offOpcode = offOpcode + 2; 1062 return VINF_SUCCESS; 1063 } 1064 # endif /* !IEM_WITH_SETJMP */ 1065 1066 /** 1067 * Fetches the next opcode word and zero extends it to a double word, returns 1068 * automatically on failure. 1069 * 1070 * @param a_pu32 Where to return the opcode double word. 1071 * @remark Implicitly references pVCpu. 1072 */ 1073 # ifndef IEM_WITH_SETJMP 1074 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \ 1075 do \ 1076 { \ 1077 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \ 1078 if (rcStrict2 != VINF_SUCCESS) \ 1079 return rcStrict2; \ 1080 } while (0) 1081 # else 1082 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu)) 1083 # endif 1084 1085 # ifndef IEM_WITH_SETJMP 1086 /** 1087 * Fetches the next opcode word, zero extending it to a quad word. 1088 * 1089 * @returns Strict VBox status code. 1090 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1091 * @param pu64 Where to return the opcode quad word. 1092 */ 1093 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT 1094 { 1095 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 1096 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode)) 1097 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64); 1098 1099 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); 1100 pVCpu->iem.s.offOpcode = offOpcode + 2; 1101 return VINF_SUCCESS; 1102 } 1103 # endif /* !IEM_WITH_SETJMP */ 1104 1105 /** 1106 * Fetches the next opcode word and zero extends it to a quad word, returns 1107 * automatically on failure. 1108 * 1109 * @param a_pu64 Where to return the opcode quad word. 1110 * @remark Implicitly references pVCpu. 1111 */ 1112 # ifndef IEM_WITH_SETJMP 1113 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \ 1114 do \ 1115 { \ 1116 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \ 1117 if (rcStrict2 != VINF_SUCCESS) \ 1118 return rcStrict2; \ 1119 } while (0) 1120 # else 1121 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu)) 1122 # endif 1123 1124 1125 # ifndef IEM_WITH_SETJMP 1126 /** 1127 * Fetches the next signed word from the opcode stream. 1128 * 1129 * @returns Strict VBox status code. 1130 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1131 * @param pi16 Where to return the signed word. 1132 */ 1133 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT 1134 { 1135 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16); 1136 } 1137 # endif /* !IEM_WITH_SETJMP */ 1138 1139 1140 /** 1141 * Fetches the next signed word from the opcode stream, returning automatically 1142 * on failure. 1143 * 1144 * @param a_pi16 Where to return the signed word. 1145 * @remark Implicitly references pVCpu. 1146 */ 1147 # ifndef IEM_WITH_SETJMP 1148 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) \ 1149 do \ 1150 { \ 1151 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \ 1152 if (rcStrict2 != VINF_SUCCESS) \ 1153 return rcStrict2; \ 1154 } while (0) 1155 # else 1156 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu)) 1157 # endif 1158 1159 # ifndef IEM_WITH_SETJMP 1160 1161 /** 1162 * Fetches the next opcode dword. 1163 * 1164 * @returns Strict VBox status code. 1165 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1166 * @param pu32 Where to return the opcode double word. 1167 */ 1168 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT 1169 { 1170 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1171 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode)) 1172 { 1173 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4; 1174 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1175 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 1176 # else 1177 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1178 pVCpu->iem.s.abOpcode[offOpcode + 1], 1179 pVCpu->iem.s.abOpcode[offOpcode + 2], 1180 pVCpu->iem.s.abOpcode[offOpcode + 3]); 1181 # endif 1182 return VINF_SUCCESS; 1183 } 1184 return iemOpcodeGetNextU32Slow(pVCpu, pu32); 1185 } 1186 1187 # else /* IEM_WITH_SETJMP */ 1188 1189 /** 1190 * Fetches the next opcode dword, longjmp on error. 1191 * 1192 * @returns The opcode dword. 1193 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1194 */ 1195 DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP 1196 { 1197 # ifdef IEM_WITH_CODE_TLB 1198 uint32_t u32Ret; 1199 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; 1200 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; 1201 if (RT_LIKELY( pbBuf != NULL 1202 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf)) 1203 { 1204 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4; 1205 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1206 u32Ret = *(uint32_t const *)&pbBuf[offBuf]; 1207 # else 1208 u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf], 1209 pbBuf[offBuf + 1], 1210 pbBuf[offBuf + 2], 1211 pbBuf[offBuf + 3]); 1212 # endif 1213 } 1214 else 1215 u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu); 1216 1217 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 1218 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1219 Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode)); 1220 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1221 *(uint32_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret; 1222 # else 1223 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret); 1224 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret); 1225 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret); 1226 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret); 1227 # endif 1228 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4; 1229 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */ 1230 1231 return u32Ret; 1232 1233 # else /* !IEM_WITH_CODE_TLB */ 1234 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1235 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode)) 1236 { 1237 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4; 1238 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1239 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 1240 # else 1241 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1242 pVCpu->iem.s.abOpcode[offOpcode + 1], 1243 pVCpu->iem.s.abOpcode[offOpcode + 2], 1244 pVCpu->iem.s.abOpcode[offOpcode + 3]); 1245 # endif 1246 } 1247 return iemOpcodeGetNextU32SlowJmp(pVCpu); 1248 # endif 1249 } 1250 1251 # endif /* IEM_WITH_SETJMP */ 1252 1253 /** 1254 * Fetches the next opcode dword, returns automatically on failure. 1255 * 1256 * @param a_pu32 Where to return the opcode dword. 1257 * @remark Implicitly references pVCpu. 1258 */ 1259 # ifndef IEM_WITH_SETJMP 1260 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) \ 1261 do \ 1262 { \ 1263 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \ 1264 if (rcStrict2 != VINF_SUCCESS) \ 1265 return rcStrict2; \ 1266 } while (0) 1267 # else 1268 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu)) 1269 # endif 1270 1271 # ifndef IEM_WITH_SETJMP 1272 /** 1273 * Fetches the next opcode dword, zero extending it to a quad word. 1274 * 1275 * @returns Strict VBox status code. 1276 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1277 * @param pu64 Where to return the opcode quad word. 1278 */ 1279 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT 1280 { 1281 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 1282 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode)) 1283 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64); 1284 1285 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1286 pVCpu->iem.s.abOpcode[offOpcode + 1], 1287 pVCpu->iem.s.abOpcode[offOpcode + 2], 1288 pVCpu->iem.s.abOpcode[offOpcode + 3]); 1289 pVCpu->iem.s.offOpcode = offOpcode + 4; 1290 return VINF_SUCCESS; 1291 } 1292 # endif /* !IEM_WITH_SETJMP */ 1293 1294 /** 1295 * Fetches the next opcode dword and zero extends it to a quad word, returns 1296 * automatically on failure. 1297 * 1298 * @param a_pu64 Where to return the opcode quad word. 1299 * @remark Implicitly references pVCpu. 1300 */ 1301 # ifndef IEM_WITH_SETJMP 1302 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \ 1303 do \ 1304 { \ 1305 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \ 1306 if (rcStrict2 != VINF_SUCCESS) \ 1307 return rcStrict2; \ 1308 } while (0) 1309 # else 1310 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu)) 1311 # endif 1312 1313 1314 # ifndef IEM_WITH_SETJMP 1315 /** 1316 * Fetches the next signed double word from the opcode stream. 1317 * 1318 * @returns Strict VBox status code. 1319 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1320 * @param pi32 Where to return the signed double word. 1321 */ 1322 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT 1323 { 1324 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32); 1325 } 1326 # endif 1327 1328 /** 1329 * Fetches the next signed double word from the opcode stream, returning 1330 * automatically on failure. 1331 * 1332 * @param a_pi32 Where to return the signed double word. 1333 * @remark Implicitly references pVCpu. 1334 */ 1335 # ifndef IEM_WITH_SETJMP 1336 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) \ 1337 do \ 1338 { \ 1339 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \ 1340 if (rcStrict2 != VINF_SUCCESS) \ 1341 return rcStrict2; \ 1342 } while (0) 1343 # else 1344 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu)) 1345 # endif 1346 1347 # ifndef IEM_WITH_SETJMP 1348 /** 1349 * Fetches the next opcode dword, sign extending it into a quad word. 1350 * 1351 * @returns Strict VBox status code. 1352 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1353 * @param pu64 Where to return the opcode quad word. 1354 */ 1355 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT 1356 { 1357 uint8_t const offOpcode = pVCpu->iem.s.offOpcode; 1358 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode)) 1359 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64); 1360 1361 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1362 pVCpu->iem.s.abOpcode[offOpcode + 1], 1363 pVCpu->iem.s.abOpcode[offOpcode + 2], 1364 pVCpu->iem.s.abOpcode[offOpcode + 3]); 1365 *pu64 = (uint64_t)(int64_t)i32; 1366 pVCpu->iem.s.offOpcode = offOpcode + 4; 1367 return VINF_SUCCESS; 1368 } 1369 # endif /* !IEM_WITH_SETJMP */ 1370 1371 /** 1372 * Fetches the next opcode double word and sign extends it to a quad word, 1373 * returns automatically on failure. 1374 * 1375 * @param a_pu64 Where to return the opcode quad word. 1376 * @remark Implicitly references pVCpu. 1377 */ 1378 # ifndef IEM_WITH_SETJMP 1379 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \ 1380 do \ 1381 { \ 1382 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \ 1383 if (rcStrict2 != VINF_SUCCESS) \ 1384 return rcStrict2; \ 1385 } while (0) 1386 # else 1387 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu)) 1388 # endif 1389 1390 # ifndef IEM_WITH_SETJMP 1391 1392 /** 1393 * Fetches the next opcode qword. 1394 * 1395 * @returns Strict VBox status code. 1396 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1397 * @param pu64 Where to return the opcode qword. 1398 */ 1399 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT 1400 { 1401 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1402 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode)) 1403 { 1404 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1405 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 1406 # else 1407 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1408 pVCpu->iem.s.abOpcode[offOpcode + 1], 1409 pVCpu->iem.s.abOpcode[offOpcode + 2], 1410 pVCpu->iem.s.abOpcode[offOpcode + 3], 1411 pVCpu->iem.s.abOpcode[offOpcode + 4], 1412 pVCpu->iem.s.abOpcode[offOpcode + 5], 1413 pVCpu->iem.s.abOpcode[offOpcode + 6], 1414 pVCpu->iem.s.abOpcode[offOpcode + 7]); 1415 # endif 1416 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8; 1417 return VINF_SUCCESS; 1418 } 1419 return iemOpcodeGetNextU64Slow(pVCpu, pu64); 1420 } 1421 1422 # else /* IEM_WITH_SETJMP */ 1423 1424 /** 1425 * Fetches the next opcode qword, longjmp on error. 1426 * 1427 * @returns The opcode qword. 1428 * @param pVCpu The cross context virtual CPU structure of the calling thread. 1429 */ 1430 DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP 1431 { 1432 # ifdef IEM_WITH_CODE_TLB 1433 uint64_t u64Ret; 1434 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; 1435 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; 1436 if (RT_LIKELY( pbBuf != NULL 1437 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf)) 1438 { 1439 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8; 1440 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1441 u64Ret = *(uint64_t const *)&pbBuf[offBuf]; 1442 # else 1443 u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf], 1444 pbBuf[offBuf + 1], 1445 pbBuf[offBuf + 2], 1446 pbBuf[offBuf + 3], 1447 pbBuf[offBuf + 4], 1448 pbBuf[offBuf + 5], 1449 pbBuf[offBuf + 6], 1450 pbBuf[offBuf + 7]); 1451 # endif 1452 } 1453 else 1454 u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu); 1455 1456 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 1457 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1458 Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode)); 1459 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1460 *(uint64_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret; 1461 # else 1462 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret); 1463 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret); 1464 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret); 1465 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret); 1466 pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret); 1467 pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret); 1468 pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret); 1469 pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret); 1470 # endif 1471 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8; 1472 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */ 1473 1474 return u64Ret; 1475 1476 # else /* !IEM_WITH_CODE_TLB */ 1477 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; 1478 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode)) 1479 { 1480 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8; 1481 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS 1482 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; 1483 # else 1484 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], 1485 pVCpu->iem.s.abOpcode[offOpcode + 1], 1486 pVCpu->iem.s.abOpcode[offOpcode + 2], 1487 pVCpu->iem.s.abOpcode[offOpcode + 3], 1488 pVCpu->iem.s.abOpcode[offOpcode + 4], 1489 pVCpu->iem.s.abOpcode[offOpcode + 5], 1490 pVCpu->iem.s.abOpcode[offOpcode + 6], 1491 pVCpu->iem.s.abOpcode[offOpcode + 7]); 1492 # endif 1493 } 1494 return iemOpcodeGetNextU64SlowJmp(pVCpu); 1495 # endif /* !IEM_WITH_CODE_TLB */ 1496 } 1497 1498 # endif /* IEM_WITH_SETJMP */ 1499 1500 /** 1501 * Fetches the next opcode quad word, returns automatically on failure. 1502 * 1503 * @param a_pu64 Where to return the opcode quad word. 1504 * @remark Implicitly references pVCpu. 1505 */ 1506 # ifndef IEM_WITH_SETJMP 1507 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) \ 1508 do \ 1509 { \ 1510 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \ 1511 if (rcStrict2 != VINF_SUCCESS) \ 1512 return rcStrict2; \ 1513 } while (0) 1514 # else 1515 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) ) 1516 # endif 1517 1518 /** 1519 * For fetching the opcode bytes for an ModR/M effective address, but throw 1520 * away the result. 1521 * 1522 * This is used when decoding undefined opcodes and such where we want to avoid 1523 * unnecessary MC blocks. 1524 * 1525 * @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is 1526 * used instead. At least for now... 1527 */ 1528 # ifndef IEM_WITH_SETJMP 1529 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \ 1530 RTGCPTR GCPtrEff; \ 1531 VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \ 1532 if (rcStrict != VINF_SUCCESS) \ 1533 return rcStrict; \ 1534 } while (0) 1535 # else 1536 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \ 1537 (void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \ 1538 } while (0) 1539 # endif 1540 1541 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */ 1542 1543 1544 /** @name Misc Worker Functions. 34 #include <iprt/errcore.h> 35 36 37 38 39 /** @name Memory access. 40 * 1545 41 * @{ 1546 42 */ 1547 1548 /**1549 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or1550 * not (kind of obsolete now).1551 *1552 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1553 */1554 #define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )1555 1556 /**1557 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).1558 *1559 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1560 * @param a_fEfl The new EFLAGS.1561 */1562 #define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)1563 1564 1565 /**1566 * Loads a NULL data selector into a selector register, both the hidden and1567 * visible parts, in protected mode.1568 *1569 * @param pVCpu The cross context virtual CPU structure of the calling thread.1570 * @param pSReg Pointer to the segment register.1571 * @param uRpl The RPL.1572 */1573 DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT1574 {1575 /** @todo Testcase: write a testcase checking what happends when loading a NULL1576 * data selector in protected mode. */1577 pSReg->Sel = uRpl;1578 pSReg->ValidSel = uRpl;1579 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;1580 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))1581 {1582 /* VT-x (Intel 3960x) observed doing something like this. */1583 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);1584 pSReg->u32Limit = UINT32_MAX;1585 pSReg->u64Base = 0;1586 }1587 else1588 {1589 pSReg->Attr.u = X86DESCATTR_UNUSABLE;1590 pSReg->u32Limit = 0;1591 pSReg->u64Base = 0;1592 }1593 }1594 1595 /** @} */1596 1597 1598 /*1599 *1600 * Helpers routines.1601 * Helpers routines.1602 * Helpers routines.1603 *1604 */1605 1606 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1607 1608 /**1609 * Recalculates the effective operand size.1610 *1611 * @param pVCpu The cross context virtual CPU structure of the calling thread.1612 */1613 DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT1614 {1615 switch (IEM_GET_CPU_MODE(pVCpu))1616 {1617 case IEMMODE_16BIT:1618 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;1619 break;1620 case IEMMODE_32BIT:1621 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;1622 break;1623 case IEMMODE_64BIT:1624 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))1625 {1626 case 0:1627 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;1628 break;1629 case IEM_OP_PRF_SIZE_OP:1630 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1631 break;1632 case IEM_OP_PRF_SIZE_REX_W:1633 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:1634 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1635 break;1636 }1637 break;1638 default:1639 AssertFailed();1640 }1641 }1642 1643 1644 /**1645 * Sets the default operand size to 64-bit and recalculates the effective1646 * operand size.1647 *1648 * @param pVCpu The cross context virtual CPU structure of the calling thread.1649 */1650 DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT1651 {1652 Assert(IEM_IS_64BIT_CODE(pVCpu));1653 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1654 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)1655 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1656 else1657 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1658 }1659 1660 1661 /**1662 * Sets the default operand size to 64-bit and recalculates the effective1663 * operand size, with intel ignoring any operand size prefix (AMD respects it).1664 *1665 * This is for the relative jumps.1666 *1667 * @param pVCpu The cross context virtual CPU structure of the calling thread.1668 */1669 DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT1670 {1671 Assert(IEM_IS_64BIT_CODE(pVCpu));1672 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1673 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP1674 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)1675 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1676 else1677 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1678 }1679 1680 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */1681 1682 1683 1684 /** @name Register Access.1685 * @{1686 */1687 1688 /**1689 * Gets a reference (pointer) to the specified hidden segment register.1690 *1691 * @returns Hidden register reference.1692 * @param pVCpu The cross context virtual CPU structure of the calling thread.1693 * @param iSegReg The segment register.1694 */1695 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1696 {1697 Assert(iSegReg < X86_SREG_COUNT);1698 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1699 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];1700 1701 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));1702 return pSReg;1703 }1704 1705 1706 /**1707 * Ensures that the given hidden segment register is up to date.1708 *1709 * @returns Hidden register reference.1710 * @param pVCpu The cross context virtual CPU structure of the calling thread.1711 * @param pSReg The segment register.1712 */1713 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT1714 {1715 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));1716 NOREF(pVCpu);1717 return pSReg;1718 }1719 1720 1721 /**1722 * Gets a reference (pointer) to the specified segment register (the selector1723 * value).1724 *1725 * @returns Pointer to the selector variable.1726 * @param pVCpu The cross context virtual CPU structure of the calling thread.1727 * @param iSegReg The segment register.1728 */1729 DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1730 {1731 Assert(iSegReg < X86_SREG_COUNT);1732 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1733 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;1734 }1735 1736 1737 /**1738 * Fetches the selector value of a segment register.1739 *1740 * @returns The selector value.1741 * @param pVCpu The cross context virtual CPU structure of the calling thread.1742 * @param iSegReg The segment register.1743 */1744 DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1745 {1746 Assert(iSegReg < X86_SREG_COUNT);1747 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1748 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;1749 }1750 1751 1752 /**1753 * Fetches the base address value of a segment register.1754 *1755 * @returns The selector value.1756 * @param pVCpu The cross context virtual CPU structure of the calling thread.1757 * @param iSegReg The segment register.1758 */1759 DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1760 {1761 Assert(iSegReg < X86_SREG_COUNT);1762 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1763 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;1764 }1765 1766 1767 /**1768 * Gets a reference (pointer) to the specified general purpose register.1769 *1770 * @returns Register reference.1771 * @param pVCpu The cross context virtual CPU structure of the calling thread.1772 * @param iReg The general purpose register.1773 */1774 DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1775 {1776 Assert(iReg < 16);1777 return &pVCpu->cpum.GstCtx.aGRegs[iReg];1778 }1779 1780 1781 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1782 /**1783 * Gets a reference (pointer) to the specified 8-bit general purpose register.1784 *1785 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.1786 *1787 * @returns Register reference.1788 * @param pVCpu The cross context virtual CPU structure of the calling thread.1789 * @param iReg The register.1790 */1791 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1792 {1793 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REX | IEM_OP_PRF_VEX)))1794 {1795 Assert(iReg < 16);1796 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;1797 }1798 /* high 8-bit register. */1799 Assert(iReg < 8);1800 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;1801 }1802 #endif1803 1804 1805 /**1806 * Gets a reference (pointer) to the specified 8-bit general purpose register,1807 * alternative version with extended (20) register index.1808 *1809 * @returns Register reference.1810 * @param pVCpu The cross context virtual CPU structure of the calling thread.1811 * @param iRegEx The register. The 16 first are regular ones,1812 * whereas 16 thru 19 maps to AH, CH, DH and BH.1813 */1814 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT1815 {1816 /** @todo This could be done by double indexing on little endian hosts:1817 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */1818 if (iRegEx < 16)1819 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;1820 1821 /* high 8-bit register. */1822 Assert(iRegEx < 20);1823 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;1824 }1825 1826 1827 /**1828 * Gets a reference (pointer) to the specified 16-bit general purpose register.1829 *1830 * @returns Register reference.1831 * @param pVCpu The cross context virtual CPU structure of the calling thread.1832 * @param iReg The register.1833 */1834 DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1835 {1836 Assert(iReg < 16);1837 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;1838 }1839 1840 1841 /**1842 * Gets a reference (pointer) to the specified 32-bit general purpose register.1843 *1844 * @returns Register reference.1845 * @param pVCpu The cross context virtual CPU structure of the calling thread.1846 * @param iReg The register.1847 */1848 DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1849 {1850 Assert(iReg < 16);1851 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1852 }1853 1854 1855 /**1856 * Gets a reference (pointer) to the specified signed 32-bit general purpose register.1857 *1858 * @returns Register reference.1859 * @param pVCpu The cross context virtual CPU structure of the calling thread.1860 * @param iReg The register.1861 */1862 DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1863 {1864 Assert(iReg < 16);1865 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1866 }1867 1868 1869 /**1870 * Gets a reference (pointer) to the specified 64-bit general purpose register.1871 *1872 * @returns Register reference.1873 * @param pVCpu The cross context virtual CPU structure of the calling thread.1874 * @param iReg The register.1875 */1876 DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1877 {1878 Assert(iReg < 64);1879 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1880 }1881 1882 1883 /**1884 * Gets a reference (pointer) to the specified signed 64-bit general purpose register.1885 *1886 * @returns Register reference.1887 * @param pVCpu The cross context virtual CPU structure of the calling thread.1888 * @param iReg The register.1889 */1890 DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1891 {1892 Assert(iReg < 16);1893 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1894 }1895 1896 1897 /**1898 * Gets a reference (pointer) to the specified segment register's base address.1899 *1900 * @returns Segment register base address reference.1901 * @param pVCpu The cross context virtual CPU structure of the calling thread.1902 * @param iSegReg The segment selector.1903 */1904 DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1905 {1906 Assert(iSegReg < X86_SREG_COUNT);1907 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1908 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;1909 }1910 1911 1912 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1913 /**1914 * Fetches the value of a 8-bit general purpose register.1915 *1916 * @returns The register value.1917 * @param pVCpu The cross context virtual CPU structure of the calling thread.1918 * @param iReg The register.1919 */1920 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1921 {1922 return *iemGRegRefU8(pVCpu, iReg);1923 }1924 #endif1925 1926 1927 /**1928 * Fetches the value of a 8-bit general purpose register, alternative version1929 * with extended (20) register index.1930 1931 * @returns The register value.1932 * @param pVCpu The cross context virtual CPU structure of the calling thread.1933 * @param iRegEx The register. The 16 first are regular ones,1934 * whereas 16 thru 19 maps to AH, CH, DH and BH.1935 */1936 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT1937 {1938 return *iemGRegRefU8Ex(pVCpu, iRegEx);1939 }1940 1941 1942 /**1943 * Fetches the value of a 16-bit general purpose register.1944 *1945 * @returns The register value.1946 * @param pVCpu The cross context virtual CPU structure of the calling thread.1947 * @param iReg The register.1948 */1949 DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1950 {1951 Assert(iReg < 16);1952 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;1953 }1954 1955 1956 /**1957 * Fetches the value of a 32-bit general purpose register.1958 *1959 * @returns The register value.1960 * @param pVCpu The cross context virtual CPU structure of the calling thread.1961 * @param iReg The register.1962 */1963 DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1964 {1965 Assert(iReg < 16);1966 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1967 }1968 1969 1970 /**1971 * Fetches the value of a 64-bit general purpose register.1972 *1973 * @returns The register value.1974 * @param pVCpu The cross context virtual CPU structure of the calling thread.1975 * @param iReg The register.1976 */1977 DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1978 {1979 Assert(iReg < 16);1980 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1981 }1982 1983 1984 /**1985 * Stores a 16-bit value to a general purpose register.1986 *1987 * @param pVCpu The cross context virtual CPU structure of the calling thread.1988 * @param iReg The register.1989 * @param uValue The value to store.1990 */1991 DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT1992 {1993 Assert(iReg < 16);1994 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;1995 }1996 1997 1998 /**1999 * Stores a 32-bit value to a general purpose register, implicitly clearing high2000 * values.2001 *2002 * @param pVCpu The cross context virtual CPU structure of the calling thread.2003 * @param iReg The register.2004 * @param uValue The value to store.2005 */2006 DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT2007 {2008 Assert(iReg < 16);2009 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;2010 }2011 2012 2013 /**2014 * Stores a 64-bit value to a general purpose register.2015 *2016 * @param pVCpu The cross context virtual CPU structure of the calling thread.2017 * @param iReg The register.2018 * @param uValue The value to store.2019 */2020 DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT2021 {2022 Assert(iReg < 16);2023 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;2024 }2025 2026 2027 /**2028 * Get the address of the top of the stack.2029 *2030 * @param pVCpu The cross context virtual CPU structure of the calling thread.2031 */2032 DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT2033 {2034 if (IEM_IS_64BIT_CODE(pVCpu))2035 return pVCpu->cpum.GstCtx.rsp;2036 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)2037 return pVCpu->cpum.GstCtx.esp;2038 return pVCpu->cpum.GstCtx.sp;2039 }2040 2041 2042 /**2043 * Updates the RIP/EIP/IP to point to the next instruction.2044 *2045 * @param pVCpu The cross context virtual CPU structure of the calling thread.2046 * @param cbInstr The number of bytes to add.2047 */2048 DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT2049 {2050 /*2051 * Advance RIP.2052 *2053 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,2054 * while in all other modes except LM64 the updates are 32-bit. This means2055 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e.2056 * 4GB and 64KB rollovers, and decide whether anything needs masking.2057 *2058 * See PC wrap around tests in bs3-cpu-weird-1.2059 */2060 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;2061 uint64_t const uRipNext = uRipPrev + cbInstr;2062 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16)))2063 || IEM_IS_64BIT_CODE(pVCpu)))2064 pVCpu->cpum.GstCtx.rip = uRipNext;2065 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)2066 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;2067 else2068 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;2069 }2070 2071 2072 /**2073 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the2074 * following EFLAGS bits are set:2075 * - X86_EFL_RF - clear it.2076 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.2077 * - X86_EFL_TF - generate single step \#DB trap.2078 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not2079 * instruction).2080 *2081 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},2082 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint2083 * takes priority over both NMIs and hardware interrupts. So, neither is2084 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are2085 * either unsupported will be triggered on-top of any \#DB raised here.)2086 *2087 * The RF flag only needs to be cleared here as it only suppresses instruction2088 * breakpoints which are not raised here (happens synchronously during2089 * instruction fetching).2090 *2091 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its2092 * status has no bearing on whether \#DB exceptions are raised.2093 *2094 * @note This must *NOT* be called by the two instructions setting the2095 * CPUMCTX_INHIBIT_SHADOW_SS flag.2096 *2097 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}2098 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching2099 * Stacks}2100 */2101 template<uint32_t const a_fTF = X86_EFL_TF>2102 static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2103 {2104 /*2105 * Normally we're just here to clear RF and/or interrupt shadow bits.2106 */2107 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0))2108 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);2109 else2110 {2111 /*2112 * Raise a #DB or/and DBGF event.2113 */2114 VBOXSTRICTRC rcStrict;2115 if (pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK))2116 {2117 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);2118 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;2119 if (pVCpu->cpum.GstCtx.eflags.uBoth & a_fTF)2120 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS;2121 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)2122 >> CPUMCTX_DBG_HIT_DRX_SHIFT;2123 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",2124 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],2125 pVCpu->cpum.GstCtx.rflags.uBoth));2126 2127 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK);2128 rcStrict = iemRaiseDebugException(pVCpu);2129 2130 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */2131 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))2132 {2133 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;2134 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));2135 }2136 }2137 else2138 {2139 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);2140 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;2141 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));2142 }2143 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;2144 Assert(rcStrict != VINF_SUCCESS);2145 return rcStrict;2146 }2147 return rcNormal;2148 }2149 2150 2151 /**2152 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.2153 *2154 * @param pVCpu The cross context virtual CPU structure of the calling thread.2155 * @param rcNormal VINF_SUCCESS to continue TB.2156 * VINF_IEM_REEXEC_BREAK to force TB exit when2157 * taking the wrong conditional branhc.2158 */2159 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2160 {2161 /*2162 * We assume that most of the time nothing actually needs doing here.2163 */2164 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);2165 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth2166 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))2167 return rcNormal;2168 return iemFinishInstructionWithFlagsSet(pVCpu, rcNormal);2169 }2170 2171 2172 /**2173 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF2174 * and CPUMCTX_INHIBIT_SHADOW.2175 *2176 * @param pVCpu The cross context virtual CPU structure of the calling thread.2177 * @param cbInstr The number of bytes to add.2178 */2179 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT2180 {2181 iemRegAddToRip(pVCpu, cbInstr);2182 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);2183 }2184 2185 2186 /**2187 * Updates the RIP to point to the next instruction and clears EFLAGS.RF2188 * and CPUMCTX_INHIBIT_SHADOW.2189 *2190 * Only called from 64-bit code.2191 *2192 * @param pVCpu The cross context virtual CPU structure of the calling thread.2193 * @param cbInstr The number of bytes to add.2194 * @param rcNormal VINF_SUCCESS to continue TB.2195 * VINF_IEM_REEXEC_BREAK to force TB exit when2196 * taking the wrong conditional branhc.2197 */2198 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2199 {2200 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;2201 return iemRegFinishClearingRF(pVCpu, rcNormal);2202 }2203 2204 2205 /**2206 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and2207 * CPUMCTX_INHIBIT_SHADOW.2208 *2209 * This is never from 64-bit code.2210 *2211 * @param pVCpu The cross context virtual CPU structure of the calling thread.2212 * @param cbInstr The number of bytes to add.2213 * @param rcNormal VINF_SUCCESS to continue TB.2214 * VINF_IEM_REEXEC_BREAK to force TB exit when2215 * taking the wrong conditional branhc.2216 */2217 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2218 {2219 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);2220 return iemRegFinishClearingRF(pVCpu, rcNormal);2221 }2222 2223 2224 /**2225 * Updates the IP to point to the next instruction and clears EFLAGS.RF and2226 * CPUMCTX_INHIBIT_SHADOW.2227 *2228 * This is only ever used from 16-bit code on a pre-386 CPU.2229 *2230 * @param pVCpu The cross context virtual CPU structure of the calling thread.2231 * @param cbInstr The number of bytes to add.2232 * @param rcNormal VINF_SUCCESS to continue TB.2233 * VINF_IEM_REEXEC_BREAK to force TB exit when2234 * taking the wrong conditional branhc.2235 */2236 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2237 {2238 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);2239 return iemRegFinishClearingRF(pVCpu, rcNormal);2240 }2241 2242 2243 /**2244 * Tail method for a finish function that does't clear flags or raise \#DB.2245 *2246 * @param pVCpu The cross context virtual CPU structure of the calling thread.2247 * @param rcNormal VINF_SUCCESS to continue TB.2248 * VINF_IEM_REEXEC_BREAK to force TB exit when2249 * taking the wrong conditional branhc.2250 */2251 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2252 {2253 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);2254 Assert(!( pVCpu->cpum.GstCtx.eflags.uBoth2255 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) );2256 RT_NOREF(pVCpu);2257 return rcNormal;2258 }2259 2260 2261 /**2262 * Updates the RIP to point to the next instruction, but does not need to clear2263 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2264 *2265 * Only called from 64-bit code.2266 *2267 * @param pVCpu The cross context virtual CPU structure of the calling thread.2268 * @param cbInstr The number of bytes to add.2269 * @param rcNormal VINF_SUCCESS to continue TB.2270 * VINF_IEM_REEXEC_BREAK to force TB exit when2271 * taking the wrong conditional branhc.2272 */2273 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2274 {2275 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;2276 return iemRegFinishNoFlags(pVCpu, rcNormal);2277 }2278 2279 2280 /**2281 * Updates the EIP to point to the next instruction, but does not need to clear2282 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2283 *2284 * This is never from 64-bit code.2285 *2286 * @param pVCpu The cross context virtual CPU structure of the calling thread.2287 * @param cbInstr The number of bytes to add.2288 * @param rcNormal VINF_SUCCESS to continue TB.2289 * VINF_IEM_REEXEC_BREAK to force TB exit when2290 * taking the wrong conditional branhc.2291 */2292 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2293 {2294 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);2295 return iemRegFinishNoFlags(pVCpu, rcNormal);2296 }2297 2298 2299 /**2300 * Updates the IP to point to the next instruction, but does not need to clear2301 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2302 *2303 * This is only ever used from 16-bit code on a pre-386 CPU.2304 *2305 * @param pVCpu The cross context virtual CPU structure of the calling thread.2306 * @param cbInstr The number of bytes to add.2307 * @param rcNormal VINF_SUCCESS to continue TB.2308 * VINF_IEM_REEXEC_BREAK to force TB exit when2309 * taking the wrong conditional branhc.2310 *2311 */2312 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2313 {2314 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);2315 return iemRegFinishNoFlags(pVCpu, rcNormal);2316 }2317 2318 2319 /**2320 * Adds a 8-bit signed jump offset to RIP from 64-bit code.2321 *2322 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2323 * segment limit.2324 *2325 * @param pVCpu The cross context virtual CPU structure of the calling thread.2326 * @param cbInstr Instruction size.2327 * @param offNextInstr The offset of the next instruction.2328 * @param enmEffOpSize Effective operand size.2329 * @param rcNormal VINF_SUCCESS to continue TB.2330 * VINF_IEM_REEXEC_BREAK to force TB exit when2331 * taking the wrong conditional branhc.2332 */2333 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2334 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2335 {2336 Assert(IEM_IS_64BIT_CODE(pVCpu));2337 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);2338 2339 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2340 if (enmEffOpSize == IEMMODE_16BIT)2341 uNewRip &= UINT16_MAX;2342 2343 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2344 pVCpu->cpum.GstCtx.rip = uNewRip;2345 else2346 return iemRaiseGeneralProtectionFault0(pVCpu);2347 2348 #ifndef IEM_WITH_CODE_TLB2349 iemOpcodeFlushLight(pVCpu, cbInstr);2350 #endif2351 2352 /*2353 * Clear RF and finish the instruction (maybe raise #DB).2354 */2355 return iemRegFinishClearingRF(pVCpu, rcNormal);2356 }2357 2358 2359 /**2360 * Adds a 8-bit signed jump offset to RIP from 64-bit code when the caller is2361 * sure it stays within the same page.2362 *2363 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2364 * segment limit.2365 *2366 * @param pVCpu The cross context virtual CPU structure of the calling thread.2367 * @param cbInstr Instruction size.2368 * @param offNextInstr The offset of the next instruction.2369 * @param enmEffOpSize Effective operand size.2370 * @param rcNormal VINF_SUCCESS to continue TB.2371 * VINF_IEM_REEXEC_BREAK to force TB exit when2372 * taking the wrong conditional branhc.2373 */2374 DECL_FORCE_INLINE(VBOXSTRICTRC)2375 iemRegRip64RelativeJumpS8IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2376 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2377 {2378 Assert(IEM_IS_64BIT_CODE(pVCpu));2379 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);2380 2381 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2382 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2383 pVCpu->cpum.GstCtx.rip = uNewRip;2384 2385 #ifndef IEM_WITH_CODE_TLB2386 iemOpcodeFlushLight(pVCpu, cbInstr);2387 #endif2388 2389 /*2390 * Clear RF and finish the instruction (maybe raise #DB).2391 */2392 return iemRegFinishClearingRF(pVCpu, rcNormal);2393 }2394 2395 2396 /**2397 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit2398 * code (never 64-bit).2399 *2400 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2401 * segment limit.2402 *2403 * @param pVCpu The cross context virtual CPU structure of the calling thread.2404 * @param cbInstr Instruction size.2405 * @param offNextInstr The offset of the next instruction.2406 * @param enmEffOpSize Effective operand size.2407 * @param rcNormal VINF_SUCCESS to continue TB.2408 * VINF_IEM_REEXEC_BREAK to force TB exit when2409 * taking the wrong conditional branhc.2410 */2411 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2412 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2413 {2414 Assert(!IEM_IS_64BIT_CODE(pVCpu));2415 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2416 2417 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2418 if (enmEffOpSize == IEMMODE_16BIT)2419 uNewEip &= UINT16_MAX;2420 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2421 pVCpu->cpum.GstCtx.rip = uNewEip;2422 else2423 return iemRaiseGeneralProtectionFault0(pVCpu);2424 2425 #ifndef IEM_WITH_CODE_TLB2426 iemOpcodeFlushLight(pVCpu, cbInstr);2427 #endif2428 2429 /*2430 * Clear RF and finish the instruction (maybe raise #DB).2431 */2432 return iemRegFinishClearingRF(pVCpu, rcNormal);2433 }2434 2435 2436 /**2437 * Adds a 8-bit signed jump offset to EIP, on 386 or later from FLAT 32-bit code2438 * (never 64-bit).2439 *2440 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2441 * segment limit.2442 *2443 * @param pVCpu The cross context virtual CPU structure of the calling thread.2444 * @param cbInstr Instruction size.2445 * @param offNextInstr The offset of the next instruction.2446 * @param enmEffOpSize Effective operand size.2447 * @param rcNormal VINF_SUCCESS to continue TB.2448 * VINF_IEM_REEXEC_BREAK to force TB exit when2449 * taking the wrong conditional branhc.2450 */2451 DECL_FORCE_INLINE(VBOXSTRICTRC)2452 iemRegEip32RelativeJumpS8FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2453 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2454 {2455 Assert(!IEM_IS_64BIT_CODE(pVCpu));2456 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2457 2458 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2459 if (enmEffOpSize == IEMMODE_16BIT)2460 uNewEip &= UINT16_MAX;2461 pVCpu->cpum.GstCtx.rip = uNewEip;2462 2463 #ifndef IEM_WITH_CODE_TLB2464 iemOpcodeFlushLight(pVCpu, cbInstr);2465 #endif2466 2467 /*2468 * Clear RF and finish the instruction (maybe raise #DB).2469 */2470 return iemRegFinishClearingRF(pVCpu, rcNormal);2471 }2472 2473 2474 /**2475 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.2476 *2477 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2478 * segment limit.2479 *2480 * @param pVCpu The cross context virtual CPU structure of the calling thread.2481 * @param cbInstr Instruction size.2482 * @param offNextInstr The offset of the next instruction.2483 * @param rcNormal VINF_SUCCESS to continue TB.2484 * VINF_IEM_REEXEC_BREAK to force TB exit when2485 * taking the wrong conditional branhc.2486 */2487 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2488 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT2489 {2490 Assert(!IEM_IS_64BIT_CODE(pVCpu));2491 2492 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;2493 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2494 pVCpu->cpum.GstCtx.rip = uNewIp;2495 else2496 return iemRaiseGeneralProtectionFault0(pVCpu);2497 2498 #ifndef IEM_WITH_CODE_TLB2499 iemOpcodeFlushLight(pVCpu, cbInstr);2500 #endif2501 2502 /*2503 * Clear RF and finish the instruction (maybe raise #DB).2504 */2505 return iemRegFinishClearingRF(pVCpu, rcNormal);2506 }2507 2508 2509 /**2510 * Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or2511 * clearing of flags.2512 *2513 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2514 * segment limit.2515 *2516 * @param pVCpu The cross context virtual CPU structure of the calling thread.2517 * @param cbInstr Instruction size.2518 * @param offNextInstr The offset of the next instruction.2519 * @param enmEffOpSize Effective operand size.2520 * @param rcNormal VINF_SUCCESS to continue TB.2521 * VINF_IEM_REEXEC_BREAK to force TB exit when2522 * taking the wrong conditional branhc.2523 */2524 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2525 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2526 {2527 Assert(IEM_IS_64BIT_CODE(pVCpu));2528 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);2529 2530 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2531 if (enmEffOpSize == IEMMODE_16BIT)2532 uNewRip &= UINT16_MAX;2533 2534 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2535 pVCpu->cpum.GstCtx.rip = uNewRip;2536 else2537 return iemRaiseGeneralProtectionFault0(pVCpu);2538 2539 #ifndef IEM_WITH_CODE_TLB2540 iemOpcodeFlushLight(pVCpu, cbInstr);2541 #endif2542 return iemRegFinishNoFlags(pVCpu, rcNormal);2543 }2544 2545 2546 /**2547 * Adds a 8-bit signed jump offset to RIP from 64-bit code when caller is sure2548 * it stays within the same page, no checking or clearing of flags.2549 *2550 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2551 * segment limit.2552 *2553 * @param pVCpu The cross context virtual CPU structure of the calling thread.2554 * @param cbInstr Instruction size.2555 * @param offNextInstr The offset of the next instruction.2556 * @param enmEffOpSize Effective operand size.2557 * @param rcNormal VINF_SUCCESS to continue TB.2558 * VINF_IEM_REEXEC_BREAK to force TB exit when2559 * taking the wrong conditional branhc.2560 */2561 DECL_FORCE_INLINE(VBOXSTRICTRC)2562 iemRegRip64RelativeJumpS8IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2563 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2564 {2565 Assert(IEM_IS_64BIT_CODE(pVCpu));2566 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);2567 2568 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2569 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2570 pVCpu->cpum.GstCtx.rip = uNewRip;2571 2572 #ifndef IEM_WITH_CODE_TLB2573 iemOpcodeFlushLight(pVCpu, cbInstr);2574 #endif2575 return iemRegFinishNoFlags(pVCpu, rcNormal);2576 }2577 2578 2579 /**2580 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit2581 * code (never 64-bit), no checking or clearing of flags.2582 *2583 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2584 * segment limit.2585 *2586 * @param pVCpu The cross context virtual CPU structure of the calling thread.2587 * @param cbInstr Instruction size.2588 * @param offNextInstr The offset of the next instruction.2589 * @param enmEffOpSize Effective operand size.2590 * @param rcNormal VINF_SUCCESS to continue TB.2591 * VINF_IEM_REEXEC_BREAK to force TB exit when2592 * taking the wrong conditional branhc.2593 */2594 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2595 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2596 {2597 Assert(!IEM_IS_64BIT_CODE(pVCpu));2598 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2599 2600 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2601 if (enmEffOpSize == IEMMODE_16BIT)2602 uNewEip &= UINT16_MAX;2603 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2604 pVCpu->cpum.GstCtx.rip = uNewEip;2605 else2606 return iemRaiseGeneralProtectionFault0(pVCpu);2607 2608 #ifndef IEM_WITH_CODE_TLB2609 iemOpcodeFlushLight(pVCpu, cbInstr);2610 #endif2611 return iemRegFinishNoFlags(pVCpu, rcNormal);2612 }2613 2614 2615 /**2616 * Adds a 8-bit signed jump offset to EIP, on 386 or later from flat 32-bit code2617 * (never 64-bit), no checking or clearing of flags.2618 *2619 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2620 * segment limit.2621 *2622 * @param pVCpu The cross context virtual CPU structure of the calling thread.2623 * @param cbInstr Instruction size.2624 * @param offNextInstr The offset of the next instruction.2625 * @param enmEffOpSize Effective operand size.2626 * @param rcNormal VINF_SUCCESS to continue TB.2627 * VINF_IEM_REEXEC_BREAK to force TB exit when2628 * taking the wrong conditional branhc.2629 */2630 DECL_FORCE_INLINE(VBOXSTRICTRC)2631 iemRegEip32RelativeJumpS8FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2632 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2633 {2634 Assert(!IEM_IS_64BIT_CODE(pVCpu));2635 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2636 2637 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2638 if (enmEffOpSize == IEMMODE_16BIT)2639 uNewEip &= UINT16_MAX;2640 pVCpu->cpum.GstCtx.rip = uNewEip;2641 2642 #ifndef IEM_WITH_CODE_TLB2643 iemOpcodeFlushLight(pVCpu, cbInstr);2644 #endif2645 return iemRegFinishNoFlags(pVCpu, rcNormal);2646 }2647 2648 2649 /**2650 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or2651 * clearing of flags.2652 *2653 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2654 * segment limit.2655 *2656 * @param pVCpu The cross context virtual CPU structure of the calling thread.2657 * @param cbInstr Instruction size.2658 * @param offNextInstr The offset of the next instruction.2659 * @param rcNormal VINF_SUCCESS to continue TB.2660 * VINF_IEM_REEXEC_BREAK to force TB exit when2661 * taking the wrong conditional branhc.2662 */2663 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2664 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT2665 {2666 Assert(!IEM_IS_64BIT_CODE(pVCpu));2667 2668 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;2669 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2670 pVCpu->cpum.GstCtx.rip = uNewIp;2671 else2672 return iemRaiseGeneralProtectionFault0(pVCpu);2673 2674 #ifndef IEM_WITH_CODE_TLB2675 iemOpcodeFlushLight(pVCpu, cbInstr);2676 #endif2677 return iemRegFinishNoFlags(pVCpu, rcNormal);2678 }2679 2680 2681 /**2682 * Adds a 16-bit signed jump offset to RIP from 64-bit code.2683 *2684 * @returns Strict VBox status code.2685 * @param pVCpu The cross context virtual CPU structure of the calling thread.2686 * @param cbInstr Instruction size.2687 * @param offNextInstr The offset of the next instruction.2688 * @param rcNormal VINF_SUCCESS to continue TB.2689 * VINF_IEM_REEXEC_BREAK to force TB exit when2690 * taking the wrong conditional branhc.2691 */2692 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2693 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2694 {2695 Assert(IEM_IS_64BIT_CODE(pVCpu));2696 2697 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);2698 2699 #ifndef IEM_WITH_CODE_TLB2700 iemOpcodeFlushLight(pVCpu, cbInstr);2701 #endif2702 2703 /*2704 * Clear RF and finish the instruction (maybe raise #DB).2705 */2706 return iemRegFinishClearingRF(pVCpu, rcNormal);2707 }2708 2709 2710 /**2711 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.2712 *2713 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2714 * segment limit.2715 *2716 * @returns Strict VBox status code.2717 * @param pVCpu The cross context virtual CPU structure of the calling thread.2718 * @param cbInstr Instruction size.2719 * @param offNextInstr The offset of the next instruction.2720 * @param rcNormal VINF_SUCCESS to continue TB.2721 * VINF_IEM_REEXEC_BREAK to force TB exit when2722 * taking the wrong conditional branhc.2723 *2724 * @note This is also used by 16-bit code in pre-386 mode, as the code is2725 * identical.2726 */2727 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2728 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2729 {2730 Assert(!IEM_IS_64BIT_CODE(pVCpu));2731 2732 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2733 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2734 pVCpu->cpum.GstCtx.rip = uNewIp;2735 else2736 return iemRaiseGeneralProtectionFault0(pVCpu);2737 2738 #ifndef IEM_WITH_CODE_TLB2739 iemOpcodeFlushLight(pVCpu, cbInstr);2740 #endif2741 2742 /*2743 * Clear RF and finish the instruction (maybe raise #DB).2744 */2745 return iemRegFinishClearingRF(pVCpu, rcNormal);2746 }2747 2748 2749 /**2750 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code.2751 *2752 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2753 * segment limit.2754 *2755 * @returns Strict VBox status code.2756 * @param pVCpu The cross context virtual CPU structure of the calling thread.2757 * @param cbInstr Instruction size.2758 * @param offNextInstr The offset of the next instruction.2759 * @param rcNormal VINF_SUCCESS to continue TB.2760 * VINF_IEM_REEXEC_BREAK to force TB exit when2761 * taking the wrong conditional branhc.2762 *2763 * @note This is also used by 16-bit code in pre-386 mode, as the code is2764 * identical.2765 */2766 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2767 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2768 {2769 Assert(!IEM_IS_64BIT_CODE(pVCpu));2770 2771 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2772 pVCpu->cpum.GstCtx.rip = uNewIp;2773 2774 #ifndef IEM_WITH_CODE_TLB2775 iemOpcodeFlushLight(pVCpu, cbInstr);2776 #endif2777 2778 /*2779 * Clear RF and finish the instruction (maybe raise #DB).2780 */2781 return iemRegFinishClearingRF(pVCpu, rcNormal);2782 }2783 2784 2785 /**2786 * Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or2787 * clearing of flags.2788 *2789 * @returns Strict VBox status code.2790 * @param pVCpu The cross context virtual CPU structure of the calling thread.2791 * @param cbInstr Instruction size.2792 * @param offNextInstr The offset of the next instruction.2793 * @param rcNormal VINF_SUCCESS to continue TB.2794 * VINF_IEM_REEXEC_BREAK to force TB exit when2795 * taking the wrong conditional branhc.2796 */2797 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2798 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2799 {2800 Assert(IEM_IS_64BIT_CODE(pVCpu));2801 2802 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);2803 2804 #ifndef IEM_WITH_CODE_TLB2805 iemOpcodeFlushLight(pVCpu, cbInstr);2806 #endif2807 return iemRegFinishNoFlags(pVCpu, rcNormal);2808 }2809 2810 2811 /**2812 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code,2813 * no checking or clearing of flags.2814 *2815 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2816 * segment limit.2817 *2818 * @returns Strict VBox status code.2819 * @param pVCpu The cross context virtual CPU structure of the calling thread.2820 * @param cbInstr Instruction size.2821 * @param offNextInstr The offset of the next instruction.2822 * @param rcNormal VINF_SUCCESS to continue TB.2823 * VINF_IEM_REEXEC_BREAK to force TB exit when2824 * taking the wrong conditional branhc.2825 *2826 * @note This is also used by 16-bit code in pre-386 mode, as the code is2827 * identical.2828 */2829 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2830 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2831 {2832 Assert(!IEM_IS_64BIT_CODE(pVCpu));2833 2834 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2835 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2836 pVCpu->cpum.GstCtx.rip = uNewIp;2837 else2838 return iemRaiseGeneralProtectionFault0(pVCpu);2839 2840 #ifndef IEM_WITH_CODE_TLB2841 iemOpcodeFlushLight(pVCpu, cbInstr);2842 #endif2843 return iemRegFinishNoFlags(pVCpu, rcNormal);2844 }2845 2846 2847 /**2848 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code, no checking or2849 * clearing of flags.2850 *2851 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2852 * segment limit.2853 *2854 * @returns Strict VBox status code.2855 * @param pVCpu The cross context virtual CPU structure of the calling thread.2856 * @param cbInstr Instruction size.2857 * @param offNextInstr The offset of the next instruction.2858 * @param rcNormal VINF_SUCCESS to continue TB.2859 * VINF_IEM_REEXEC_BREAK to force TB exit when2860 * taking the wrong conditional branhc.2861 *2862 * @note This is also used by 16-bit code in pre-386 mode, as the code is2863 * identical.2864 */2865 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2866 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2867 {2868 Assert(!IEM_IS_64BIT_CODE(pVCpu));2869 2870 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2871 pVCpu->cpum.GstCtx.rip = uNewIp;2872 2873 #ifndef IEM_WITH_CODE_TLB2874 iemOpcodeFlushLight(pVCpu, cbInstr);2875 #endif2876 return iemRegFinishNoFlags(pVCpu, rcNormal);2877 }2878 2879 2880 /**2881 * Adds a 32-bit signed jump offset to RIP from 64-bit code.2882 *2883 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2884 * segment limit.2885 *2886 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the2887 * only alternative for relative jumps in 64-bit code and that is already2888 * handled in the decoder stage.2889 *2890 * @returns Strict VBox status code.2891 * @param pVCpu The cross context virtual CPU structure of the calling thread.2892 * @param cbInstr Instruction size.2893 * @param offNextInstr The offset of the next instruction.2894 * @param rcNormal VINF_SUCCESS to continue TB.2895 * VINF_IEM_REEXEC_BREAK to force TB exit when2896 * taking the wrong conditional branhc.2897 */2898 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2899 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2900 {2901 Assert(IEM_IS_64BIT_CODE(pVCpu));2902 2903 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2904 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2905 pVCpu->cpum.GstCtx.rip = uNewRip;2906 else2907 return iemRaiseGeneralProtectionFault0(pVCpu);2908 2909 #ifndef IEM_WITH_CODE_TLB2910 iemOpcodeFlushLight(pVCpu, cbInstr);2911 #endif2912 2913 /*2914 * Clear RF and finish the instruction (maybe raise #DB).2915 */2916 return iemRegFinishClearingRF(pVCpu, rcNormal);2917 }2918 2919 2920 /**2921 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is2922 * sure the target is in the same page.2923 *2924 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2925 * segment limit.2926 *2927 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the2928 * only alternative for relative jumps in 64-bit code and that is already2929 * handled in the decoder stage.2930 *2931 * @returns Strict VBox status code.2932 * @param pVCpu The cross context virtual CPU structure of the calling thread.2933 * @param cbInstr Instruction size.2934 * @param offNextInstr The offset of the next instruction.2935 * @param rcNormal VINF_SUCCESS to continue TB.2936 * VINF_IEM_REEXEC_BREAK to force TB exit when2937 * taking the wrong conditional branhc.2938 */2939 DECL_FORCE_INLINE(VBOXSTRICTRC)2940 iemRegRip64RelativeJumpS32IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2941 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2942 {2943 Assert(IEM_IS_64BIT_CODE(pVCpu));2944 2945 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2946 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2947 pVCpu->cpum.GstCtx.rip = uNewRip;2948 2949 #ifndef IEM_WITH_CODE_TLB2950 iemOpcodeFlushLight(pVCpu, cbInstr);2951 #endif2952 2953 /*2954 * Clear RF and finish the instruction (maybe raise #DB).2955 */2956 return iemRegFinishClearingRF(pVCpu, rcNormal);2957 }2958 2959 2960 /**2961 * Adds a 32-bit signed jump offset to RIP from 64-bit code.2962 *2963 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2964 * segment limit.2965 *2966 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the2967 * only alternative for relative jumps in 32-bit code and that is already2968 * handled in the decoder stage.2969 *2970 * @returns Strict VBox status code.2971 * @param pVCpu The cross context virtual CPU structure of the calling thread.2972 * @param cbInstr Instruction size.2973 * @param offNextInstr The offset of the next instruction.2974 * @param rcNormal VINF_SUCCESS to continue TB.2975 * VINF_IEM_REEXEC_BREAK to force TB exit when2976 * taking the wrong conditional branhc.2977 */2978 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2979 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2980 {2981 Assert(!IEM_IS_64BIT_CODE(pVCpu));2982 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);2983 2984 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;2985 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2986 pVCpu->cpum.GstCtx.rip = uNewEip;2987 else2988 return iemRaiseGeneralProtectionFault0(pVCpu);2989 2990 #ifndef IEM_WITH_CODE_TLB2991 iemOpcodeFlushLight(pVCpu, cbInstr);2992 #endif2993 2994 /*2995 * Clear RF and finish the instruction (maybe raise #DB).2996 */2997 return iemRegFinishClearingRF(pVCpu, rcNormal);2998 }2999 3000 3001 /**3002 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code.3003 *3004 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3005 * segment limit.3006 *3007 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3008 * only alternative for relative jumps in 32-bit code and that is already3009 * handled in the decoder stage.3010 *3011 * @returns Strict VBox status code.3012 * @param pVCpu The cross context virtual CPU structure of the calling thread.3013 * @param cbInstr Instruction size.3014 * @param offNextInstr The offset of the next instruction.3015 * @param rcNormal VINF_SUCCESS to continue TB.3016 * VINF_IEM_REEXEC_BREAK to force TB exit when3017 * taking the wrong conditional branhc.3018 */3019 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,3020 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3021 {3022 Assert(!IEM_IS_64BIT_CODE(pVCpu));3023 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3024 3025 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3026 pVCpu->cpum.GstCtx.rip = uNewEip;3027 3028 #ifndef IEM_WITH_CODE_TLB3029 iemOpcodeFlushLight(pVCpu, cbInstr);3030 #endif3031 3032 /*3033 * Clear RF and finish the instruction (maybe raise #DB).3034 */3035 return iemRegFinishClearingRF(pVCpu, rcNormal);3036 }3037 3038 3039 3040 /**3041 * Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or3042 * clearing of flags.3043 *3044 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3045 * segment limit.3046 *3047 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the3048 * only alternative for relative jumps in 64-bit code and that is already3049 * handled in the decoder stage.3050 *3051 * @returns Strict VBox status code.3052 * @param pVCpu The cross context virtual CPU structure of the calling thread.3053 * @param cbInstr Instruction size.3054 * @param offNextInstr The offset of the next instruction.3055 * @param rcNormal VINF_SUCCESS to continue TB.3056 * VINF_IEM_REEXEC_BREAK to force TB exit when3057 * taking the wrong conditional branhc.3058 */3059 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3060 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3061 {3062 Assert(IEM_IS_64BIT_CODE(pVCpu));3063 3064 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;3065 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3066 pVCpu->cpum.GstCtx.rip = uNewRip;3067 else3068 return iemRaiseGeneralProtectionFault0(pVCpu);3069 3070 #ifndef IEM_WITH_CODE_TLB3071 iemOpcodeFlushLight(pVCpu, cbInstr);3072 #endif3073 return iemRegFinishNoFlags(pVCpu, rcNormal);3074 }3075 3076 3077 /**3078 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is3079 * sure it stays within the same page, no checking or clearing of flags.3080 *3081 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3082 * segment limit.3083 *3084 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the3085 * only alternative for relative jumps in 64-bit code and that is already3086 * handled in the decoder stage.3087 *3088 * @returns Strict VBox status code.3089 * @param pVCpu The cross context virtual CPU structure of the calling thread.3090 * @param cbInstr Instruction size.3091 * @param offNextInstr The offset of the next instruction.3092 * @param rcNormal VINF_SUCCESS to continue TB.3093 * VINF_IEM_REEXEC_BREAK to force TB exit when3094 * taking the wrong conditional branhc.3095 */3096 DECL_FORCE_INLINE(VBOXSTRICTRC)3097 iemRegRip64RelativeJumpS32IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3098 {3099 Assert(IEM_IS_64BIT_CODE(pVCpu));3100 3101 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;3102 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));3103 pVCpu->cpum.GstCtx.rip = uNewRip;3104 3105 #ifndef IEM_WITH_CODE_TLB3106 iemOpcodeFlushLight(pVCpu, cbInstr);3107 #endif3108 return iemRegFinishNoFlags(pVCpu, rcNormal);3109 }3110 3111 3112 /**3113 * Adds a 32-bit signed jump offset to RIP from 32-bit code, no checking or3114 * clearing of flags.3115 *3116 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3117 * segment limit.3118 *3119 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3120 * only alternative for relative jumps in 32-bit code and that is already3121 * handled in the decoder stage.3122 *3123 * @returns Strict VBox status code.3124 * @param pVCpu The cross context virtual CPU structure of the calling thread.3125 * @param cbInstr Instruction size.3126 * @param offNextInstr The offset of the next instruction.3127 * @param rcNormal VINF_SUCCESS to continue TB.3128 * VINF_IEM_REEXEC_BREAK to force TB exit when3129 * taking the wrong conditional branhc.3130 */3131 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3132 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3133 {3134 Assert(!IEM_IS_64BIT_CODE(pVCpu));3135 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3136 3137 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3138 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3139 pVCpu->cpum.GstCtx.rip = uNewEip;3140 else3141 return iemRaiseGeneralProtectionFault0(pVCpu);3142 3143 #ifndef IEM_WITH_CODE_TLB3144 iemOpcodeFlushLight(pVCpu, cbInstr);3145 #endif3146 return iemRegFinishNoFlags(pVCpu, rcNormal);3147 }3148 3149 3150 /**3151 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code, no checking or3152 * clearing of flags.3153 *3154 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3155 * segment limit.3156 *3157 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3158 * only alternative for relative jumps in 32-bit code and that is already3159 * handled in the decoder stage.3160 *3161 * @returns Strict VBox status code.3162 * @param pVCpu The cross context virtual CPU structure of the calling thread.3163 * @param cbInstr Instruction size.3164 * @param offNextInstr The offset of the next instruction.3165 * @param rcNormal VINF_SUCCESS to continue TB.3166 * VINF_IEM_REEXEC_BREAK to force TB exit when3167 * taking the wrong conditional branhc.3168 */3169 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3170 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3171 {3172 Assert(!IEM_IS_64BIT_CODE(pVCpu));3173 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3174 3175 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3176 pVCpu->cpum.GstCtx.rip = uNewEip;3177 3178 #ifndef IEM_WITH_CODE_TLB3179 iemOpcodeFlushLight(pVCpu, cbInstr);3180 #endif3181 return iemRegFinishNoFlags(pVCpu, rcNormal);3182 }3183 3184 3185 /**3186 * Extended version of iemFinishInstructionWithFlagsSet that goes with3187 * iemRegAddToRipAndFinishingClearingRfEx.3188 *3189 * See iemFinishInstructionWithFlagsSet() for details.3190 */3191 static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT3192 {3193 /*3194 * Raise a #DB.3195 */3196 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);3197 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;3198 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS3199 | ( (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)3200 >> CPUMCTX_DBG_HIT_DRX_SHIFT);3201 /** @todo Do we set all pending \#DB events, or just one? */3202 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",3203 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],3204 pVCpu->cpum.GstCtx.rflags.uBoth));3205 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);3206 return iemRaiseDebugException(pVCpu);3207 }3208 3209 3210 /**3211 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and3212 * others potentially updating EFLAGS.TF.3213 *3214 * The single step event must be generated using the TF value at the start of3215 * the instruction, not the new value set by it.3216 *3217 * @param pVCpu The cross context virtual CPU structure of the calling thread.3218 * @param cbInstr The number of bytes to add.3219 * @param fEflOld The EFLAGS at the start of the instruction3220 * execution.3221 */3222 DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT3223 {3224 iemRegAddToRip(pVCpu, cbInstr);3225 if (!(fEflOld & X86_EFL_TF))3226 {3227 /* Specialized iemRegFinishClearingRF edition here that doesn't check X86_EFL_TF. */3228 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);3229 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth3230 & (X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))3231 return VINF_SUCCESS;3232 return iemFinishInstructionWithFlagsSet<0 /*a_fTF*/>(pVCpu, VINF_SUCCESS); /* TF=0, so ignore it. */3233 }3234 return iemFinishInstructionWithTfSet(pVCpu);3235 }3236 3237 3238 #ifndef IEM_WITH_OPAQUE_DECODER_STATE3239 /**3240 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.3241 *3242 * @param pVCpu The cross context virtual CPU structure of the calling thread.3243 */3244 DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT3245 {3246 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));3247 }3248 #endif3249 3250 3251 #ifdef IEM_WITH_CODE_TLB3252 3253 /**3254 * Performs a near jump to the specified address, no checking or clearing of3255 * flags3256 *3257 * May raise a \#GP(0) if the new IP outside the code segment limit.3258 *3259 * @param pVCpu The cross context virtual CPU structure of the calling thread.3260 * @param uNewIp The new IP value.3261 */3262 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT3263 {3264 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3265 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))3266 pVCpu->cpum.GstCtx.rip = uNewIp;3267 else3268 return iemRaiseGeneralProtectionFault0(pVCpu);3269 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3270 }3271 3272 3273 /**3274 * Performs a near jump to the specified address, no checking or clearing of3275 * flags3276 *3277 * May raise a \#GP(0) if the new RIP is outside the code segment limit.3278 *3279 * @param pVCpu The cross context virtual CPU structure of the calling thread.3280 * @param uNewEip The new EIP value.3281 */3282 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT3283 {3284 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3285 Assert(!IEM_IS_64BIT_CODE(pVCpu));3286 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3287 pVCpu->cpum.GstCtx.rip = uNewEip;3288 else3289 return iemRaiseGeneralProtectionFault0(pVCpu);3290 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3291 }3292 3293 3294 /**3295 * Performs a near jump to the specified address, no checking or clearing of3296 * flags.3297 *3298 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3299 * segment limit.3300 *3301 * @param pVCpu The cross context virtual CPU structure of the calling thread.3302 * @param uNewRip The new RIP value.3303 */3304 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT3305 {3306 Assert(IEM_IS_64BIT_CODE(pVCpu));3307 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3308 pVCpu->cpum.GstCtx.rip = uNewRip;3309 else3310 return iemRaiseGeneralProtectionFault0(pVCpu);3311 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3312 }3313 3314 #endif /* IEM_WITH_CODE_TLB */3315 3316 /**3317 * Performs a near jump to the specified address.3318 *3319 * May raise a \#GP(0) if the new IP outside the code segment limit.3320 *3321 * @param pVCpu The cross context virtual CPU structure of the calling thread.3322 * @param uNewIp The new IP value.3323 * @param cbInstr The instruction length, for flushing in the non-TLB case.3324 */3325 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp, uint8_t cbInstr) RT_NOEXCEPT3326 {3327 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3328 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))3329 pVCpu->cpum.GstCtx.rip = uNewIp;3330 else3331 return iemRaiseGeneralProtectionFault0(pVCpu);3332 #ifndef IEM_WITH_CODE_TLB3333 iemOpcodeFlushLight(pVCpu, cbInstr);3334 #else3335 RT_NOREF_PV(cbInstr);3336 #endif3337 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3338 }3339 3340 3341 /**3342 * Performs a near jump to the specified address.3343 *3344 * May raise a \#GP(0) if the new RIP is outside the code segment limit.3345 *3346 * @param pVCpu The cross context virtual CPU structure of the calling thread.3347 * @param uNewEip The new EIP value.3348 * @param cbInstr The instruction length, for flushing in the non-TLB case.3349 */3350 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip, uint8_t cbInstr) RT_NOEXCEPT3351 {3352 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3353 Assert(!IEM_IS_64BIT_CODE(pVCpu));3354 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3355 pVCpu->cpum.GstCtx.rip = uNewEip;3356 else3357 return iemRaiseGeneralProtectionFault0(pVCpu);3358 #ifndef IEM_WITH_CODE_TLB3359 iemOpcodeFlushLight(pVCpu, cbInstr);3360 #else3361 RT_NOREF_PV(cbInstr);3362 #endif3363 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3364 }3365 3366 3367 /**3368 * Performs a near jump to the specified address.3369 *3370 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3371 * segment limit.3372 *3373 * @param pVCpu The cross context virtual CPU structure of the calling thread.3374 * @param uNewRip The new RIP value.3375 * @param cbInstr The instruction length, for flushing in the non-TLB case.3376 */3377 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip, uint8_t cbInstr) RT_NOEXCEPT3378 {3379 Assert(IEM_IS_64BIT_CODE(pVCpu));3380 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3381 pVCpu->cpum.GstCtx.rip = uNewRip;3382 else3383 return iemRaiseGeneralProtectionFault0(pVCpu);3384 #ifndef IEM_WITH_CODE_TLB3385 iemOpcodeFlushLight(pVCpu, cbInstr);3386 #else3387 RT_NOREF_PV(cbInstr);3388 #endif3389 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3390 }3391 3392 3393 /**3394 * Implements a 16-bit relative call, no checking or clearing of3395 * flags.3396 *3397 * @param pVCpu The cross context virtual CPU structure of the calling thread.3398 * @param cbInstr The instruction length.3399 * @param offDisp The 16-bit displacement.3400 */3401 DECL_FORCE_INLINE(VBOXSTRICTRC)3402 iemRegRipRelativeCallS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT3403 {3404 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;3405 uint16_t const uNewIp = uOldIp + offDisp;3406 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3407 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)3408 { /* likely */ }3409 else3410 return iemRaiseGeneralProtectionFault0(pVCpu);3411 3412 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);3413 if (rcStrict == VINF_SUCCESS)3414 { /* likely */ }3415 else3416 return rcStrict;3417 3418 pVCpu->cpum.GstCtx.rip = uNewIp;3419 #ifndef IEM_WITH_CODE_TLB3420 iemOpcodeFlushLight(pVCpu, cbInstr);3421 #endif3422 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3423 }3424 3425 3426 /**3427 * Implements a 16-bit relative call.3428 *3429 * @param pVCpu The cross context virtual CPU structure of the calling thread.3430 * @param cbInstr The instruction length.3431 * @param offDisp The 16-bit displacement.3432 */3433 DECL_FORCE_INLINE(VBOXSTRICTRC)3434 iemRegRipRelativeCallS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT3435 {3436 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;3437 uint16_t const uNewIp = uOldIp + offDisp;3438 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3439 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)3440 { /* likely */ }3441 else3442 return iemRaiseGeneralProtectionFault0(pVCpu);3443 3444 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);3445 if (rcStrict == VINF_SUCCESS)3446 { /* likely */ }3447 else3448 return rcStrict;3449 3450 pVCpu->cpum.GstCtx.rip = uNewIp;3451 #ifndef IEM_WITH_CODE_TLB3452 iemOpcodeFlushLight(pVCpu, cbInstr);3453 #endif3454 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3455 }3456 3457 3458 /**3459 * Implements a 32-bit relative call, no checking or clearing of flags.3460 *3461 * @param pVCpu The cross context virtual CPU structure of the calling thread.3462 * @param cbInstr The instruction length.3463 * @param offDisp The 32-bit displacement.3464 */3465 DECL_FORCE_INLINE(VBOXSTRICTRC)3466 iemRegEip32RelativeCallS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT3467 {3468 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));3469 3470 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3471 uint32_t const uNewRip = uOldRip + offDisp;3472 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3473 { /* likely */ }3474 else3475 return iemRaiseGeneralProtectionFault0(pVCpu);3476 3477 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3478 if (rcStrict == VINF_SUCCESS)3479 { /* likely */ }3480 else3481 return rcStrict;3482 3483 pVCpu->cpum.GstCtx.rip = uNewRip;3484 #ifndef IEM_WITH_CODE_TLB3485 iemOpcodeFlushLight(pVCpu, cbInstr);3486 #endif3487 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3488 }3489 3490 3491 /**3492 * Implements a 32-bit relative call.3493 *3494 * @param pVCpu The cross context virtual CPU structure of the calling thread.3495 * @param cbInstr The instruction length.3496 * @param offDisp The 32-bit displacement.3497 */3498 DECL_FORCE_INLINE(VBOXSTRICTRC)3499 iemRegEip32RelativeCallS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT3500 {3501 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));3502 3503 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3504 uint32_t const uNewRip = uOldRip + offDisp;3505 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3506 { /* likely */ }3507 else3508 return iemRaiseGeneralProtectionFault0(pVCpu);3509 3510 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3511 if (rcStrict == VINF_SUCCESS)3512 { /* likely */ }3513 else3514 return rcStrict;3515 3516 pVCpu->cpum.GstCtx.rip = uNewRip;3517 #ifndef IEM_WITH_CODE_TLB3518 iemOpcodeFlushLight(pVCpu, cbInstr);3519 #endif3520 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3521 }3522 3523 3524 /**3525 * Implements a 64-bit relative call, no checking or clearing of flags.3526 *3527 * @param pVCpu The cross context virtual CPU structure of the calling thread.3528 * @param cbInstr The instruction length.3529 * @param offDisp The 64-bit displacement.3530 */3531 DECL_FORCE_INLINE(VBOXSTRICTRC)3532 iemRegRip64RelativeCallS64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT3533 {3534 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3535 uint64_t const uNewRip = uOldRip + (int64_t)offDisp;3536 if (IEM_IS_CANONICAL(uNewRip))3537 { /* likely */ }3538 else3539 return iemRaiseNotCanonical(pVCpu);3540 3541 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3542 if (rcStrict == VINF_SUCCESS)3543 { /* likely */ }3544 else3545 return rcStrict;3546 3547 pVCpu->cpum.GstCtx.rip = uNewRip;3548 #ifndef IEM_WITH_CODE_TLB3549 iemOpcodeFlushLight(pVCpu, cbInstr);3550 #endif3551 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3552 }3553 3554 3555 /**3556 * Implements a 64-bit relative call.3557 *3558 * @param pVCpu The cross context virtual CPU structure of the calling thread.3559 * @param cbInstr The instruction length.3560 * @param offDisp The 64-bit displacement.3561 */3562 DECL_FORCE_INLINE(VBOXSTRICTRC)3563 iemRegRip64RelativeCallS64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT3564 {3565 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3566 uint64_t const uNewRip = uOldRip + (int64_t)offDisp;3567 if (IEM_IS_CANONICAL(uNewRip))3568 { /* likely */ }3569 else3570 return iemRaiseNotCanonical(pVCpu);3571 3572 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3573 if (rcStrict == VINF_SUCCESS)3574 { /* likely */ }3575 else3576 return rcStrict;3577 3578 pVCpu->cpum.GstCtx.rip = uNewRip;3579 #ifndef IEM_WITH_CODE_TLB3580 iemOpcodeFlushLight(pVCpu, cbInstr);3581 #endif3582 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3583 }3584 3585 3586 /**3587 * Implements an 16-bit indirect call, no checking or clearing of3588 * flags.3589 *3590 * @param pVCpu The cross context virtual CPU structure of the calling thread.3591 * @param cbInstr The instruction length.3592 * @param uNewRip The new RIP value.3593 */3594 DECL_FORCE_INLINE(VBOXSTRICTRC)3595 iemRegIp16IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3596 {3597 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3598 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3599 { /* likely */ }3600 else3601 return iemRaiseGeneralProtectionFault0(pVCpu);3602 3603 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3604 if (rcStrict == VINF_SUCCESS)3605 { /* likely */ }3606 else3607 return rcStrict;3608 3609 pVCpu->cpum.GstCtx.rip = uNewRip;3610 #ifndef IEM_WITH_CODE_TLB3611 iemOpcodeFlushLight(pVCpu, cbInstr);3612 #endif3613 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3614 }3615 3616 3617 /**3618 * Implements an 16-bit indirect call, no checking or clearing of3619 * flags.3620 *3621 * @param pVCpu The cross context virtual CPU structure of the calling thread.3622 * @param cbInstr The instruction length.3623 * @param uNewRip The new RIP value.3624 */3625 DECL_FORCE_INLINE(VBOXSTRICTRC)3626 iemRegEip32IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3627 {3628 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3629 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3630 { /* likely */ }3631 else3632 return iemRaiseGeneralProtectionFault0(pVCpu);3633 3634 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3635 if (rcStrict == VINF_SUCCESS)3636 { /* likely */ }3637 else3638 return rcStrict;3639 3640 pVCpu->cpum.GstCtx.rip = uNewRip;3641 #ifndef IEM_WITH_CODE_TLB3642 iemOpcodeFlushLight(pVCpu, cbInstr);3643 #endif3644 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3645 }3646 3647 3648 /**3649 * Implements an 16-bit indirect call.3650 *3651 * @param pVCpu The cross context virtual CPU structure of the calling thread.3652 * @param cbInstr The instruction length.3653 * @param uNewRip The new RIP value.3654 */3655 DECL_FORCE_INLINE(VBOXSTRICTRC)3656 iemRegIp16IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3657 {3658 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3659 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3660 { /* likely */ }3661 else3662 return iemRaiseGeneralProtectionFault0(pVCpu);3663 3664 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3665 if (rcStrict == VINF_SUCCESS)3666 { /* likely */ }3667 else3668 return rcStrict;3669 3670 pVCpu->cpum.GstCtx.rip = uNewRip;3671 #ifndef IEM_WITH_CODE_TLB3672 iemOpcodeFlushLight(pVCpu, cbInstr);3673 #endif3674 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3675 }3676 3677 3678 /**3679 * Implements an 16-bit indirect call.3680 *3681 * @param pVCpu The cross context virtual CPU structure of the calling thread.3682 * @param cbInstr The instruction length.3683 * @param uNewRip The new RIP value.3684 */3685 DECL_FORCE_INLINE(VBOXSTRICTRC)3686 iemRegEip32IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3687 {3688 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3689 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3690 { /* likely */ }3691 else3692 return iemRaiseGeneralProtectionFault0(pVCpu);3693 3694 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3695 if (rcStrict == VINF_SUCCESS)3696 { /* likely */ }3697 else3698 return rcStrict;3699 3700 pVCpu->cpum.GstCtx.rip = uNewRip;3701 #ifndef IEM_WITH_CODE_TLB3702 iemOpcodeFlushLight(pVCpu, cbInstr);3703 #endif3704 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3705 }3706 3707 3708 /**3709 * Implements an 32-bit indirect call, no checking or clearing of3710 * flags.3711 *3712 * @param pVCpu The cross context virtual CPU structure of the calling thread.3713 * @param cbInstr The instruction length.3714 * @param uNewRip The new RIP value.3715 */3716 DECL_FORCE_INLINE(VBOXSTRICTRC)3717 iemRegEip32IndirectCallU32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT3718 {3719 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3720 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3721 { /* likely */ }3722 else3723 return iemRaiseGeneralProtectionFault0(pVCpu);3724 3725 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3726 if (rcStrict == VINF_SUCCESS)3727 { /* likely */ }3728 else3729 return rcStrict;3730 3731 pVCpu->cpum.GstCtx.rip = uNewRip;3732 #ifndef IEM_WITH_CODE_TLB3733 iemOpcodeFlushLight(pVCpu, cbInstr);3734 #endif3735 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3736 }3737 3738 3739 /**3740 * Implements an 32-bit indirect call.3741 *3742 * @param pVCpu The cross context virtual CPU structure of the calling thread.3743 * @param cbInstr The instruction length.3744 * @param uNewRip The new RIP value.3745 */3746 DECL_FORCE_INLINE(VBOXSTRICTRC)3747 iemRegEip32IndirectCallU32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT3748 {3749 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3750 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3751 { /* likely */ }3752 else3753 return iemRaiseGeneralProtectionFault0(pVCpu);3754 3755 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3756 if (rcStrict == VINF_SUCCESS)3757 { /* likely */ }3758 else3759 return rcStrict;3760 3761 pVCpu->cpum.GstCtx.rip = uNewRip;3762 #ifndef IEM_WITH_CODE_TLB3763 iemOpcodeFlushLight(pVCpu, cbInstr);3764 #endif3765 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3766 }3767 3768 3769 /**3770 * Implements an 64-bit indirect call, no checking or clearing of3771 * flags.3772 *3773 * @param pVCpu The cross context virtual CPU structure of the calling thread.3774 * @param cbInstr The instruction length.3775 * @param uNewRip The new RIP value.3776 */3777 DECL_FORCE_INLINE(VBOXSTRICTRC)3778 iemRegRip64IndirectCallU64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT3779 {3780 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3781 if (IEM_IS_CANONICAL(uNewRip))3782 { /* likely */ }3783 else3784 return iemRaiseGeneralProtectionFault0(pVCpu);3785 3786 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3787 if (rcStrict == VINF_SUCCESS)3788 { /* likely */ }3789 else3790 return rcStrict;3791 3792 pVCpu->cpum.GstCtx.rip = uNewRip;3793 #ifndef IEM_WITH_CODE_TLB3794 iemOpcodeFlushLight(pVCpu, cbInstr);3795 #endif3796 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3797 }3798 3799 3800 /**3801 * Implements an 64-bit indirect call.3802 *3803 * @param pVCpu The cross context virtual CPU structure of the calling thread.3804 * @param cbInstr The instruction length.3805 * @param uNewRip The new RIP value.3806 */3807 DECL_FORCE_INLINE(VBOXSTRICTRC)3808 iemRegRip64IndirectCallU64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT3809 {3810 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3811 if (IEM_IS_CANONICAL(uNewRip))3812 { /* likely */ }3813 else3814 return iemRaiseGeneralProtectionFault0(pVCpu);3815 3816 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3817 if (rcStrict == VINF_SUCCESS)3818 { /* likely */ }3819 else3820 return rcStrict;3821 3822 pVCpu->cpum.GstCtx.rip = uNewRip;3823 #ifndef IEM_WITH_CODE_TLB3824 iemOpcodeFlushLight(pVCpu, cbInstr);3825 #endif3826 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3827 }3828 3829 3830 3831 /**3832 * Adds to the stack pointer.3833 *3834 * @param pVCpu The cross context virtual CPU structure of the calling thread.3835 * @param cbToAdd The number of bytes to add (8-bit!).3836 */3837 DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT3838 {3839 if (IEM_IS_64BIT_CODE(pVCpu))3840 pVCpu->cpum.GstCtx.rsp += cbToAdd;3841 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3842 pVCpu->cpum.GstCtx.esp += cbToAdd;3843 else3844 pVCpu->cpum.GstCtx.sp += cbToAdd;3845 }3846 3847 3848 /**3849 * Subtracts from the stack pointer.3850 *3851 * @param pVCpu The cross context virtual CPU structure of the calling thread.3852 * @param cbToSub The number of bytes to subtract (8-bit!).3853 */3854 DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT3855 {3856 if (IEM_IS_64BIT_CODE(pVCpu))3857 pVCpu->cpum.GstCtx.rsp -= cbToSub;3858 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3859 pVCpu->cpum.GstCtx.esp -= cbToSub;3860 else3861 pVCpu->cpum.GstCtx.sp -= cbToSub;3862 }3863 3864 3865 /**3866 * Adds to the temporary stack pointer.3867 *3868 * @param pVCpu The cross context virtual CPU structure of the calling thread.3869 * @param pTmpRsp The temporary SP/ESP/RSP to update.3870 * @param cbToAdd The number of bytes to add (16-bit).3871 */3872 DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT3873 {3874 if (IEM_IS_64BIT_CODE(pVCpu))3875 pTmpRsp->u += cbToAdd;3876 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3877 pTmpRsp->DWords.dw0 += cbToAdd;3878 else3879 pTmpRsp->Words.w0 += cbToAdd;3880 }3881 3882 3883 /**3884 * Subtracts from the temporary stack pointer.3885 *3886 * @param pVCpu The cross context virtual CPU structure of the calling thread.3887 * @param pTmpRsp The temporary SP/ESP/RSP to update.3888 * @param cbToSub The number of bytes to subtract.3889 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is3890 * expecting that.3891 */3892 DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT3893 {3894 if (IEM_IS_64BIT_CODE(pVCpu))3895 pTmpRsp->u -= cbToSub;3896 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3897 pTmpRsp->DWords.dw0 -= cbToSub;3898 else3899 pTmpRsp->Words.w0 -= cbToSub;3900 }3901 3902 3903 /**3904 * Calculates the effective stack address for a push of the specified size as3905 * well as the new RSP value (upper bits may be masked).3906 *3907 * @returns Effective stack addressf for the push.3908 * @param pVCpu The cross context virtual CPU structure of the calling thread.3909 * @param cbItem The size of the stack item to pop.3910 * @param puNewRsp Where to return the new RSP value.3911 */3912 DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT3913 {3914 RTUINT64U uTmpRsp;3915 RTGCPTR GCPtrTop;3916 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;3917 3918 if (IEM_IS_64BIT_CODE(pVCpu))3919 GCPtrTop = uTmpRsp.u -= cbItem;3920 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3921 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;3922 else3923 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;3924 *puNewRsp = uTmpRsp.u;3925 return GCPtrTop;3926 }3927 3928 3929 /**3930 * Gets the current stack pointer and calculates the value after a pop of the3931 * specified size.3932 *3933 * @returns Current stack pointer.3934 * @param pVCpu The cross context virtual CPU structure of the calling thread.3935 * @param cbItem The size of the stack item to pop.3936 * @param puNewRsp Where to return the new RSP value.3937 */3938 DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT3939 {3940 RTUINT64U uTmpRsp;3941 RTGCPTR GCPtrTop;3942 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;3943 3944 if (IEM_IS_64BIT_CODE(pVCpu))3945 {3946 GCPtrTop = uTmpRsp.u;3947 uTmpRsp.u += cbItem;3948 }3949 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3950 {3951 GCPtrTop = uTmpRsp.DWords.dw0;3952 uTmpRsp.DWords.dw0 += cbItem;3953 }3954 else3955 {3956 GCPtrTop = uTmpRsp.Words.w0;3957 uTmpRsp.Words.w0 += cbItem;3958 }3959 *puNewRsp = uTmpRsp.u;3960 return GCPtrTop;3961 }3962 3963 3964 /**3965 * Calculates the effective stack address for a push of the specified size as3966 * well as the new temporary RSP value (upper bits may be masked).3967 *3968 * @returns Effective stack addressf for the push.3969 * @param pVCpu The cross context virtual CPU structure of the calling thread.3970 * @param pTmpRsp The temporary stack pointer. This is updated.3971 * @param cbItem The size of the stack item to pop.3972 */3973 DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT3974 {3975 RTGCPTR GCPtrTop;3976 3977 if (IEM_IS_64BIT_CODE(pVCpu))3978 GCPtrTop = pTmpRsp->u -= cbItem;3979 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3980 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;3981 else3982 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;3983 return GCPtrTop;3984 }3985 3986 3987 /**3988 * Gets the effective stack address for a pop of the specified size and3989 * calculates and updates the temporary RSP.3990 *3991 * @returns Current stack pointer.3992 * @param pVCpu The cross context virtual CPU structure of the calling thread.3993 * @param pTmpRsp The temporary stack pointer. This is updated.3994 * @param cbItem The size of the stack item to pop.3995 */3996 DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT3997 {3998 RTGCPTR GCPtrTop;3999 if (IEM_IS_64BIT_CODE(pVCpu))4000 {4001 GCPtrTop = pTmpRsp->u;4002 pTmpRsp->u += cbItem;4003 }4004 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)4005 {4006 GCPtrTop = pTmpRsp->DWords.dw0;4007 pTmpRsp->DWords.dw0 += cbItem;4008 }4009 else4010 {4011 GCPtrTop = pTmpRsp->Words.w0;4012 pTmpRsp->Words.w0 += cbItem;4013 }4014 return GCPtrTop;4015 }4016 4017 4018 /** Common body for iemRegRipNearReturnAndFinishClearingRF()4019 * and iemRegRipNearReturnAndFinishNoFlags(). */4020 template<bool a_fWithFlags>4021 DECL_FORCE_INLINE(VBOXSTRICTRC)4022 iemRegRipNearReturnCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4023 {4024 /* Fetch the new RIP from the stack. */4025 VBOXSTRICTRC rcStrict;4026 RTUINT64U NewRip;4027 RTUINT64U NewRsp;4028 NewRsp.u = pVCpu->cpum.GstCtx.rsp;4029 switch (enmEffOpSize)4030 {4031 case IEMMODE_16BIT:4032 NewRip.u = 0;4033 rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);4034 break;4035 case IEMMODE_32BIT:4036 NewRip.u = 0;4037 rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);4038 break;4039 case IEMMODE_64BIT:4040 rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);4041 break;4042 IEM_NOT_REACHED_DEFAULT_CASE_RET();4043 }4044 if (rcStrict != VINF_SUCCESS)4045 return rcStrict;4046 4047 /* Check the new ew RIP before loading it. */4048 /** @todo Should test this as the intel+amd pseudo code doesn't mention half4049 * of it. The canonical test is performed here and for call. */4050 if (enmEffOpSize != IEMMODE_64BIT)4051 {4052 if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit))4053 { /* likely */ }4054 else4055 {4056 Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));4057 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);4058 }4059 }4060 else4061 {4062 if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u)))4063 { /* likely */ }4064 else4065 {4066 Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));4067 return iemRaiseNotCanonical(pVCpu);4068 }4069 }4070 4071 /* Apply cbPop */4072 if (cbPop)4073 iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);4074 4075 /* Commit it. */4076 pVCpu->cpum.GstCtx.rip = NewRip.u;4077 pVCpu->cpum.GstCtx.rsp = NewRsp.u;4078 4079 /* Flush the prefetch buffer. */4080 #ifndef IEM_WITH_CODE_TLB4081 iemOpcodeFlushLight(pVCpu, cbInstr);4082 #endif4083 RT_NOREF(cbInstr);4084 4085 4086 if (a_fWithFlags)4087 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);4088 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);4089 }4090 4091 4092 /**4093 * Implements retn and retn imm16.4094 *4095 * @param pVCpu The cross context virtual CPU structure of the4096 * calling thread.4097 * @param cbInstr The current instruction length.4098 * @param enmEffOpSize The effective operand size. This is constant.4099 * @param cbPop The amount of arguments to pop from the stack4100 * (bytes). This can be constant (zero).4101 */4102 DECL_FORCE_INLINE(VBOXSTRICTRC)4103 iemRegRipNearReturnAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4104 {4105 return iemRegRipNearReturnCommon<true /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize);4106 }4107 4108 4109 /**4110 * Implements retn and retn imm16, no checking or clearing of4111 * flags.4112 *4113 * @param pVCpu The cross context virtual CPU structure of the4114 * calling thread.4115 * @param cbInstr The current instruction length.4116 * @param enmEffOpSize The effective operand size. This is constant.4117 * @param cbPop The amount of arguments to pop from the stack4118 * (bytes). This can be constant (zero).4119 */4120 DECL_FORCE_INLINE(VBOXSTRICTRC)4121 iemRegRipNearReturnAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4122 {4123 return iemRegRipNearReturnCommon<false /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize);4124 }4125 4126 /** @} */4127 4128 4129 /** @name FPU access and helpers.4130 *4131 * @{4132 */4133 4134 4135 /**4136 * Hook for preparing to use the host FPU.4137 *4138 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4139 *4140 * @param pVCpu The cross context virtual CPU structure of the calling thread.4141 */4142 DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT4143 {4144 #ifdef IN_RING34145 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4146 #else4147 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);4148 #endif4149 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4150 }4151 4152 4153 /**4154 * Hook for preparing to use the host FPU for SSE.4155 *4156 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4157 *4158 * @param pVCpu The cross context virtual CPU structure of the calling thread.4159 */4160 DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT4161 {4162 iemFpuPrepareUsage(pVCpu);4163 }4164 4165 4166 /**4167 * Hook for preparing to use the host FPU for AVX.4168 *4169 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4170 *4171 * @param pVCpu The cross context virtual CPU structure of the calling thread.4172 */4173 DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT4174 {4175 iemFpuPrepareUsage(pVCpu);4176 }4177 4178 4179 /**4180 * Hook for actualizing the guest FPU state before the interpreter reads it.4181 *4182 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4183 *4184 * @param pVCpu The cross context virtual CPU structure of the calling thread.4185 */4186 DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4187 {4188 #ifdef IN_RING34189 NOREF(pVCpu);4190 #else4191 CPUMRZFpuStateActualizeForRead(pVCpu);4192 #endif4193 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4194 }4195 4196 4197 /**4198 * Hook for actualizing the guest FPU state before the interpreter changes it.4199 *4200 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4201 *4202 * @param pVCpu The cross context virtual CPU structure of the calling thread.4203 */4204 DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4205 {4206 #ifdef IN_RING34207 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4208 #else4209 CPUMRZFpuStateActualizeForChange(pVCpu);4210 #endif4211 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4212 }4213 4214 4215 /**4216 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read4217 * only.4218 *4219 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4220 *4221 * @param pVCpu The cross context virtual CPU structure of the calling thread.4222 */4223 DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4224 {4225 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)4226 NOREF(pVCpu);4227 #else4228 CPUMRZFpuStateActualizeSseForRead(pVCpu);4229 #endif4230 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4231 }4232 4233 4234 /**4235 * Hook for actualizing the guest XMM0..15 and MXCSR register state for4236 * read+write.4237 *4238 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4239 *4240 * @param pVCpu The cross context virtual CPU structure of the calling thread.4241 */4242 DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4243 {4244 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)4245 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4246 #else4247 CPUMRZFpuStateActualizeForChange(pVCpu);4248 #endif4249 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4250 4251 /* Make sure any changes are loaded the next time around. */4252 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;4253 }4254 4255 4256 /**4257 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read4258 * only.4259 *4260 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4261 *4262 * @param pVCpu The cross context virtual CPU structure of the calling thread.4263 */4264 DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4265 {4266 #ifdef IN_RING34267 NOREF(pVCpu);4268 #else4269 CPUMRZFpuStateActualizeAvxForRead(pVCpu);4270 #endif4271 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4272 }4273 4274 4275 /**4276 * Hook for actualizing the guest YMM0..15 and MXCSR register state for4277 * read+write.4278 *4279 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4280 *4281 * @param pVCpu The cross context virtual CPU structure of the calling thread.4282 */4283 DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4284 {4285 #ifdef IN_RING34286 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4287 #else4288 CPUMRZFpuStateActualizeForChange(pVCpu);4289 #endif4290 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4291 4292 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */4293 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;4294 }4295 4296 4297 /**4298 * Stores a QNaN value into a FPU register.4299 *4300 * @param pReg Pointer to the register.4301 */4302 DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT4303 {4304 pReg->au32[0] = UINT32_C(0x00000000);4305 pReg->au32[1] = UINT32_C(0xc0000000);4306 pReg->au16[4] = UINT16_C(0xffff);4307 }4308 4309 4310 /**4311 * Updates the FOP, FPU.CS and FPUIP registers, extended version.4312 *4313 * @param pVCpu The cross context virtual CPU structure of the calling thread.4314 * @param pFpuCtx The FPU context.4315 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).4316 */4317 DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT4318 {4319 Assert(uFpuOpcode != UINT16_MAX);4320 pFpuCtx->FOP = uFpuOpcode;4321 /** @todo x87.CS and FPUIP needs to be kept seperately. */4322 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))4323 {4324 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled4325 * happens in real mode here based on the fnsave and fnstenv images. */4326 pFpuCtx->CS = 0;4327 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);4328 }4329 else if (!IEM_IS_LONG_MODE(pVCpu))4330 {4331 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;4332 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;4333 }4334 else4335 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;4336 }4337 4338 4339 /**4340 * Marks the specified stack register as free (for FFREE).4341 *4342 * @param pVCpu The cross context virtual CPU structure of the calling thread.4343 * @param iStReg The register to free.4344 */4345 DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT4346 {4347 Assert(iStReg < 8);4348 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4349 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4350 pFpuCtx->FTW &= ~RT_BIT(iReg);4351 }4352 4353 4354 /**4355 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.4356 *4357 * @param pVCpu The cross context virtual CPU structure of the calling thread.4358 */4359 DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT4360 {4361 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4362 uint16_t uFsw = pFpuCtx->FSW;4363 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;4364 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;4365 uFsw &= ~X86_FSW_TOP_MASK;4366 uFsw |= uTop;4367 pFpuCtx->FSW = uFsw;4368 }4369 4370 4371 /**4372 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.4373 *4374 * @param pVCpu The cross context virtual CPU structure of the calling thread.4375 */4376 DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT4377 {4378 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4379 uint16_t uFsw = pFpuCtx->FSW;4380 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;4381 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;4382 uFsw &= ~X86_FSW_TOP_MASK;4383 uFsw |= uTop;4384 pFpuCtx->FSW = uFsw;4385 }4386 4387 4388 4389 4390 DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT4391 {4392 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4393 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4394 if (pFpuCtx->FTW & RT_BIT(iReg))4395 return VINF_SUCCESS;4396 return VERR_NOT_FOUND;4397 }4398 4399 4400 DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT4401 {4402 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4403 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4404 if (pFpuCtx->FTW & RT_BIT(iReg))4405 {4406 *ppRef = &pFpuCtx->aRegs[iStReg].r80;4407 return VINF_SUCCESS;4408 }4409 return VERR_NOT_FOUND;4410 }4411 4412 4413 DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,4414 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT4415 {4416 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4417 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4418 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;4419 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;4420 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))4421 {4422 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;4423 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;4424 return VINF_SUCCESS;4425 }4426 return VERR_NOT_FOUND;4427 }4428 4429 4430 DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT4431 {4432 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4433 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4434 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;4435 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;4436 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))4437 {4438 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;4439 return VINF_SUCCESS;4440 }4441 return VERR_NOT_FOUND;4442 }4443 4444 4445 /**4446 * Rotates the stack registers when setting new TOS.4447 *4448 * @param pFpuCtx The FPU context.4449 * @param iNewTop New TOS value.4450 * @remarks We only do this to speed up fxsave/fxrstor which4451 * arrange the FP registers in stack order.4452 * MUST be done before writing the new TOS (FSW).4453 */4454 DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT4455 {4456 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4457 RTFLOAT80U ar80Temp[8];4458 4459 if (iOldTop == iNewTop)4460 return;4461 4462 /* Unscrew the stack and get it into 'native' order. */4463 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;4464 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;4465 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;4466 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;4467 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;4468 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;4469 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;4470 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;4471 4472 /* Now rotate the stack to the new position. */4473 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];4474 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];4475 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];4476 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];4477 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];4478 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];4479 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];4480 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];4481 }4482 4483 4484 /**4485 * Updates the FPU exception status after FCW is changed.4486 *4487 * @param pFpuCtx The FPU context.4488 */4489 DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT4490 {4491 uint16_t u16Fsw = pFpuCtx->FSW;4492 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))4493 u16Fsw |= X86_FSW_ES | X86_FSW_B;4494 else4495 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);4496 pFpuCtx->FSW = u16Fsw;4497 }4498 4499 4500 /**4501 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.4502 *4503 * @returns The full FTW.4504 * @param pFpuCtx The FPU context.4505 */4506 DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT4507 {4508 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;4509 uint16_t u16Ftw = 0;4510 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4511 for (unsigned iSt = 0; iSt < 8; iSt++)4512 {4513 unsigned const iReg = (iSt + iTop) & 7;4514 if (!(u8Ftw & RT_BIT(iReg)))4515 u16Ftw |= 3 << (iReg * 2); /* empty */4516 else4517 {4518 uint16_t uTag;4519 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;4520 if (pr80Reg->s.uExponent == 0x7fff)4521 uTag = 2; /* Exponent is all 1's => Special. */4522 else if (pr80Reg->s.uExponent == 0x0000)4523 {4524 if (pr80Reg->s.uMantissa == 0x0000)4525 uTag = 1; /* All bits are zero => Zero. */4526 else4527 uTag = 2; /* Must be special. */4528 }4529 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */4530 uTag = 0; /* Valid. */4531 else4532 uTag = 2; /* Must be special. */4533 4534 u16Ftw |= uTag << (iReg * 2);4535 }4536 }4537 4538 return u16Ftw;4539 }4540 4541 4542 /**4543 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).4544 *4545 * @returns The compressed FTW.4546 * @param u16FullFtw The full FTW to convert.4547 */4548 DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT4549 {4550 uint8_t u8Ftw = 0;4551 for (unsigned i = 0; i < 8; i++)4552 {4553 if ((u16FullFtw & 3) != 3 /*empty*/)4554 u8Ftw |= RT_BIT(i);4555 u16FullFtw >>= 2;4556 }4557 4558 return u8Ftw;4559 }4560 4561 /** @} */4562 4563 4564 /** @name Memory access.4565 *4566 * @{4567 */4568 4569 43 4570 44 /** … … 4662 136 } 4663 137 4664 4665 /**4666 * Maps a physical page.4667 *4668 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).4669 * @param pVCpu The cross context virtual CPU structure of the calling thread.4670 * @param GCPhysMem The physical address.4671 * @param fAccess The intended access.4672 * @param ppvMem Where to return the mapping address.4673 * @param pLock The PGM lock.4674 */4675 DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,4676 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT4677 {4678 #ifdef IEM_LOG_MEMORY_WRITES4679 if (fAccess & IEM_ACCESS_TYPE_WRITE)4680 return VERR_PGM_PHYS_TLB_CATCH_ALL;4681 #endif4682 4683 /** @todo This API may require some improving later. A private deal with PGM4684 * regarding locking and unlocking needs to be struct. A couple of TLBs4685 * living in PGM, but with publicly accessible inlined access methods4686 * could perhaps be an even better solution. */4687 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,4688 GCPhysMem,4689 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),4690 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),4691 ppvMem,4692 pLock);4693 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/4694 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));4695 4696 return rc;4697 }4698 4699 4700 /**4701 * Unmap a page previously mapped by iemMemPageMap.4702 *4703 * @param pVCpu The cross context virtual CPU structure of the calling thread.4704 * @param GCPhysMem The physical address.4705 * @param fAccess The intended access.4706 * @param pvMem What iemMemPageMap returned.4707 * @param pLock The PGM lock.4708 */4709 DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,4710 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT4711 {4712 NOREF(pVCpu);4713 NOREF(GCPhysMem);4714 NOREF(fAccess);4715 NOREF(pvMem);4716 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);4717 }4718 138 4719 139 #ifdef IEM_WITH_SETJMP … … 4879 299 4880 300 /* 4881 * Unmap helpers.4882 */4883 4884 #ifdef IEM_WITH_SETJMP4885 4886 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4887 {4888 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4889 if (RT_LIKELY(bMapInfo == 0))4890 return;4891 # endif4892 iemMemCommitAndUnmapRwSafeJmp(pVCpu, bMapInfo);4893 }4894 4895 4896 DECL_INLINE_THROW(void) iemMemCommitAndUnmapAtJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4897 {4898 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4899 if (RT_LIKELY(bMapInfo == 0))4900 return;4901 # endif4902 iemMemCommitAndUnmapAtSafeJmp(pVCpu, bMapInfo);4903 }4904 4905 4906 DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4907 {4908 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4909 if (RT_LIKELY(bMapInfo == 0))4910 return;4911 # endif4912 iemMemCommitAndUnmapWoSafeJmp(pVCpu, bMapInfo);4913 }4914 4915 4916 DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP4917 {4918 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4919 if (RT_LIKELY(bMapInfo == 0))4920 return;4921 # endif4922 iemMemCommitAndUnmapRoSafeJmp(pVCpu, bMapInfo);4923 }4924 4925 DECLINLINE(void) iemMemRollbackAndUnmapWo(PVMCPUCC pVCpu, uint8_t bMapInfo) RT_NOEXCEPT4926 {4927 # if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)4928 if (RT_LIKELY(bMapInfo == 0))4929 return;4930 # endif4931 iemMemRollbackAndUnmapWoSafe(pVCpu, bMapInfo);4932 }4933 4934 #endif /* IEM_WITH_SETJMP */4935 4936 4937 /*4938 301 * Instantiate R/W inline templates. 4939 302 */ … … 5101 464 /** @} */ 5102 465 5103 5104 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX 5105 5106 /** 5107 * Gets CR0 fixed-0 bits in VMX operation. 5108 * 5109 * We do this rather than fetching what we report to the guest (in 5110 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same 5111 * values regardless of whether unrestricted-guest feature is available on the CPU. 5112 * 5113 * @returns CR0 fixed-0 bits. 5114 * @param pVCpu The cross context virtual CPU structure. 5115 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode 5116 * must be returned. When @c false, the CR0 fixed-0 5117 * bits for VMX root mode is returned. 5118 * 5119 */ 5120 DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT 5121 { 5122 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); 5123 5124 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs; 5125 if ( fVmxNonRootMode 5126 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)) 5127 return VMX_V_CR0_FIXED0_UX; 5128 return VMX_V_CR0_FIXED0; 5129 } 5130 5131 5132 # ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */ 5133 /** 5134 * Sets virtual-APIC write emulation as pending. 5135 * 5136 * @param pVCpu The cross context virtual CPU structure. 5137 * @param offApic The offset in the virtual-APIC page that was written. 5138 */ 5139 DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT 5140 { 5141 Assert(offApic < XAPIC_OFF_END + 4); 5142 5143 /* 5144 * Record the currently updated APIC offset, as we need this later for figuring 5145 * out whether to perform TPR, EOI or self-IPI virtualization as well as well 5146 * as for supplying the exit qualification when causing an APIC-write VM-exit. 5147 */ 5148 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic; 5149 5150 /* 5151 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI 5152 * virtualization or APIC-write emulation). 5153 */ 5154 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) 5155 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE); 5156 } 5157 # endif /* XAPIC_OFF_END */ 5158 5159 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ 5160 5161 #if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) 5162 /** 5163 * Adds an entry to the TLB trace buffer. 5164 * 5165 * @note Don't use directly, only via the IEMTLBTRACE_XXX macros. 5166 */ 5167 DECLINLINE(void) iemTlbTrace(PVMCPU pVCpu, IEMTLBTRACETYPE enmType, uint64_t u64Param, uint64_t u64Param2 = 0, 5168 uint8_t bParam = 0, uint32_t u32Param = 0/*, uint16_t u16Param = 0 */) 5169 { 5170 uint32_t const fMask = RT_BIT_32(pVCpu->iem.s.cTlbTraceEntriesShift) - 1; 5171 PIEMTLBTRACEENTRY const pEntry = &pVCpu->iem.s.paTlbTraceEntries[pVCpu->iem.s.idxTlbTraceEntry++ & fMask]; 5172 pEntry->u64Param = u64Param; 5173 pEntry->u64Param2 = u64Param2; 5174 pEntry->u16Param = 0; //u16Param; 5175 pEntry->u32Param = u32Param; 5176 pEntry->bParam = bParam; 5177 pEntry->enmType = enmType; 5178 pEntry->rip = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base; 5179 } 5180 #endif 5181 5182 #endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */ 466 #endif /* !VMM_INCLUDED_SRC_VMMAll_target_x86_IEMInlineMem_x86_h */ -
trunk/src/VBox/VMM/VMMR3/IEMR3.cpp
r108186 r108260 61 61 # include "IEMThreadedFunctions.h" 62 62 # include "IEMInline.h" 63 # ifdef VBOX_VMM_TARGET_X86 64 # include "VMMAll/target-x86/IEMInline-x86.h" 65 # endif 63 66 #endif 64 67 -
trunk/src/VBox/VMM/include/IEMInline.h
r108246 r108260 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - Inlined Functions .3 * IEM - Interpreted Execution Manager - Inlined Functions, Common. 4 4 */ 5 5 … … 33 33 34 34 #include <VBox/err.h> 35 36 37 /* Documentation and forward declarations for target specific inline functions: */ 38 39 /** 40 * Calculates the the IEM_F_XXX flags. 41 * 42 * @returns IEM_F_XXX combination match the current CPU state. 43 * @param pVCpu The cross context virtual CPU structure of the 44 * calling thread. 45 */ 46 DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT; 47 48 #if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING) 49 /** 50 * Invalidates the decoder state and asserts various stuff - strict builds only. 51 * 52 * @param pVCpu The cross context virtual CPU structure of the 53 * calling thread. 54 */ 55 DECLINLINE(void) iemInitExecTargetStrict(PVMCPUCC pVCpu) RT_NOEXCEPT; 56 #endif 57 35 58 36 59 … … 172 195 173 196 174 /**175 * Calculates the IEM_F_X86_AC flags.176 *177 * @returns IEM_F_X86_AC or zero178 * @param pVCpu The cross context virtual CPU structure of the179 * calling thread.180 */181 DECL_FORCE_INLINE(uint32_t) iemCalcExecAcFlag(PVMCPUCC pVCpu) RT_NOEXCEPT182 {183 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS);184 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));185 186 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC187 || (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_AM | X86_CR0_PE)) != (X86_CR0_AM | X86_CR0_PE)188 || ( !pVCpu->cpum.GstCtx.eflags.Bits.u1VM189 && pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl != 3))190 return 0;191 return IEM_F_X86_AC;192 }193 194 195 /**196 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.197 *198 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will199 * reject expand down data segments and conforming code segments.200 *201 * ASSUMES that the CPU is in 32-bit mode.202 *203 * @note Will return zero when if any of the segment register state is marked204 * external, this must be factored into assertions checking fExec205 * consistency.206 *207 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.208 * @param pVCpu The cross context virtual CPU structure of the209 * calling thread.210 * @sa iemCalc32BitFlatIndicatorEsDs211 */212 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT213 {214 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);215 return ( ( pVCpu->cpum.GstCtx.es.Attr.u216 | pVCpu->cpum.GstCtx.cs.Attr.u217 | pVCpu->cpum.GstCtx.ss.Attr.u218 | pVCpu->cpum.GstCtx.ds.Attr.u)219 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))220 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)221 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)222 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1)223 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1)224 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))225 == 0226 && ( pVCpu->cpum.GstCtx.es.u64Base227 | pVCpu->cpum.GstCtx.cs.u64Base228 | pVCpu->cpum.GstCtx.ss.u64Base229 | pVCpu->cpum.GstCtx.ds.u64Base)230 == 0231 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))232 ? IEM_F_MODE_X86_32BIT_FLAT : 0;233 }234 235 236 /**237 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are238 * flat already.239 *240 * This is used by sysenter.241 *242 * @note Will return zero when if any of the segment register state is marked243 * external, this must be factored into assertions checking fExec244 * consistency.245 *246 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.247 * @param pVCpu The cross context virtual CPU structure of the248 * calling thread.249 * @sa iemCalc32BitFlatIndicator250 */251 DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT252 {253 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);254 return ( ( pVCpu->cpum.GstCtx.es.Attr.u255 | pVCpu->cpum.GstCtx.ds.Attr.u)256 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))257 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)258 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)259 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))260 == 0261 && ( pVCpu->cpum.GstCtx.es.u64Base262 | pVCpu->cpum.GstCtx.ds.u64Base)263 == 0264 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))265 ? IEM_F_MODE_X86_32BIT_FLAT : 0;266 }267 268 269 /**270 * Calculates the IEM_F_MODE_XXX, CPL and AC flags.271 *272 * @returns IEM_F_MODE_XXX, IEM_F_X86_CPL_MASK and IEM_F_X86_AC.273 * @param pVCpu The cross context virtual CPU structure of the274 * calling thread.275 */276 DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT277 {278 /*279 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx280 * here to try get this done as efficiently as possible.281 */282 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS);283 284 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)285 {286 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)287 {288 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));289 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);290 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC291 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)292 || fExec != (3U << IEM_F_X86_CPL_SHIFT))293 { /* likely */ }294 else295 fExec |= IEM_F_X86_AC;296 297 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)298 {299 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));300 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu);301 }302 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long303 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))304 fExec |= IEM_F_MODE_X86_64BIT;305 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)306 fExec |= IEM_F_MODE_X86_16BIT_PROT;307 else308 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386;309 return fExec;310 }311 if ( !pVCpu->cpum.GstCtx.eflags.Bits.u1AC312 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM))313 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);314 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT) | IEM_F_X86_AC;315 }316 317 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */318 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))319 {320 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)321 return IEM_F_MODE_X86_16BIT;322 return IEM_F_MODE_X86_16BIT_PRE_386;323 }324 325 /* 32-bit unreal mode. */326 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu);327 }328 329 330 /**331 * Calculates the AMD-V and VT-x related context flags.332 *333 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and334 * IEM_F_X86_CTX_VMX.335 * @param pVCpu The cross context virtual CPU structure of the336 * calling thread.337 */338 DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT339 {340 /*341 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled342 * and CPUMIsGuestInNestedHwvirtMode to some extent.343 */344 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);345 346 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);347 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)348 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);349 if (RT_LIKELY(!fTmp))350 return 0; /* likely */351 352 if (fTmp & X86_CR4_VMXE)353 {354 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);355 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)356 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST;357 return IEM_F_X86_CTX_VMX;358 }359 360 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);361 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)362 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST;363 return IEM_F_X86_CTX_SVM;364 }365 366 #ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */367 368 /**369 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.370 *371 * @returns IEM_F_BRK_PENDING_XXX or zero.372 * @param pVCpu The cross context virtual CPU structure of the373 * calling thread.374 */375 DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT376 {377 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);378 379 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)380 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))381 return 0;382 return iemCalcExecDbgFlagsSlow(pVCpu);383 }384 385 /**386 * Calculates the the IEM_F_XXX flags.387 *388 * @returns IEM_F_XXX combination match the current CPU state.389 * @param pVCpu The cross context virtual CPU structure of the390 * calling thread.391 */392 DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT393 {394 return iemCalcExecModeAndCplFlags(pVCpu)395 | iemCalcExecHwVirtFlags(pVCpu)396 /* SMM is not yet implemented */397 | iemCalcExecDbgFlags(pVCpu)398 ;399 }400 401 402 /**403 * Re-calculates the MODE and CPL parts of IEMCPU::fExec.404 *405 * @param pVCpu The cross context virtual CPU structure of the406 * calling thread.407 */408 DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplAndAcFlags(PVMCPUCC pVCpu)409 {410 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK | IEM_F_X86_AC))411 | iemCalcExecModeAndCplFlags(pVCpu);412 }413 414 415 /**416 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.417 *418 * @param pVCpu The cross context virtual CPU structure of the419 * calling thread.420 */421 DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)422 {423 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)424 | iemCalcExecDbgFlags(pVCpu);425 }426 427 #endif /* VBOX_INCLUDED_vmm_dbgf_h */428 429 430 197 #ifndef IEM_WITH_OPAQUE_DECODER_STATE 431 198 … … 448 215 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK); 449 216 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); 450 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));451 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));452 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));453 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));454 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));455 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));456 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));457 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));458 217 459 218 pVCpu->iem.s.rcPassUp = VINF_SUCCESS; … … 463 222 464 223 # ifdef VBOX_STRICT 465 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe; 466 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe; 467 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe; 468 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe; 469 pVCpu->iem.s.fPrefixes = 0xfeedbeef; 470 pVCpu->iem.s.uRexReg = 127; 471 pVCpu->iem.s.uRexB = 127; 472 pVCpu->iem.s.offModRm = 127; 473 pVCpu->iem.s.uRexIndex = 127; 474 pVCpu->iem.s.iEffSeg = 127; 475 pVCpu->iem.s.idxPrefix = 127; 476 pVCpu->iem.s.uVex3rdReg = 127; 477 pVCpu->iem.s.uVexLength = 127; 478 pVCpu->iem.s.fEvexStuff = 127; 479 pVCpu->iem.s.uFpuOpcode = UINT16_MAX; 480 # ifdef IEM_WITH_CODE_TLB 481 pVCpu->iem.s.offInstrNextByte = UINT16_MAX; 482 pVCpu->iem.s.pbInstrBuf = NULL; 483 pVCpu->iem.s.cbInstrBuf = UINT16_MAX; 484 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX; 485 pVCpu->iem.s.offCurInstrStart = INT16_MAX; 486 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff); 487 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF 488 pVCpu->iem.s.offOpcode = 127; 489 # endif 490 # else 491 pVCpu->iem.s.offOpcode = 127; 492 pVCpu->iem.s.cbOpcode = 127; 493 # endif 494 # endif /* VBOX_STRICT */ 224 iemInitExecTargetStrict(pVCpu); 225 # endif 495 226 } 496 227 … … 552 283 } 553 284 554 555 /**556 * Macro used by the IEMExec* method to check the given instruction length.557 *558 * Will return on failure!559 *560 * @param a_cbInstr The given instruction length.561 * @param a_cbMin The minimum length.562 */563 # define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \564 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \565 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)566 567 568 # ifndef IEM_WITH_SETJMP569 570 /**571 * Fetches the first opcode byte.572 *573 * @returns Strict VBox status code.574 * @param pVCpu The cross context virtual CPU structure of the575 * calling thread.576 * @param pu8 Where to return the opcode byte.577 */578 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT579 {580 /*581 * Check for hardware instruction breakpoints.582 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs.583 */584 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))585 { /* likely */ }586 else587 {588 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,589 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base,590 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS)591 || IEM_IS_GUEST_CPU_AMD(pVCpu));592 if (RT_LIKELY(rcStrict == VINF_SUCCESS))593 { /* likely */ }594 else595 {596 *pu8 = 0xff; /* shut up gcc. sigh */597 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)598 return iemRaiseDebugException(pVCpu);599 return rcStrict;600 }601 }602 603 /*604 * Fetch the first opcode byte.605 */606 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;607 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))608 {609 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;610 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];611 return VINF_SUCCESS;612 }613 return iemOpcodeGetNextU8Slow(pVCpu, pu8);614 }615 616 # else /* IEM_WITH_SETJMP */617 618 /**619 * Fetches the first opcode byte, longjmp on error.620 *621 * @returns The opcode byte.622 * @param pVCpu The cross context virtual CPU structure of the calling thread.623 */624 DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP625 {626 /*627 * Check for hardware instruction breakpoints.628 * Note! Guest breakpoints are only checked after POP SS or MOV SS on AMD CPUs.629 */630 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))631 { /* likely */ }632 else633 {634 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,635 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base,636 !(pVCpu->cpum.GstCtx.rflags.uBoth & CPUMCTX_INHIBIT_SHADOW_SS)637 || IEM_IS_GUEST_CPU_AMD(pVCpu));638 if (RT_LIKELY(rcStrict == VINF_SUCCESS))639 { /* likely */ }640 else641 {642 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)643 rcStrict = iemRaiseDebugException(pVCpu);644 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));645 }646 }647 648 /*649 * Fetch the first opcode byte.650 */651 # ifdef IEM_WITH_CODE_TLB652 uint8_t bRet;653 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;654 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;655 if (RT_LIKELY( pbBuf != NULL656 && offBuf < pVCpu->iem.s.cbInstrBuf))657 {658 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;659 bRet = pbBuf[offBuf];660 }661 else662 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);663 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF664 Assert(pVCpu->iem.s.offOpcode == 0);665 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;666 # endif667 return bRet;668 669 # else /* !IEM_WITH_CODE_TLB */670 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;671 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))672 {673 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;674 return pVCpu->iem.s.abOpcode[offOpcode];675 }676 return iemOpcodeGetNextU8SlowJmp(pVCpu);677 # endif678 }679 680 # endif /* IEM_WITH_SETJMP */681 682 /**683 * Fetches the first opcode byte, returns/throws automatically on failure.684 *685 * @param a_pu8 Where to return the opcode byte.686 * @remark Implicitly references pVCpu.687 */688 # ifndef IEM_WITH_SETJMP689 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) \690 do \691 { \692 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \693 if (rcStrict2 == VINF_SUCCESS) \694 { /* likely */ } \695 else \696 return rcStrict2; \697 } while (0)698 # else699 # define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))700 # endif /* IEM_WITH_SETJMP */701 702 703 # ifndef IEM_WITH_SETJMP704 705 /**706 * Fetches the next opcode byte.707 *708 * @returns Strict VBox status code.709 * @param pVCpu The cross context virtual CPU structure of the710 * calling thread.711 * @param pu8 Where to return the opcode byte.712 */713 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT714 {715 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;716 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))717 {718 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;719 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];720 return VINF_SUCCESS;721 }722 return iemOpcodeGetNextU8Slow(pVCpu, pu8);723 }724 725 # else /* IEM_WITH_SETJMP */726 727 /**728 * Fetches the next opcode byte, longjmp on error.729 *730 * @returns The opcode byte.731 * @param pVCpu The cross context virtual CPU structure of the calling thread.732 */733 DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP734 {735 # ifdef IEM_WITH_CODE_TLB736 uint8_t bRet;737 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;738 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;739 if (RT_LIKELY( pbBuf != NULL740 && offBuf < pVCpu->iem.s.cbInstrBuf))741 {742 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;743 bRet = pbBuf[offBuf];744 }745 else746 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);747 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF748 Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));749 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;750 # endif751 return bRet;752 753 # else /* !IEM_WITH_CODE_TLB */754 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;755 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))756 {757 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;758 return pVCpu->iem.s.abOpcode[offOpcode];759 }760 return iemOpcodeGetNextU8SlowJmp(pVCpu);761 # endif762 }763 764 # endif /* IEM_WITH_SETJMP */765 766 /**767 * Fetches the next opcode byte, returns automatically on failure.768 *769 * @param a_pu8 Where to return the opcode byte.770 * @remark Implicitly references pVCpu.771 */772 # ifndef IEM_WITH_SETJMP773 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) \774 do \775 { \776 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \777 if (rcStrict2 == VINF_SUCCESS) \778 { /* likely */ } \779 else \780 return rcStrict2; \781 } while (0)782 # else783 # define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))784 # endif /* IEM_WITH_SETJMP */785 786 787 # ifndef IEM_WITH_SETJMP788 /**789 * Fetches the next signed byte from the opcode stream.790 *791 * @returns Strict VBox status code.792 * @param pVCpu The cross context virtual CPU structure of the calling thread.793 * @param pi8 Where to return the signed byte.794 */795 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT796 {797 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);798 }799 # endif /* !IEM_WITH_SETJMP */800 801 802 /**803 * Fetches the next signed byte from the opcode stream, returning automatically804 * on failure.805 *806 * @param a_pi8 Where to return the signed byte.807 * @remark Implicitly references pVCpu.808 */809 # ifndef IEM_WITH_SETJMP810 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) \811 do \812 { \813 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \814 if (rcStrict2 != VINF_SUCCESS) \815 return rcStrict2; \816 } while (0)817 # else /* IEM_WITH_SETJMP */818 # define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))819 820 # endif /* IEM_WITH_SETJMP */821 822 823 # ifndef IEM_WITH_SETJMP824 /**825 * Fetches the next signed byte from the opcode stream, extending it to826 * unsigned 16-bit.827 *828 * @returns Strict VBox status code.829 * @param pVCpu The cross context virtual CPU structure of the calling thread.830 * @param pu16 Where to return the unsigned word.831 */832 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT833 {834 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;835 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))836 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);837 838 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];839 pVCpu->iem.s.offOpcode = offOpcode + 1;840 return VINF_SUCCESS;841 }842 # endif /* !IEM_WITH_SETJMP */843 844 /**845 * Fetches the next signed byte from the opcode stream and sign-extending it to846 * a word, returning automatically on failure.847 *848 * @param a_pu16 Where to return the word.849 * @remark Implicitly references pVCpu.850 */851 # ifndef IEM_WITH_SETJMP852 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \853 do \854 { \855 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \856 if (rcStrict2 != VINF_SUCCESS) \857 return rcStrict2; \858 } while (0)859 # else860 # define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))861 # endif862 863 # ifndef IEM_WITH_SETJMP864 /**865 * Fetches the next signed byte from the opcode stream, extending it to866 * unsigned 32-bit.867 *868 * @returns Strict VBox status code.869 * @param pVCpu The cross context virtual CPU structure of the calling thread.870 * @param pu32 Where to return the unsigned dword.871 */872 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT873 {874 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;875 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))876 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);877 878 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];879 pVCpu->iem.s.offOpcode = offOpcode + 1;880 return VINF_SUCCESS;881 }882 # endif /* !IEM_WITH_SETJMP */883 884 /**885 * Fetches the next signed byte from the opcode stream and sign-extending it to886 * a word, returning automatically on failure.887 *888 * @param a_pu32 Where to return the word.889 * @remark Implicitly references pVCpu.890 */891 # ifndef IEM_WITH_SETJMP892 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \893 do \894 { \895 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \896 if (rcStrict2 != VINF_SUCCESS) \897 return rcStrict2; \898 } while (0)899 # else900 # define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))901 # endif902 903 904 # ifndef IEM_WITH_SETJMP905 /**906 * Fetches the next signed byte from the opcode stream, extending it to907 * unsigned 64-bit.908 *909 * @returns Strict VBox status code.910 * @param pVCpu The cross context virtual CPU structure of the calling thread.911 * @param pu64 Where to return the unsigned qword.912 */913 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT914 {915 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;916 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))917 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);918 919 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];920 pVCpu->iem.s.offOpcode = offOpcode + 1;921 return VINF_SUCCESS;922 }923 # endif /* !IEM_WITH_SETJMP */924 925 /**926 * Fetches the next signed byte from the opcode stream and sign-extending it to927 * a word, returning automatically on failure.928 *929 * @param a_pu64 Where to return the word.930 * @remark Implicitly references pVCpu.931 */932 # ifndef IEM_WITH_SETJMP933 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \934 do \935 { \936 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \937 if (rcStrict2 != VINF_SUCCESS) \938 return rcStrict2; \939 } while (0)940 # else941 # define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))942 # endif943 944 945 # ifndef IEM_WITH_SETJMP946 947 /**948 * Fetches the next opcode word.949 *950 * @returns Strict VBox status code.951 * @param pVCpu The cross context virtual CPU structure of the calling thread.952 * @param pu16 Where to return the opcode word.953 */954 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT955 {956 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;957 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))958 {959 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;960 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS961 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];962 # else963 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);964 # endif965 return VINF_SUCCESS;966 }967 return iemOpcodeGetNextU16Slow(pVCpu, pu16);968 }969 970 # else /* IEM_WITH_SETJMP */971 972 /**973 * Fetches the next opcode word, longjmp on error.974 *975 * @returns The opcode word.976 * @param pVCpu The cross context virtual CPU structure of the calling thread.977 */978 DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP979 {980 # ifdef IEM_WITH_CODE_TLB981 uint16_t u16Ret;982 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;983 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;984 if (RT_LIKELY( pbBuf != NULL985 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))986 {987 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;988 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS989 u16Ret = *(uint16_t const *)&pbBuf[offBuf];990 # else991 u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);992 # endif993 }994 else995 u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);996 997 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF998 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;999 Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));1000 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1001 *(uint16_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;1002 # else1003 pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);1004 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);1005 # endif1006 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;1007 # endif1008 1009 return u16Ret;1010 1011 # else /* !IEM_WITH_CODE_TLB */1012 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1013 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))1014 {1015 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;1016 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1017 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1018 # else1019 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1020 # endif1021 }1022 return iemOpcodeGetNextU16SlowJmp(pVCpu);1023 # endif /* !IEM_WITH_CODE_TLB */1024 }1025 1026 # endif /* IEM_WITH_SETJMP */1027 1028 /**1029 * Fetches the next opcode word, returns automatically on failure.1030 *1031 * @param a_pu16 Where to return the opcode word.1032 * @remark Implicitly references pVCpu.1033 */1034 # ifndef IEM_WITH_SETJMP1035 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) \1036 do \1037 { \1038 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \1039 if (rcStrict2 != VINF_SUCCESS) \1040 return rcStrict2; \1041 } while (0)1042 # else1043 # define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))1044 # endif1045 1046 # ifndef IEM_WITH_SETJMP1047 /**1048 * Fetches the next opcode word, zero extending it to a double word.1049 *1050 * @returns Strict VBox status code.1051 * @param pVCpu The cross context virtual CPU structure of the calling thread.1052 * @param pu32 Where to return the opcode double word.1053 */1054 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT1055 {1056 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1057 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))1058 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);1059 1060 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1061 pVCpu->iem.s.offOpcode = offOpcode + 2;1062 return VINF_SUCCESS;1063 }1064 # endif /* !IEM_WITH_SETJMP */1065 1066 /**1067 * Fetches the next opcode word and zero extends it to a double word, returns1068 * automatically on failure.1069 *1070 * @param a_pu32 Where to return the opcode double word.1071 * @remark Implicitly references pVCpu.1072 */1073 # ifndef IEM_WITH_SETJMP1074 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \1075 do \1076 { \1077 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \1078 if (rcStrict2 != VINF_SUCCESS) \1079 return rcStrict2; \1080 } while (0)1081 # else1082 # define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))1083 # endif1084 1085 # ifndef IEM_WITH_SETJMP1086 /**1087 * Fetches the next opcode word, zero extending it to a quad word.1088 *1089 * @returns Strict VBox status code.1090 * @param pVCpu The cross context virtual CPU structure of the calling thread.1091 * @param pu64 Where to return the opcode quad word.1092 */1093 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1094 {1095 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1096 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))1097 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);1098 1099 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);1100 pVCpu->iem.s.offOpcode = offOpcode + 2;1101 return VINF_SUCCESS;1102 }1103 # endif /* !IEM_WITH_SETJMP */1104 1105 /**1106 * Fetches the next opcode word and zero extends it to a quad word, returns1107 * automatically on failure.1108 *1109 * @param a_pu64 Where to return the opcode quad word.1110 * @remark Implicitly references pVCpu.1111 */1112 # ifndef IEM_WITH_SETJMP1113 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \1114 do \1115 { \1116 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \1117 if (rcStrict2 != VINF_SUCCESS) \1118 return rcStrict2; \1119 } while (0)1120 # else1121 # define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))1122 # endif1123 1124 1125 # ifndef IEM_WITH_SETJMP1126 /**1127 * Fetches the next signed word from the opcode stream.1128 *1129 * @returns Strict VBox status code.1130 * @param pVCpu The cross context virtual CPU structure of the calling thread.1131 * @param pi16 Where to return the signed word.1132 */1133 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT1134 {1135 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);1136 }1137 # endif /* !IEM_WITH_SETJMP */1138 1139 1140 /**1141 * Fetches the next signed word from the opcode stream, returning automatically1142 * on failure.1143 *1144 * @param a_pi16 Where to return the signed word.1145 * @remark Implicitly references pVCpu.1146 */1147 # ifndef IEM_WITH_SETJMP1148 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) \1149 do \1150 { \1151 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \1152 if (rcStrict2 != VINF_SUCCESS) \1153 return rcStrict2; \1154 } while (0)1155 # else1156 # define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))1157 # endif1158 1159 # ifndef IEM_WITH_SETJMP1160 1161 /**1162 * Fetches the next opcode dword.1163 *1164 * @returns Strict VBox status code.1165 * @param pVCpu The cross context virtual CPU structure of the calling thread.1166 * @param pu32 Where to return the opcode double word.1167 */1168 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT1169 {1170 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1171 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))1172 {1173 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;1174 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1175 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1176 # else1177 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1178 pVCpu->iem.s.abOpcode[offOpcode + 1],1179 pVCpu->iem.s.abOpcode[offOpcode + 2],1180 pVCpu->iem.s.abOpcode[offOpcode + 3]);1181 # endif1182 return VINF_SUCCESS;1183 }1184 return iemOpcodeGetNextU32Slow(pVCpu, pu32);1185 }1186 1187 # else /* IEM_WITH_SETJMP */1188 1189 /**1190 * Fetches the next opcode dword, longjmp on error.1191 *1192 * @returns The opcode dword.1193 * @param pVCpu The cross context virtual CPU structure of the calling thread.1194 */1195 DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP1196 {1197 # ifdef IEM_WITH_CODE_TLB1198 uint32_t u32Ret;1199 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;1200 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;1201 if (RT_LIKELY( pbBuf != NULL1202 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))1203 {1204 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;1205 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1206 u32Ret = *(uint32_t const *)&pbBuf[offBuf];1207 # else1208 u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],1209 pbBuf[offBuf + 1],1210 pbBuf[offBuf + 2],1211 pbBuf[offBuf + 3]);1212 # endif1213 }1214 else1215 u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);1216 1217 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF1218 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1219 Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));1220 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1221 *(uint32_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;1222 # else1223 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);1224 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);1225 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);1226 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);1227 # endif1228 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;1229 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */1230 1231 return u32Ret;1232 1233 # else /* !IEM_WITH_CODE_TLB */1234 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1235 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))1236 {1237 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;1238 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1239 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1240 # else1241 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1242 pVCpu->iem.s.abOpcode[offOpcode + 1],1243 pVCpu->iem.s.abOpcode[offOpcode + 2],1244 pVCpu->iem.s.abOpcode[offOpcode + 3]);1245 # endif1246 }1247 return iemOpcodeGetNextU32SlowJmp(pVCpu);1248 # endif1249 }1250 1251 # endif /* IEM_WITH_SETJMP */1252 1253 /**1254 * Fetches the next opcode dword, returns automatically on failure.1255 *1256 * @param a_pu32 Where to return the opcode dword.1257 * @remark Implicitly references pVCpu.1258 */1259 # ifndef IEM_WITH_SETJMP1260 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) \1261 do \1262 { \1263 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \1264 if (rcStrict2 != VINF_SUCCESS) \1265 return rcStrict2; \1266 } while (0)1267 # else1268 # define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))1269 # endif1270 1271 # ifndef IEM_WITH_SETJMP1272 /**1273 * Fetches the next opcode dword, zero extending it to a quad word.1274 *1275 * @returns Strict VBox status code.1276 * @param pVCpu The cross context virtual CPU structure of the calling thread.1277 * @param pu64 Where to return the opcode quad word.1278 */1279 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1280 {1281 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1282 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))1283 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);1284 1285 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1286 pVCpu->iem.s.abOpcode[offOpcode + 1],1287 pVCpu->iem.s.abOpcode[offOpcode + 2],1288 pVCpu->iem.s.abOpcode[offOpcode + 3]);1289 pVCpu->iem.s.offOpcode = offOpcode + 4;1290 return VINF_SUCCESS;1291 }1292 # endif /* !IEM_WITH_SETJMP */1293 1294 /**1295 * Fetches the next opcode dword and zero extends it to a quad word, returns1296 * automatically on failure.1297 *1298 * @param a_pu64 Where to return the opcode quad word.1299 * @remark Implicitly references pVCpu.1300 */1301 # ifndef IEM_WITH_SETJMP1302 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \1303 do \1304 { \1305 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \1306 if (rcStrict2 != VINF_SUCCESS) \1307 return rcStrict2; \1308 } while (0)1309 # else1310 # define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))1311 # endif1312 1313 1314 # ifndef IEM_WITH_SETJMP1315 /**1316 * Fetches the next signed double word from the opcode stream.1317 *1318 * @returns Strict VBox status code.1319 * @param pVCpu The cross context virtual CPU structure of the calling thread.1320 * @param pi32 Where to return the signed double word.1321 */1322 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT1323 {1324 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);1325 }1326 # endif1327 1328 /**1329 * Fetches the next signed double word from the opcode stream, returning1330 * automatically on failure.1331 *1332 * @param a_pi32 Where to return the signed double word.1333 * @remark Implicitly references pVCpu.1334 */1335 # ifndef IEM_WITH_SETJMP1336 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) \1337 do \1338 { \1339 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \1340 if (rcStrict2 != VINF_SUCCESS) \1341 return rcStrict2; \1342 } while (0)1343 # else1344 # define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))1345 # endif1346 1347 # ifndef IEM_WITH_SETJMP1348 /**1349 * Fetches the next opcode dword, sign extending it into a quad word.1350 *1351 * @returns Strict VBox status code.1352 * @param pVCpu The cross context virtual CPU structure of the calling thread.1353 * @param pu64 Where to return the opcode quad word.1354 */1355 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1356 {1357 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;1358 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))1359 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);1360 1361 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1362 pVCpu->iem.s.abOpcode[offOpcode + 1],1363 pVCpu->iem.s.abOpcode[offOpcode + 2],1364 pVCpu->iem.s.abOpcode[offOpcode + 3]);1365 *pu64 = (uint64_t)(int64_t)i32;1366 pVCpu->iem.s.offOpcode = offOpcode + 4;1367 return VINF_SUCCESS;1368 }1369 # endif /* !IEM_WITH_SETJMP */1370 1371 /**1372 * Fetches the next opcode double word and sign extends it to a quad word,1373 * returns automatically on failure.1374 *1375 * @param a_pu64 Where to return the opcode quad word.1376 * @remark Implicitly references pVCpu.1377 */1378 # ifndef IEM_WITH_SETJMP1379 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \1380 do \1381 { \1382 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \1383 if (rcStrict2 != VINF_SUCCESS) \1384 return rcStrict2; \1385 } while (0)1386 # else1387 # define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))1388 # endif1389 1390 # ifndef IEM_WITH_SETJMP1391 1392 /**1393 * Fetches the next opcode qword.1394 *1395 * @returns Strict VBox status code.1396 * @param pVCpu The cross context virtual CPU structure of the calling thread.1397 * @param pu64 Where to return the opcode qword.1398 */1399 DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT1400 {1401 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1402 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))1403 {1404 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1405 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1406 # else1407 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1408 pVCpu->iem.s.abOpcode[offOpcode + 1],1409 pVCpu->iem.s.abOpcode[offOpcode + 2],1410 pVCpu->iem.s.abOpcode[offOpcode + 3],1411 pVCpu->iem.s.abOpcode[offOpcode + 4],1412 pVCpu->iem.s.abOpcode[offOpcode + 5],1413 pVCpu->iem.s.abOpcode[offOpcode + 6],1414 pVCpu->iem.s.abOpcode[offOpcode + 7]);1415 # endif1416 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;1417 return VINF_SUCCESS;1418 }1419 return iemOpcodeGetNextU64Slow(pVCpu, pu64);1420 }1421 1422 # else /* IEM_WITH_SETJMP */1423 1424 /**1425 * Fetches the next opcode qword, longjmp on error.1426 *1427 * @returns The opcode qword.1428 * @param pVCpu The cross context virtual CPU structure of the calling thread.1429 */1430 DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP1431 {1432 # ifdef IEM_WITH_CODE_TLB1433 uint64_t u64Ret;1434 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;1435 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;1436 if (RT_LIKELY( pbBuf != NULL1437 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))1438 {1439 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;1440 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1441 u64Ret = *(uint64_t const *)&pbBuf[offBuf];1442 # else1443 u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],1444 pbBuf[offBuf + 1],1445 pbBuf[offBuf + 2],1446 pbBuf[offBuf + 3],1447 pbBuf[offBuf + 4],1448 pbBuf[offBuf + 5],1449 pbBuf[offBuf + 6],1450 pbBuf[offBuf + 7]);1451 # endif1452 }1453 else1454 u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);1455 1456 # ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF1457 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1458 Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));1459 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1460 *(uint64_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;1461 # else1462 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);1463 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);1464 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);1465 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);1466 pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);1467 pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);1468 pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);1469 pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);1470 # endif1471 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;1472 # endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */1473 1474 return u64Ret;1475 1476 # else /* !IEM_WITH_CODE_TLB */1477 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;1478 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))1479 {1480 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;1481 # ifdef IEM_USE_UNALIGNED_DATA_ACCESS1482 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];1483 # else1484 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],1485 pVCpu->iem.s.abOpcode[offOpcode + 1],1486 pVCpu->iem.s.abOpcode[offOpcode + 2],1487 pVCpu->iem.s.abOpcode[offOpcode + 3],1488 pVCpu->iem.s.abOpcode[offOpcode + 4],1489 pVCpu->iem.s.abOpcode[offOpcode + 5],1490 pVCpu->iem.s.abOpcode[offOpcode + 6],1491 pVCpu->iem.s.abOpcode[offOpcode + 7]);1492 # endif1493 }1494 return iemOpcodeGetNextU64SlowJmp(pVCpu);1495 # endif /* !IEM_WITH_CODE_TLB */1496 }1497 1498 # endif /* IEM_WITH_SETJMP */1499 1500 /**1501 * Fetches the next opcode quad word, returns automatically on failure.1502 *1503 * @param a_pu64 Where to return the opcode quad word.1504 * @remark Implicitly references pVCpu.1505 */1506 # ifndef IEM_WITH_SETJMP1507 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) \1508 do \1509 { \1510 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \1511 if (rcStrict2 != VINF_SUCCESS) \1512 return rcStrict2; \1513 } while (0)1514 # else1515 # define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )1516 # endif1517 1518 /**1519 * For fetching the opcode bytes for an ModR/M effective address, but throw1520 * away the result.1521 *1522 * This is used when decoding undefined opcodes and such where we want to avoid1523 * unnecessary MC blocks.1524 *1525 * @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is1526 * used instead. At least for now...1527 */1528 # ifndef IEM_WITH_SETJMP1529 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \1530 RTGCPTR GCPtrEff; \1531 VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \1532 if (rcStrict != VINF_SUCCESS) \1533 return rcStrict; \1534 } while (0)1535 # else1536 # define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \1537 (void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \1538 } while (0)1539 # endif1540 1541 285 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */ 1542 286 1543 287 1544 /** @name Misc Worker Functions. 288 289 /** @name Memory access. 290 * 1545 291 * @{ 1546 292 */ 1547 1548 /**1549 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or1550 * not (kind of obsolete now).1551 *1552 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1553 */1554 #define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )1555 1556 /**1557 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).1558 *1559 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.1560 * @param a_fEfl The new EFLAGS.1561 */1562 #define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)1563 1564 1565 /**1566 * Loads a NULL data selector into a selector register, both the hidden and1567 * visible parts, in protected mode.1568 *1569 * @param pVCpu The cross context virtual CPU structure of the calling thread.1570 * @param pSReg Pointer to the segment register.1571 * @param uRpl The RPL.1572 */1573 DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT1574 {1575 /** @todo Testcase: write a testcase checking what happends when loading a NULL1576 * data selector in protected mode. */1577 pSReg->Sel = uRpl;1578 pSReg->ValidSel = uRpl;1579 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;1580 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))1581 {1582 /* VT-x (Intel 3960x) observed doing something like this. */1583 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);1584 pSReg->u32Limit = UINT32_MAX;1585 pSReg->u64Base = 0;1586 }1587 else1588 {1589 pSReg->Attr.u = X86DESCATTR_UNUSABLE;1590 pSReg->u32Limit = 0;1591 pSReg->u64Base = 0;1592 }1593 }1594 1595 /** @} */1596 1597 1598 /*1599 *1600 * Helpers routines.1601 * Helpers routines.1602 * Helpers routines.1603 *1604 */1605 1606 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1607 1608 /**1609 * Recalculates the effective operand size.1610 *1611 * @param pVCpu The cross context virtual CPU structure of the calling thread.1612 */1613 DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT1614 {1615 switch (IEM_GET_CPU_MODE(pVCpu))1616 {1617 case IEMMODE_16BIT:1618 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;1619 break;1620 case IEMMODE_32BIT:1621 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;1622 break;1623 case IEMMODE_64BIT:1624 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))1625 {1626 case 0:1627 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;1628 break;1629 case IEM_OP_PRF_SIZE_OP:1630 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1631 break;1632 case IEM_OP_PRF_SIZE_REX_W:1633 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:1634 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1635 break;1636 }1637 break;1638 default:1639 AssertFailed();1640 }1641 }1642 1643 1644 /**1645 * Sets the default operand size to 64-bit and recalculates the effective1646 * operand size.1647 *1648 * @param pVCpu The cross context virtual CPU structure of the calling thread.1649 */1650 DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT1651 {1652 Assert(IEM_IS_64BIT_CODE(pVCpu));1653 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1654 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)1655 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1656 else1657 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1658 }1659 1660 1661 /**1662 * Sets the default operand size to 64-bit and recalculates the effective1663 * operand size, with intel ignoring any operand size prefix (AMD respects it).1664 *1665 * This is for the relative jumps.1666 *1667 * @param pVCpu The cross context virtual CPU structure of the calling thread.1668 */1669 DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT1670 {1671 Assert(IEM_IS_64BIT_CODE(pVCpu));1672 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;1673 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP1674 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)1675 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;1676 else1677 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;1678 }1679 1680 #endif /* !IEM_WITH_OPAQUE_DECODER_STATE */1681 1682 1683 1684 /** @name Register Access.1685 * @{1686 */1687 1688 /**1689 * Gets a reference (pointer) to the specified hidden segment register.1690 *1691 * @returns Hidden register reference.1692 * @param pVCpu The cross context virtual CPU structure of the calling thread.1693 * @param iSegReg The segment register.1694 */1695 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1696 {1697 Assert(iSegReg < X86_SREG_COUNT);1698 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1699 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];1700 1701 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));1702 return pSReg;1703 }1704 1705 1706 /**1707 * Ensures that the given hidden segment register is up to date.1708 *1709 * @returns Hidden register reference.1710 * @param pVCpu The cross context virtual CPU structure of the calling thread.1711 * @param pSReg The segment register.1712 */1713 DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT1714 {1715 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));1716 NOREF(pVCpu);1717 return pSReg;1718 }1719 1720 1721 /**1722 * Gets a reference (pointer) to the specified segment register (the selector1723 * value).1724 *1725 * @returns Pointer to the selector variable.1726 * @param pVCpu The cross context virtual CPU structure of the calling thread.1727 * @param iSegReg The segment register.1728 */1729 DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1730 {1731 Assert(iSegReg < X86_SREG_COUNT);1732 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1733 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;1734 }1735 1736 1737 /**1738 * Fetches the selector value of a segment register.1739 *1740 * @returns The selector value.1741 * @param pVCpu The cross context virtual CPU structure of the calling thread.1742 * @param iSegReg The segment register.1743 */1744 DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1745 {1746 Assert(iSegReg < X86_SREG_COUNT);1747 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1748 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;1749 }1750 1751 1752 /**1753 * Fetches the base address value of a segment register.1754 *1755 * @returns The selector value.1756 * @param pVCpu The cross context virtual CPU structure of the calling thread.1757 * @param iSegReg The segment register.1758 */1759 DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1760 {1761 Assert(iSegReg < X86_SREG_COUNT);1762 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1763 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;1764 }1765 1766 1767 /**1768 * Gets a reference (pointer) to the specified general purpose register.1769 *1770 * @returns Register reference.1771 * @param pVCpu The cross context virtual CPU structure of the calling thread.1772 * @param iReg The general purpose register.1773 */1774 DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1775 {1776 Assert(iReg < 16);1777 return &pVCpu->cpum.GstCtx.aGRegs[iReg];1778 }1779 1780 1781 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1782 /**1783 * Gets a reference (pointer) to the specified 8-bit general purpose register.1784 *1785 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.1786 *1787 * @returns Register reference.1788 * @param pVCpu The cross context virtual CPU structure of the calling thread.1789 * @param iReg The register.1790 */1791 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1792 {1793 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REX | IEM_OP_PRF_VEX)))1794 {1795 Assert(iReg < 16);1796 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;1797 }1798 /* high 8-bit register. */1799 Assert(iReg < 8);1800 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;1801 }1802 #endif1803 1804 1805 /**1806 * Gets a reference (pointer) to the specified 8-bit general purpose register,1807 * alternative version with extended (20) register index.1808 *1809 * @returns Register reference.1810 * @param pVCpu The cross context virtual CPU structure of the calling thread.1811 * @param iRegEx The register. The 16 first are regular ones,1812 * whereas 16 thru 19 maps to AH, CH, DH and BH.1813 */1814 DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT1815 {1816 /** @todo This could be done by double indexing on little endian hosts:1817 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */1818 if (iRegEx < 16)1819 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;1820 1821 /* high 8-bit register. */1822 Assert(iRegEx < 20);1823 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;1824 }1825 1826 1827 /**1828 * Gets a reference (pointer) to the specified 16-bit general purpose register.1829 *1830 * @returns Register reference.1831 * @param pVCpu The cross context virtual CPU structure of the calling thread.1832 * @param iReg The register.1833 */1834 DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1835 {1836 Assert(iReg < 16);1837 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;1838 }1839 1840 1841 /**1842 * Gets a reference (pointer) to the specified 32-bit general purpose register.1843 *1844 * @returns Register reference.1845 * @param pVCpu The cross context virtual CPU structure of the calling thread.1846 * @param iReg The register.1847 */1848 DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1849 {1850 Assert(iReg < 16);1851 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1852 }1853 1854 1855 /**1856 * Gets a reference (pointer) to the specified signed 32-bit general purpose register.1857 *1858 * @returns Register reference.1859 * @param pVCpu The cross context virtual CPU structure of the calling thread.1860 * @param iReg The register.1861 */1862 DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1863 {1864 Assert(iReg < 16);1865 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1866 }1867 1868 1869 /**1870 * Gets a reference (pointer) to the specified 64-bit general purpose register.1871 *1872 * @returns Register reference.1873 * @param pVCpu The cross context virtual CPU structure of the calling thread.1874 * @param iReg The register.1875 */1876 DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1877 {1878 Assert(iReg < 64);1879 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1880 }1881 1882 1883 /**1884 * Gets a reference (pointer) to the specified signed 64-bit general purpose register.1885 *1886 * @returns Register reference.1887 * @param pVCpu The cross context virtual CPU structure of the calling thread.1888 * @param iReg The register.1889 */1890 DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1891 {1892 Assert(iReg < 16);1893 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1894 }1895 1896 1897 /**1898 * Gets a reference (pointer) to the specified segment register's base address.1899 *1900 * @returns Segment register base address reference.1901 * @param pVCpu The cross context virtual CPU structure of the calling thread.1902 * @param iSegReg The segment selector.1903 */1904 DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT1905 {1906 Assert(iSegReg < X86_SREG_COUNT);1907 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));1908 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;1909 }1910 1911 1912 #ifndef IEM_WITH_OPAQUE_DECODER_STATE1913 /**1914 * Fetches the value of a 8-bit general purpose register.1915 *1916 * @returns The register value.1917 * @param pVCpu The cross context virtual CPU structure of the calling thread.1918 * @param iReg The register.1919 */1920 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1921 {1922 return *iemGRegRefU8(pVCpu, iReg);1923 }1924 #endif1925 1926 1927 /**1928 * Fetches the value of a 8-bit general purpose register, alternative version1929 * with extended (20) register index.1930 1931 * @returns The register value.1932 * @param pVCpu The cross context virtual CPU structure of the calling thread.1933 * @param iRegEx The register. The 16 first are regular ones,1934 * whereas 16 thru 19 maps to AH, CH, DH and BH.1935 */1936 DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT1937 {1938 return *iemGRegRefU8Ex(pVCpu, iRegEx);1939 }1940 1941 1942 /**1943 * Fetches the value of a 16-bit general purpose register.1944 *1945 * @returns The register value.1946 * @param pVCpu The cross context virtual CPU structure of the calling thread.1947 * @param iReg The register.1948 */1949 DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1950 {1951 Assert(iReg < 16);1952 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;1953 }1954 1955 1956 /**1957 * Fetches the value of a 32-bit general purpose register.1958 *1959 * @returns The register value.1960 * @param pVCpu The cross context virtual CPU structure of the calling thread.1961 * @param iReg The register.1962 */1963 DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1964 {1965 Assert(iReg < 16);1966 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;1967 }1968 1969 1970 /**1971 * Fetches the value of a 64-bit general purpose register.1972 *1973 * @returns The register value.1974 * @param pVCpu The cross context virtual CPU structure of the calling thread.1975 * @param iReg The register.1976 */1977 DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT1978 {1979 Assert(iReg < 16);1980 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;1981 }1982 1983 1984 /**1985 * Stores a 16-bit value to a general purpose register.1986 *1987 * @param pVCpu The cross context virtual CPU structure of the calling thread.1988 * @param iReg The register.1989 * @param uValue The value to store.1990 */1991 DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT1992 {1993 Assert(iReg < 16);1994 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;1995 }1996 1997 1998 /**1999 * Stores a 32-bit value to a general purpose register, implicitly clearing high2000 * values.2001 *2002 * @param pVCpu The cross context virtual CPU structure of the calling thread.2003 * @param iReg The register.2004 * @param uValue The value to store.2005 */2006 DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT2007 {2008 Assert(iReg < 16);2009 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;2010 }2011 2012 2013 /**2014 * Stores a 64-bit value to a general purpose register.2015 *2016 * @param pVCpu The cross context virtual CPU structure of the calling thread.2017 * @param iReg The register.2018 * @param uValue The value to store.2019 */2020 DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT2021 {2022 Assert(iReg < 16);2023 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;2024 }2025 2026 2027 /**2028 * Get the address of the top of the stack.2029 *2030 * @param pVCpu The cross context virtual CPU structure of the calling thread.2031 */2032 DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT2033 {2034 if (IEM_IS_64BIT_CODE(pVCpu))2035 return pVCpu->cpum.GstCtx.rsp;2036 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)2037 return pVCpu->cpum.GstCtx.esp;2038 return pVCpu->cpum.GstCtx.sp;2039 }2040 2041 2042 /**2043 * Updates the RIP/EIP/IP to point to the next instruction.2044 *2045 * @param pVCpu The cross context virtual CPU structure of the calling thread.2046 * @param cbInstr The number of bytes to add.2047 */2048 DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT2049 {2050 /*2051 * Advance RIP.2052 *2053 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,2054 * while in all other modes except LM64 the updates are 32-bit. This means2055 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e.2056 * 4GB and 64KB rollovers, and decide whether anything needs masking.2057 *2058 * See PC wrap around tests in bs3-cpu-weird-1.2059 */2060 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;2061 uint64_t const uRipNext = uRipPrev + cbInstr;2062 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16)))2063 || IEM_IS_64BIT_CODE(pVCpu)))2064 pVCpu->cpum.GstCtx.rip = uRipNext;2065 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)2066 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;2067 else2068 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;2069 }2070 2071 2072 /**2073 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the2074 * following EFLAGS bits are set:2075 * - X86_EFL_RF - clear it.2076 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.2077 * - X86_EFL_TF - generate single step \#DB trap.2078 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not2079 * instruction).2080 *2081 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},2082 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint2083 * takes priority over both NMIs and hardware interrupts. So, neither is2084 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are2085 * either unsupported will be triggered on-top of any \#DB raised here.)2086 *2087 * The RF flag only needs to be cleared here as it only suppresses instruction2088 * breakpoints which are not raised here (happens synchronously during2089 * instruction fetching).2090 *2091 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its2092 * status has no bearing on whether \#DB exceptions are raised.2093 *2094 * @note This must *NOT* be called by the two instructions setting the2095 * CPUMCTX_INHIBIT_SHADOW_SS flag.2096 *2097 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}2098 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching2099 * Stacks}2100 */2101 template<uint32_t const a_fTF = X86_EFL_TF>2102 static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2103 {2104 /*2105 * Normally we're just here to clear RF and/or interrupt shadow bits.2106 */2107 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0))2108 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);2109 else2110 {2111 /*2112 * Raise a #DB or/and DBGF event.2113 */2114 VBOXSTRICTRC rcStrict;2115 if (pVCpu->cpum.GstCtx.eflags.uBoth & (a_fTF | CPUMCTX_DBG_HIT_DRX_MASK))2116 {2117 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);2118 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;2119 if (pVCpu->cpum.GstCtx.eflags.uBoth & a_fTF)2120 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS;2121 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)2122 >> CPUMCTX_DBG_HIT_DRX_SHIFT;2123 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",2124 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],2125 pVCpu->cpum.GstCtx.rflags.uBoth));2126 2127 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK);2128 rcStrict = iemRaiseDebugException(pVCpu);2129 2130 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */2131 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))2132 {2133 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;2134 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));2135 }2136 }2137 else2138 {2139 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);2140 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;2141 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));2142 }2143 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;2144 Assert(rcStrict != VINF_SUCCESS);2145 return rcStrict;2146 }2147 return rcNormal;2148 }2149 2150 2151 /**2152 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.2153 *2154 * @param pVCpu The cross context virtual CPU structure of the calling thread.2155 * @param rcNormal VINF_SUCCESS to continue TB.2156 * VINF_IEM_REEXEC_BREAK to force TB exit when2157 * taking the wrong conditional branhc.2158 */2159 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2160 {2161 /*2162 * We assume that most of the time nothing actually needs doing here.2163 */2164 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);2165 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth2166 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))2167 return rcNormal;2168 return iemFinishInstructionWithFlagsSet(pVCpu, rcNormal);2169 }2170 2171 2172 /**2173 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF2174 * and CPUMCTX_INHIBIT_SHADOW.2175 *2176 * @param pVCpu The cross context virtual CPU structure of the calling thread.2177 * @param cbInstr The number of bytes to add.2178 */2179 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT2180 {2181 iemRegAddToRip(pVCpu, cbInstr);2182 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);2183 }2184 2185 2186 /**2187 * Updates the RIP to point to the next instruction and clears EFLAGS.RF2188 * and CPUMCTX_INHIBIT_SHADOW.2189 *2190 * Only called from 64-bit code.2191 *2192 * @param pVCpu The cross context virtual CPU structure of the calling thread.2193 * @param cbInstr The number of bytes to add.2194 * @param rcNormal VINF_SUCCESS to continue TB.2195 * VINF_IEM_REEXEC_BREAK to force TB exit when2196 * taking the wrong conditional branhc.2197 */2198 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2199 {2200 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;2201 return iemRegFinishClearingRF(pVCpu, rcNormal);2202 }2203 2204 2205 /**2206 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and2207 * CPUMCTX_INHIBIT_SHADOW.2208 *2209 * This is never from 64-bit code.2210 *2211 * @param pVCpu The cross context virtual CPU structure of the calling thread.2212 * @param cbInstr The number of bytes to add.2213 * @param rcNormal VINF_SUCCESS to continue TB.2214 * VINF_IEM_REEXEC_BREAK to force TB exit when2215 * taking the wrong conditional branhc.2216 */2217 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2218 {2219 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);2220 return iemRegFinishClearingRF(pVCpu, rcNormal);2221 }2222 2223 2224 /**2225 * Updates the IP to point to the next instruction and clears EFLAGS.RF and2226 * CPUMCTX_INHIBIT_SHADOW.2227 *2228 * This is only ever used from 16-bit code on a pre-386 CPU.2229 *2230 * @param pVCpu The cross context virtual CPU structure of the calling thread.2231 * @param cbInstr The number of bytes to add.2232 * @param rcNormal VINF_SUCCESS to continue TB.2233 * VINF_IEM_REEXEC_BREAK to force TB exit when2234 * taking the wrong conditional branhc.2235 */2236 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2237 {2238 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);2239 return iemRegFinishClearingRF(pVCpu, rcNormal);2240 }2241 2242 2243 /**2244 * Tail method for a finish function that does't clear flags or raise \#DB.2245 *2246 * @param pVCpu The cross context virtual CPU structure of the calling thread.2247 * @param rcNormal VINF_SUCCESS to continue TB.2248 * VINF_IEM_REEXEC_BREAK to force TB exit when2249 * taking the wrong conditional branhc.2250 */2251 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT2252 {2253 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);2254 Assert(!( pVCpu->cpum.GstCtx.eflags.uBoth2255 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) );2256 RT_NOREF(pVCpu);2257 return rcNormal;2258 }2259 2260 2261 /**2262 * Updates the RIP to point to the next instruction, but does not need to clear2263 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2264 *2265 * Only called from 64-bit code.2266 *2267 * @param pVCpu The cross context virtual CPU structure of the calling thread.2268 * @param cbInstr The number of bytes to add.2269 * @param rcNormal VINF_SUCCESS to continue TB.2270 * VINF_IEM_REEXEC_BREAK to force TB exit when2271 * taking the wrong conditional branhc.2272 */2273 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2274 {2275 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;2276 return iemRegFinishNoFlags(pVCpu, rcNormal);2277 }2278 2279 2280 /**2281 * Updates the EIP to point to the next instruction, but does not need to clear2282 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2283 *2284 * This is never from 64-bit code.2285 *2286 * @param pVCpu The cross context virtual CPU structure of the calling thread.2287 * @param cbInstr The number of bytes to add.2288 * @param rcNormal VINF_SUCCESS to continue TB.2289 * VINF_IEM_REEXEC_BREAK to force TB exit when2290 * taking the wrong conditional branhc.2291 */2292 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2293 {2294 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);2295 return iemRegFinishNoFlags(pVCpu, rcNormal);2296 }2297 2298 2299 /**2300 * Updates the IP to point to the next instruction, but does not need to clear2301 * EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.2302 *2303 * This is only ever used from 16-bit code on a pre-386 CPU.2304 *2305 * @param pVCpu The cross context virtual CPU structure of the calling thread.2306 * @param cbInstr The number of bytes to add.2307 * @param rcNormal VINF_SUCCESS to continue TB.2308 * VINF_IEM_REEXEC_BREAK to force TB exit when2309 * taking the wrong conditional branhc.2310 *2311 */2312 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int rcNormal) RT_NOEXCEPT2313 {2314 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);2315 return iemRegFinishNoFlags(pVCpu, rcNormal);2316 }2317 2318 2319 /**2320 * Adds a 8-bit signed jump offset to RIP from 64-bit code.2321 *2322 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2323 * segment limit.2324 *2325 * @param pVCpu The cross context virtual CPU structure of the calling thread.2326 * @param cbInstr Instruction size.2327 * @param offNextInstr The offset of the next instruction.2328 * @param enmEffOpSize Effective operand size.2329 * @param rcNormal VINF_SUCCESS to continue TB.2330 * VINF_IEM_REEXEC_BREAK to force TB exit when2331 * taking the wrong conditional branhc.2332 */2333 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2334 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2335 {2336 Assert(IEM_IS_64BIT_CODE(pVCpu));2337 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);2338 2339 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2340 if (enmEffOpSize == IEMMODE_16BIT)2341 uNewRip &= UINT16_MAX;2342 2343 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2344 pVCpu->cpum.GstCtx.rip = uNewRip;2345 else2346 return iemRaiseGeneralProtectionFault0(pVCpu);2347 2348 #ifndef IEM_WITH_CODE_TLB2349 iemOpcodeFlushLight(pVCpu, cbInstr);2350 #endif2351 2352 /*2353 * Clear RF and finish the instruction (maybe raise #DB).2354 */2355 return iemRegFinishClearingRF(pVCpu, rcNormal);2356 }2357 2358 2359 /**2360 * Adds a 8-bit signed jump offset to RIP from 64-bit code when the caller is2361 * sure it stays within the same page.2362 *2363 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2364 * segment limit.2365 *2366 * @param pVCpu The cross context virtual CPU structure of the calling thread.2367 * @param cbInstr Instruction size.2368 * @param offNextInstr The offset of the next instruction.2369 * @param enmEffOpSize Effective operand size.2370 * @param rcNormal VINF_SUCCESS to continue TB.2371 * VINF_IEM_REEXEC_BREAK to force TB exit when2372 * taking the wrong conditional branhc.2373 */2374 DECL_FORCE_INLINE(VBOXSTRICTRC)2375 iemRegRip64RelativeJumpS8IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2376 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2377 {2378 Assert(IEM_IS_64BIT_CODE(pVCpu));2379 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);2380 2381 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2382 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2383 pVCpu->cpum.GstCtx.rip = uNewRip;2384 2385 #ifndef IEM_WITH_CODE_TLB2386 iemOpcodeFlushLight(pVCpu, cbInstr);2387 #endif2388 2389 /*2390 * Clear RF and finish the instruction (maybe raise #DB).2391 */2392 return iemRegFinishClearingRF(pVCpu, rcNormal);2393 }2394 2395 2396 /**2397 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit2398 * code (never 64-bit).2399 *2400 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2401 * segment limit.2402 *2403 * @param pVCpu The cross context virtual CPU structure of the calling thread.2404 * @param cbInstr Instruction size.2405 * @param offNextInstr The offset of the next instruction.2406 * @param enmEffOpSize Effective operand size.2407 * @param rcNormal VINF_SUCCESS to continue TB.2408 * VINF_IEM_REEXEC_BREAK to force TB exit when2409 * taking the wrong conditional branhc.2410 */2411 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2412 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2413 {2414 Assert(!IEM_IS_64BIT_CODE(pVCpu));2415 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2416 2417 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2418 if (enmEffOpSize == IEMMODE_16BIT)2419 uNewEip &= UINT16_MAX;2420 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2421 pVCpu->cpum.GstCtx.rip = uNewEip;2422 else2423 return iemRaiseGeneralProtectionFault0(pVCpu);2424 2425 #ifndef IEM_WITH_CODE_TLB2426 iemOpcodeFlushLight(pVCpu, cbInstr);2427 #endif2428 2429 /*2430 * Clear RF and finish the instruction (maybe raise #DB).2431 */2432 return iemRegFinishClearingRF(pVCpu, rcNormal);2433 }2434 2435 2436 /**2437 * Adds a 8-bit signed jump offset to EIP, on 386 or later from FLAT 32-bit code2438 * (never 64-bit).2439 *2440 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2441 * segment limit.2442 *2443 * @param pVCpu The cross context virtual CPU structure of the calling thread.2444 * @param cbInstr Instruction size.2445 * @param offNextInstr The offset of the next instruction.2446 * @param enmEffOpSize Effective operand size.2447 * @param rcNormal VINF_SUCCESS to continue TB.2448 * VINF_IEM_REEXEC_BREAK to force TB exit when2449 * taking the wrong conditional branhc.2450 */2451 DECL_FORCE_INLINE(VBOXSTRICTRC)2452 iemRegEip32RelativeJumpS8FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2453 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2454 {2455 Assert(!IEM_IS_64BIT_CODE(pVCpu));2456 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2457 2458 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2459 if (enmEffOpSize == IEMMODE_16BIT)2460 uNewEip &= UINT16_MAX;2461 pVCpu->cpum.GstCtx.rip = uNewEip;2462 2463 #ifndef IEM_WITH_CODE_TLB2464 iemOpcodeFlushLight(pVCpu, cbInstr);2465 #endif2466 2467 /*2468 * Clear RF and finish the instruction (maybe raise #DB).2469 */2470 return iemRegFinishClearingRF(pVCpu, rcNormal);2471 }2472 2473 2474 /**2475 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.2476 *2477 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2478 * segment limit.2479 *2480 * @param pVCpu The cross context virtual CPU structure of the calling thread.2481 * @param cbInstr Instruction size.2482 * @param offNextInstr The offset of the next instruction.2483 * @param rcNormal VINF_SUCCESS to continue TB.2484 * VINF_IEM_REEXEC_BREAK to force TB exit when2485 * taking the wrong conditional branhc.2486 */2487 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2488 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT2489 {2490 Assert(!IEM_IS_64BIT_CODE(pVCpu));2491 2492 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;2493 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2494 pVCpu->cpum.GstCtx.rip = uNewIp;2495 else2496 return iemRaiseGeneralProtectionFault0(pVCpu);2497 2498 #ifndef IEM_WITH_CODE_TLB2499 iemOpcodeFlushLight(pVCpu, cbInstr);2500 #endif2501 2502 /*2503 * Clear RF and finish the instruction (maybe raise #DB).2504 */2505 return iemRegFinishClearingRF(pVCpu, rcNormal);2506 }2507 2508 2509 /**2510 * Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or2511 * clearing of flags.2512 *2513 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2514 * segment limit.2515 *2516 * @param pVCpu The cross context virtual CPU structure of the calling thread.2517 * @param cbInstr Instruction size.2518 * @param offNextInstr The offset of the next instruction.2519 * @param enmEffOpSize Effective operand size.2520 * @param rcNormal VINF_SUCCESS to continue TB.2521 * VINF_IEM_REEXEC_BREAK to force TB exit when2522 * taking the wrong conditional branhc.2523 */2524 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2525 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2526 {2527 Assert(IEM_IS_64BIT_CODE(pVCpu));2528 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);2529 2530 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2531 if (enmEffOpSize == IEMMODE_16BIT)2532 uNewRip &= UINT16_MAX;2533 2534 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2535 pVCpu->cpum.GstCtx.rip = uNewRip;2536 else2537 return iemRaiseGeneralProtectionFault0(pVCpu);2538 2539 #ifndef IEM_WITH_CODE_TLB2540 iemOpcodeFlushLight(pVCpu, cbInstr);2541 #endif2542 return iemRegFinishNoFlags(pVCpu, rcNormal);2543 }2544 2545 2546 /**2547 * Adds a 8-bit signed jump offset to RIP from 64-bit code when caller is sure2548 * it stays within the same page, no checking or clearing of flags.2549 *2550 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2551 * segment limit.2552 *2553 * @param pVCpu The cross context virtual CPU structure of the calling thread.2554 * @param cbInstr Instruction size.2555 * @param offNextInstr The offset of the next instruction.2556 * @param enmEffOpSize Effective operand size.2557 * @param rcNormal VINF_SUCCESS to continue TB.2558 * VINF_IEM_REEXEC_BREAK to force TB exit when2559 * taking the wrong conditional branhc.2560 */2561 DECL_FORCE_INLINE(VBOXSTRICTRC)2562 iemRegRip64RelativeJumpS8IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2563 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2564 {2565 Assert(IEM_IS_64BIT_CODE(pVCpu));2566 Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);2567 2568 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2569 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2570 pVCpu->cpum.GstCtx.rip = uNewRip;2571 2572 #ifndef IEM_WITH_CODE_TLB2573 iemOpcodeFlushLight(pVCpu, cbInstr);2574 #endif2575 return iemRegFinishNoFlags(pVCpu, rcNormal);2576 }2577 2578 2579 /**2580 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit2581 * code (never 64-bit), no checking or clearing of flags.2582 *2583 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2584 * segment limit.2585 *2586 * @param pVCpu The cross context virtual CPU structure of the calling thread.2587 * @param cbInstr Instruction size.2588 * @param offNextInstr The offset of the next instruction.2589 * @param enmEffOpSize Effective operand size.2590 * @param rcNormal VINF_SUCCESS to continue TB.2591 * VINF_IEM_REEXEC_BREAK to force TB exit when2592 * taking the wrong conditional branhc.2593 */2594 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2595 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2596 {2597 Assert(!IEM_IS_64BIT_CODE(pVCpu));2598 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2599 2600 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2601 if (enmEffOpSize == IEMMODE_16BIT)2602 uNewEip &= UINT16_MAX;2603 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2604 pVCpu->cpum.GstCtx.rip = uNewEip;2605 else2606 return iemRaiseGeneralProtectionFault0(pVCpu);2607 2608 #ifndef IEM_WITH_CODE_TLB2609 iemOpcodeFlushLight(pVCpu, cbInstr);2610 #endif2611 return iemRegFinishNoFlags(pVCpu, rcNormal);2612 }2613 2614 2615 /**2616 * Adds a 8-bit signed jump offset to EIP, on 386 or later from flat 32-bit code2617 * (never 64-bit), no checking or clearing of flags.2618 *2619 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2620 * segment limit.2621 *2622 * @param pVCpu The cross context virtual CPU structure of the calling thread.2623 * @param cbInstr Instruction size.2624 * @param offNextInstr The offset of the next instruction.2625 * @param enmEffOpSize Effective operand size.2626 * @param rcNormal VINF_SUCCESS to continue TB.2627 * VINF_IEM_REEXEC_BREAK to force TB exit when2628 * taking the wrong conditional branhc.2629 */2630 DECL_FORCE_INLINE(VBOXSTRICTRC)2631 iemRegEip32RelativeJumpS8FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,2632 IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT2633 {2634 Assert(!IEM_IS_64BIT_CODE(pVCpu));2635 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);2636 2637 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;2638 if (enmEffOpSize == IEMMODE_16BIT)2639 uNewEip &= UINT16_MAX;2640 pVCpu->cpum.GstCtx.rip = uNewEip;2641 2642 #ifndef IEM_WITH_CODE_TLB2643 iemOpcodeFlushLight(pVCpu, cbInstr);2644 #endif2645 return iemRegFinishNoFlags(pVCpu, rcNormal);2646 }2647 2648 2649 /**2650 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or2651 * clearing of flags.2652 *2653 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2654 * segment limit.2655 *2656 * @param pVCpu The cross context virtual CPU structure of the calling thread.2657 * @param cbInstr Instruction size.2658 * @param offNextInstr The offset of the next instruction.2659 * @param rcNormal VINF_SUCCESS to continue TB.2660 * VINF_IEM_REEXEC_BREAK to force TB exit when2661 * taking the wrong conditional branhc.2662 */2663 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2664 int8_t offNextInstr, int rcNormal) RT_NOEXCEPT2665 {2666 Assert(!IEM_IS_64BIT_CODE(pVCpu));2667 2668 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;2669 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2670 pVCpu->cpum.GstCtx.rip = uNewIp;2671 else2672 return iemRaiseGeneralProtectionFault0(pVCpu);2673 2674 #ifndef IEM_WITH_CODE_TLB2675 iemOpcodeFlushLight(pVCpu, cbInstr);2676 #endif2677 return iemRegFinishNoFlags(pVCpu, rcNormal);2678 }2679 2680 2681 /**2682 * Adds a 16-bit signed jump offset to RIP from 64-bit code.2683 *2684 * @returns Strict VBox status code.2685 * @param pVCpu The cross context virtual CPU structure of the calling thread.2686 * @param cbInstr Instruction size.2687 * @param offNextInstr The offset of the next instruction.2688 * @param rcNormal VINF_SUCCESS to continue TB.2689 * VINF_IEM_REEXEC_BREAK to force TB exit when2690 * taking the wrong conditional branhc.2691 */2692 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2693 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2694 {2695 Assert(IEM_IS_64BIT_CODE(pVCpu));2696 2697 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);2698 2699 #ifndef IEM_WITH_CODE_TLB2700 iemOpcodeFlushLight(pVCpu, cbInstr);2701 #endif2702 2703 /*2704 * Clear RF and finish the instruction (maybe raise #DB).2705 */2706 return iemRegFinishClearingRF(pVCpu, rcNormal);2707 }2708 2709 2710 /**2711 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.2712 *2713 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2714 * segment limit.2715 *2716 * @returns Strict VBox status code.2717 * @param pVCpu The cross context virtual CPU structure of the calling thread.2718 * @param cbInstr Instruction size.2719 * @param offNextInstr The offset of the next instruction.2720 * @param rcNormal VINF_SUCCESS to continue TB.2721 * VINF_IEM_REEXEC_BREAK to force TB exit when2722 * taking the wrong conditional branhc.2723 *2724 * @note This is also used by 16-bit code in pre-386 mode, as the code is2725 * identical.2726 */2727 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2728 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2729 {2730 Assert(!IEM_IS_64BIT_CODE(pVCpu));2731 2732 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2733 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2734 pVCpu->cpum.GstCtx.rip = uNewIp;2735 else2736 return iemRaiseGeneralProtectionFault0(pVCpu);2737 2738 #ifndef IEM_WITH_CODE_TLB2739 iemOpcodeFlushLight(pVCpu, cbInstr);2740 #endif2741 2742 /*2743 * Clear RF and finish the instruction (maybe raise #DB).2744 */2745 return iemRegFinishClearingRF(pVCpu, rcNormal);2746 }2747 2748 2749 /**2750 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code.2751 *2752 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2753 * segment limit.2754 *2755 * @returns Strict VBox status code.2756 * @param pVCpu The cross context virtual CPU structure of the calling thread.2757 * @param cbInstr Instruction size.2758 * @param offNextInstr The offset of the next instruction.2759 * @param rcNormal VINF_SUCCESS to continue TB.2760 * VINF_IEM_REEXEC_BREAK to force TB exit when2761 * taking the wrong conditional branhc.2762 *2763 * @note This is also used by 16-bit code in pre-386 mode, as the code is2764 * identical.2765 */2766 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2767 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2768 {2769 Assert(!IEM_IS_64BIT_CODE(pVCpu));2770 2771 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2772 pVCpu->cpum.GstCtx.rip = uNewIp;2773 2774 #ifndef IEM_WITH_CODE_TLB2775 iemOpcodeFlushLight(pVCpu, cbInstr);2776 #endif2777 2778 /*2779 * Clear RF and finish the instruction (maybe raise #DB).2780 */2781 return iemRegFinishClearingRF(pVCpu, rcNormal);2782 }2783 2784 2785 /**2786 * Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or2787 * clearing of flags.2788 *2789 * @returns Strict VBox status code.2790 * @param pVCpu The cross context virtual CPU structure of the calling thread.2791 * @param cbInstr Instruction size.2792 * @param offNextInstr The offset of the next instruction.2793 * @param rcNormal VINF_SUCCESS to continue TB.2794 * VINF_IEM_REEXEC_BREAK to force TB exit when2795 * taking the wrong conditional branhc.2796 */2797 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2798 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2799 {2800 Assert(IEM_IS_64BIT_CODE(pVCpu));2801 2802 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);2803 2804 #ifndef IEM_WITH_CODE_TLB2805 iemOpcodeFlushLight(pVCpu, cbInstr);2806 #endif2807 return iemRegFinishNoFlags(pVCpu, rcNormal);2808 }2809 2810 2811 /**2812 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code,2813 * no checking or clearing of flags.2814 *2815 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2816 * segment limit.2817 *2818 * @returns Strict VBox status code.2819 * @param pVCpu The cross context virtual CPU structure of the calling thread.2820 * @param cbInstr Instruction size.2821 * @param offNextInstr The offset of the next instruction.2822 * @param rcNormal VINF_SUCCESS to continue TB.2823 * VINF_IEM_REEXEC_BREAK to force TB exit when2824 * taking the wrong conditional branhc.2825 *2826 * @note This is also used by 16-bit code in pre-386 mode, as the code is2827 * identical.2828 */2829 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2830 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2831 {2832 Assert(!IEM_IS_64BIT_CODE(pVCpu));2833 2834 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2835 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))2836 pVCpu->cpum.GstCtx.rip = uNewIp;2837 else2838 return iemRaiseGeneralProtectionFault0(pVCpu);2839 2840 #ifndef IEM_WITH_CODE_TLB2841 iemOpcodeFlushLight(pVCpu, cbInstr);2842 #endif2843 return iemRegFinishNoFlags(pVCpu, rcNormal);2844 }2845 2846 2847 /**2848 * Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code, no checking or2849 * clearing of flags.2850 *2851 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2852 * segment limit.2853 *2854 * @returns Strict VBox status code.2855 * @param pVCpu The cross context virtual CPU structure of the calling thread.2856 * @param cbInstr Instruction size.2857 * @param offNextInstr The offset of the next instruction.2858 * @param rcNormal VINF_SUCCESS to continue TB.2859 * VINF_IEM_REEXEC_BREAK to force TB exit when2860 * taking the wrong conditional branhc.2861 *2862 * @note This is also used by 16-bit code in pre-386 mode, as the code is2863 * identical.2864 */2865 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,2866 int16_t offNextInstr, int rcNormal) RT_NOEXCEPT2867 {2868 Assert(!IEM_IS_64BIT_CODE(pVCpu));2869 2870 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;2871 pVCpu->cpum.GstCtx.rip = uNewIp;2872 2873 #ifndef IEM_WITH_CODE_TLB2874 iemOpcodeFlushLight(pVCpu, cbInstr);2875 #endif2876 return iemRegFinishNoFlags(pVCpu, rcNormal);2877 }2878 2879 2880 /**2881 * Adds a 32-bit signed jump offset to RIP from 64-bit code.2882 *2883 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2884 * segment limit.2885 *2886 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the2887 * only alternative for relative jumps in 64-bit code and that is already2888 * handled in the decoder stage.2889 *2890 * @returns Strict VBox status code.2891 * @param pVCpu The cross context virtual CPU structure of the calling thread.2892 * @param cbInstr Instruction size.2893 * @param offNextInstr The offset of the next instruction.2894 * @param rcNormal VINF_SUCCESS to continue TB.2895 * VINF_IEM_REEXEC_BREAK to force TB exit when2896 * taking the wrong conditional branhc.2897 */2898 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2899 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2900 {2901 Assert(IEM_IS_64BIT_CODE(pVCpu));2902 2903 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2904 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))2905 pVCpu->cpum.GstCtx.rip = uNewRip;2906 else2907 return iemRaiseGeneralProtectionFault0(pVCpu);2908 2909 #ifndef IEM_WITH_CODE_TLB2910 iemOpcodeFlushLight(pVCpu, cbInstr);2911 #endif2912 2913 /*2914 * Clear RF and finish the instruction (maybe raise #DB).2915 */2916 return iemRegFinishClearingRF(pVCpu, rcNormal);2917 }2918 2919 2920 /**2921 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is2922 * sure the target is in the same page.2923 *2924 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2925 * segment limit.2926 *2927 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the2928 * only alternative for relative jumps in 64-bit code and that is already2929 * handled in the decoder stage.2930 *2931 * @returns Strict VBox status code.2932 * @param pVCpu The cross context virtual CPU structure of the calling thread.2933 * @param cbInstr Instruction size.2934 * @param offNextInstr The offset of the next instruction.2935 * @param rcNormal VINF_SUCCESS to continue TB.2936 * VINF_IEM_REEXEC_BREAK to force TB exit when2937 * taking the wrong conditional branhc.2938 */2939 DECL_FORCE_INLINE(VBOXSTRICTRC)2940 iemRegRip64RelativeJumpS32IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2941 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2942 {2943 Assert(IEM_IS_64BIT_CODE(pVCpu));2944 2945 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;2946 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));2947 pVCpu->cpum.GstCtx.rip = uNewRip;2948 2949 #ifndef IEM_WITH_CODE_TLB2950 iemOpcodeFlushLight(pVCpu, cbInstr);2951 #endif2952 2953 /*2954 * Clear RF and finish the instruction (maybe raise #DB).2955 */2956 return iemRegFinishClearingRF(pVCpu, rcNormal);2957 }2958 2959 2960 /**2961 * Adds a 32-bit signed jump offset to RIP from 64-bit code.2962 *2963 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code2964 * segment limit.2965 *2966 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the2967 * only alternative for relative jumps in 32-bit code and that is already2968 * handled in the decoder stage.2969 *2970 * @returns Strict VBox status code.2971 * @param pVCpu The cross context virtual CPU structure of the calling thread.2972 * @param cbInstr Instruction size.2973 * @param offNextInstr The offset of the next instruction.2974 * @param rcNormal VINF_SUCCESS to continue TB.2975 * VINF_IEM_REEXEC_BREAK to force TB exit when2976 * taking the wrong conditional branhc.2977 */2978 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,2979 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT2980 {2981 Assert(!IEM_IS_64BIT_CODE(pVCpu));2982 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);2983 2984 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;2985 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))2986 pVCpu->cpum.GstCtx.rip = uNewEip;2987 else2988 return iemRaiseGeneralProtectionFault0(pVCpu);2989 2990 #ifndef IEM_WITH_CODE_TLB2991 iemOpcodeFlushLight(pVCpu, cbInstr);2992 #endif2993 2994 /*2995 * Clear RF and finish the instruction (maybe raise #DB).2996 */2997 return iemRegFinishClearingRF(pVCpu, rcNormal);2998 }2999 3000 3001 /**3002 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code.3003 *3004 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3005 * segment limit.3006 *3007 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3008 * only alternative for relative jumps in 32-bit code and that is already3009 * handled in the decoder stage.3010 *3011 * @returns Strict VBox status code.3012 * @param pVCpu The cross context virtual CPU structure of the calling thread.3013 * @param cbInstr Instruction size.3014 * @param offNextInstr The offset of the next instruction.3015 * @param rcNormal VINF_SUCCESS to continue TB.3016 * VINF_IEM_REEXEC_BREAK to force TB exit when3017 * taking the wrong conditional branhc.3018 */3019 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,3020 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3021 {3022 Assert(!IEM_IS_64BIT_CODE(pVCpu));3023 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3024 3025 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3026 pVCpu->cpum.GstCtx.rip = uNewEip;3027 3028 #ifndef IEM_WITH_CODE_TLB3029 iemOpcodeFlushLight(pVCpu, cbInstr);3030 #endif3031 3032 /*3033 * Clear RF and finish the instruction (maybe raise #DB).3034 */3035 return iemRegFinishClearingRF(pVCpu, rcNormal);3036 }3037 3038 3039 3040 /**3041 * Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or3042 * clearing of flags.3043 *3044 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3045 * segment limit.3046 *3047 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the3048 * only alternative for relative jumps in 64-bit code and that is already3049 * handled in the decoder stage.3050 *3051 * @returns Strict VBox status code.3052 * @param pVCpu The cross context virtual CPU structure of the calling thread.3053 * @param cbInstr Instruction size.3054 * @param offNextInstr The offset of the next instruction.3055 * @param rcNormal VINF_SUCCESS to continue TB.3056 * VINF_IEM_REEXEC_BREAK to force TB exit when3057 * taking the wrong conditional branhc.3058 */3059 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3060 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3061 {3062 Assert(IEM_IS_64BIT_CODE(pVCpu));3063 3064 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;3065 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3066 pVCpu->cpum.GstCtx.rip = uNewRip;3067 else3068 return iemRaiseGeneralProtectionFault0(pVCpu);3069 3070 #ifndef IEM_WITH_CODE_TLB3071 iemOpcodeFlushLight(pVCpu, cbInstr);3072 #endif3073 return iemRegFinishNoFlags(pVCpu, rcNormal);3074 }3075 3076 3077 /**3078 * Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is3079 * sure it stays within the same page, no checking or clearing of flags.3080 *3081 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3082 * segment limit.3083 *3084 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the3085 * only alternative for relative jumps in 64-bit code and that is already3086 * handled in the decoder stage.3087 *3088 * @returns Strict VBox status code.3089 * @param pVCpu The cross context virtual CPU structure of the calling thread.3090 * @param cbInstr Instruction size.3091 * @param offNextInstr The offset of the next instruction.3092 * @param rcNormal VINF_SUCCESS to continue TB.3093 * VINF_IEM_REEXEC_BREAK to force TB exit when3094 * taking the wrong conditional branhc.3095 */3096 DECL_FORCE_INLINE(VBOXSTRICTRC)3097 iemRegRip64RelativeJumpS32IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3098 {3099 Assert(IEM_IS_64BIT_CODE(pVCpu));3100 3101 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;3102 Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));3103 pVCpu->cpum.GstCtx.rip = uNewRip;3104 3105 #ifndef IEM_WITH_CODE_TLB3106 iemOpcodeFlushLight(pVCpu, cbInstr);3107 #endif3108 return iemRegFinishNoFlags(pVCpu, rcNormal);3109 }3110 3111 3112 /**3113 * Adds a 32-bit signed jump offset to RIP from 32-bit code, no checking or3114 * clearing of flags.3115 *3116 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3117 * segment limit.3118 *3119 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3120 * only alternative for relative jumps in 32-bit code and that is already3121 * handled in the decoder stage.3122 *3123 * @returns Strict VBox status code.3124 * @param pVCpu The cross context virtual CPU structure of the calling thread.3125 * @param cbInstr Instruction size.3126 * @param offNextInstr The offset of the next instruction.3127 * @param rcNormal VINF_SUCCESS to continue TB.3128 * VINF_IEM_REEXEC_BREAK to force TB exit when3129 * taking the wrong conditional branhc.3130 */3131 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3132 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3133 {3134 Assert(!IEM_IS_64BIT_CODE(pVCpu));3135 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3136 3137 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3138 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3139 pVCpu->cpum.GstCtx.rip = uNewEip;3140 else3141 return iemRaiseGeneralProtectionFault0(pVCpu);3142 3143 #ifndef IEM_WITH_CODE_TLB3144 iemOpcodeFlushLight(pVCpu, cbInstr);3145 #endif3146 return iemRegFinishNoFlags(pVCpu, rcNormal);3147 }3148 3149 3150 /**3151 * Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code, no checking or3152 * clearing of flags.3153 *3154 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3155 * segment limit.3156 *3157 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the3158 * only alternative for relative jumps in 32-bit code and that is already3159 * handled in the decoder stage.3160 *3161 * @returns Strict VBox status code.3162 * @param pVCpu The cross context virtual CPU structure of the calling thread.3163 * @param cbInstr Instruction size.3164 * @param offNextInstr The offset of the next instruction.3165 * @param rcNormal VINF_SUCCESS to continue TB.3166 * VINF_IEM_REEXEC_BREAK to force TB exit when3167 * taking the wrong conditional branhc.3168 */3169 DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,3170 int32_t offNextInstr, int rcNormal) RT_NOEXCEPT3171 {3172 Assert(!IEM_IS_64BIT_CODE(pVCpu));3173 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3174 3175 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;3176 pVCpu->cpum.GstCtx.rip = uNewEip;3177 3178 #ifndef IEM_WITH_CODE_TLB3179 iemOpcodeFlushLight(pVCpu, cbInstr);3180 #endif3181 return iemRegFinishNoFlags(pVCpu, rcNormal);3182 }3183 3184 3185 /**3186 * Extended version of iemFinishInstructionWithFlagsSet that goes with3187 * iemRegAddToRipAndFinishingClearingRfEx.3188 *3189 * See iemFinishInstructionWithFlagsSet() for details.3190 */3191 static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT3192 {3193 /*3194 * Raise a #DB.3195 */3196 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);3197 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;3198 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS3199 | ( (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)3200 >> CPUMCTX_DBG_HIT_DRX_SHIFT);3201 /** @todo Do we set all pending \#DB events, or just one? */3202 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",3203 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],3204 pVCpu->cpum.GstCtx.rflags.uBoth));3205 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);3206 return iemRaiseDebugException(pVCpu);3207 }3208 3209 3210 /**3211 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and3212 * others potentially updating EFLAGS.TF.3213 *3214 * The single step event must be generated using the TF value at the start of3215 * the instruction, not the new value set by it.3216 *3217 * @param pVCpu The cross context virtual CPU structure of the calling thread.3218 * @param cbInstr The number of bytes to add.3219 * @param fEflOld The EFLAGS at the start of the instruction3220 * execution.3221 */3222 DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT3223 {3224 iemRegAddToRip(pVCpu, cbInstr);3225 if (!(fEflOld & X86_EFL_TF))3226 {3227 /* Specialized iemRegFinishClearingRF edition here that doesn't check X86_EFL_TF. */3228 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);3229 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth3230 & (X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))3231 return VINF_SUCCESS;3232 return iemFinishInstructionWithFlagsSet<0 /*a_fTF*/>(pVCpu, VINF_SUCCESS); /* TF=0, so ignore it. */3233 }3234 return iemFinishInstructionWithTfSet(pVCpu);3235 }3236 3237 3238 #ifndef IEM_WITH_OPAQUE_DECODER_STATE3239 /**3240 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.3241 *3242 * @param pVCpu The cross context virtual CPU structure of the calling thread.3243 */3244 DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT3245 {3246 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));3247 }3248 #endif3249 3250 3251 #ifdef IEM_WITH_CODE_TLB3252 3253 /**3254 * Performs a near jump to the specified address, no checking or clearing of3255 * flags3256 *3257 * May raise a \#GP(0) if the new IP outside the code segment limit.3258 *3259 * @param pVCpu The cross context virtual CPU structure of the calling thread.3260 * @param uNewIp The new IP value.3261 */3262 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT3263 {3264 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3265 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))3266 pVCpu->cpum.GstCtx.rip = uNewIp;3267 else3268 return iemRaiseGeneralProtectionFault0(pVCpu);3269 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3270 }3271 3272 3273 /**3274 * Performs a near jump to the specified address, no checking or clearing of3275 * flags3276 *3277 * May raise a \#GP(0) if the new RIP is outside the code segment limit.3278 *3279 * @param pVCpu The cross context virtual CPU structure of the calling thread.3280 * @param uNewEip The new EIP value.3281 */3282 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT3283 {3284 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3285 Assert(!IEM_IS_64BIT_CODE(pVCpu));3286 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3287 pVCpu->cpum.GstCtx.rip = uNewEip;3288 else3289 return iemRaiseGeneralProtectionFault0(pVCpu);3290 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3291 }3292 3293 3294 /**3295 * Performs a near jump to the specified address, no checking or clearing of3296 * flags.3297 *3298 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3299 * segment limit.3300 *3301 * @param pVCpu The cross context virtual CPU structure of the calling thread.3302 * @param uNewRip The new RIP value.3303 */3304 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT3305 {3306 Assert(IEM_IS_64BIT_CODE(pVCpu));3307 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3308 pVCpu->cpum.GstCtx.rip = uNewRip;3309 else3310 return iemRaiseGeneralProtectionFault0(pVCpu);3311 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3312 }3313 3314 #endif /* IEM_WITH_CODE_TLB */3315 3316 /**3317 * Performs a near jump to the specified address.3318 *3319 * May raise a \#GP(0) if the new IP outside the code segment limit.3320 *3321 * @param pVCpu The cross context virtual CPU structure of the calling thread.3322 * @param uNewIp The new IP value.3323 * @param cbInstr The instruction length, for flushing in the non-TLB case.3324 */3325 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp, uint8_t cbInstr) RT_NOEXCEPT3326 {3327 if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3328 || IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))3329 pVCpu->cpum.GstCtx.rip = uNewIp;3330 else3331 return iemRaiseGeneralProtectionFault0(pVCpu);3332 #ifndef IEM_WITH_CODE_TLB3333 iemOpcodeFlushLight(pVCpu, cbInstr);3334 #else3335 RT_NOREF_PV(cbInstr);3336 #endif3337 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3338 }3339 3340 3341 /**3342 * Performs a near jump to the specified address.3343 *3344 * May raise a \#GP(0) if the new RIP is outside the code segment limit.3345 *3346 * @param pVCpu The cross context virtual CPU structure of the calling thread.3347 * @param uNewEip The new EIP value.3348 * @param cbInstr The instruction length, for flushing in the non-TLB case.3349 */3350 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip, uint8_t cbInstr) RT_NOEXCEPT3351 {3352 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);3353 Assert(!IEM_IS_64BIT_CODE(pVCpu));3354 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))3355 pVCpu->cpum.GstCtx.rip = uNewEip;3356 else3357 return iemRaiseGeneralProtectionFault0(pVCpu);3358 #ifndef IEM_WITH_CODE_TLB3359 iemOpcodeFlushLight(pVCpu, cbInstr);3360 #else3361 RT_NOREF_PV(cbInstr);3362 #endif3363 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3364 }3365 3366 3367 /**3368 * Performs a near jump to the specified address.3369 *3370 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code3371 * segment limit.3372 *3373 * @param pVCpu The cross context virtual CPU structure of the calling thread.3374 * @param uNewRip The new RIP value.3375 * @param cbInstr The instruction length, for flushing in the non-TLB case.3376 */3377 DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip, uint8_t cbInstr) RT_NOEXCEPT3378 {3379 Assert(IEM_IS_64BIT_CODE(pVCpu));3380 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))3381 pVCpu->cpum.GstCtx.rip = uNewRip;3382 else3383 return iemRaiseGeneralProtectionFault0(pVCpu);3384 #ifndef IEM_WITH_CODE_TLB3385 iemOpcodeFlushLight(pVCpu, cbInstr);3386 #else3387 RT_NOREF_PV(cbInstr);3388 #endif3389 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3390 }3391 3392 3393 /**3394 * Implements a 16-bit relative call, no checking or clearing of3395 * flags.3396 *3397 * @param pVCpu The cross context virtual CPU structure of the calling thread.3398 * @param cbInstr The instruction length.3399 * @param offDisp The 16-bit displacement.3400 */3401 DECL_FORCE_INLINE(VBOXSTRICTRC)3402 iemRegRipRelativeCallS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT3403 {3404 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;3405 uint16_t const uNewIp = uOldIp + offDisp;3406 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3407 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)3408 { /* likely */ }3409 else3410 return iemRaiseGeneralProtectionFault0(pVCpu);3411 3412 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);3413 if (rcStrict == VINF_SUCCESS)3414 { /* likely */ }3415 else3416 return rcStrict;3417 3418 pVCpu->cpum.GstCtx.rip = uNewIp;3419 #ifndef IEM_WITH_CODE_TLB3420 iemOpcodeFlushLight(pVCpu, cbInstr);3421 #endif3422 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3423 }3424 3425 3426 /**3427 * Implements a 16-bit relative call.3428 *3429 * @param pVCpu The cross context virtual CPU structure of the calling thread.3430 * @param cbInstr The instruction length.3431 * @param offDisp The 16-bit displacement.3432 */3433 DECL_FORCE_INLINE(VBOXSTRICTRC)3434 iemRegRipRelativeCallS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT3435 {3436 uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;3437 uint16_t const uNewIp = uOldIp + offDisp;3438 if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit3439 || IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)3440 { /* likely */ }3441 else3442 return iemRaiseGeneralProtectionFault0(pVCpu);3443 3444 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);3445 if (rcStrict == VINF_SUCCESS)3446 { /* likely */ }3447 else3448 return rcStrict;3449 3450 pVCpu->cpum.GstCtx.rip = uNewIp;3451 #ifndef IEM_WITH_CODE_TLB3452 iemOpcodeFlushLight(pVCpu, cbInstr);3453 #endif3454 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3455 }3456 3457 3458 /**3459 * Implements a 32-bit relative call, no checking or clearing of flags.3460 *3461 * @param pVCpu The cross context virtual CPU structure of the calling thread.3462 * @param cbInstr The instruction length.3463 * @param offDisp The 32-bit displacement.3464 */3465 DECL_FORCE_INLINE(VBOXSTRICTRC)3466 iemRegEip32RelativeCallS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT3467 {3468 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));3469 3470 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3471 uint32_t const uNewRip = uOldRip + offDisp;3472 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3473 { /* likely */ }3474 else3475 return iemRaiseGeneralProtectionFault0(pVCpu);3476 3477 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3478 if (rcStrict == VINF_SUCCESS)3479 { /* likely */ }3480 else3481 return rcStrict;3482 3483 pVCpu->cpum.GstCtx.rip = uNewRip;3484 #ifndef IEM_WITH_CODE_TLB3485 iemOpcodeFlushLight(pVCpu, cbInstr);3486 #endif3487 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3488 }3489 3490 3491 /**3492 * Implements a 32-bit relative call.3493 *3494 * @param pVCpu The cross context virtual CPU structure of the calling thread.3495 * @param cbInstr The instruction length.3496 * @param offDisp The 32-bit displacement.3497 */3498 DECL_FORCE_INLINE(VBOXSTRICTRC)3499 iemRegEip32RelativeCallS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT3500 {3501 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));3502 3503 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3504 uint32_t const uNewRip = uOldRip + offDisp;3505 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3506 { /* likely */ }3507 else3508 return iemRaiseGeneralProtectionFault0(pVCpu);3509 3510 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3511 if (rcStrict == VINF_SUCCESS)3512 { /* likely */ }3513 else3514 return rcStrict;3515 3516 pVCpu->cpum.GstCtx.rip = uNewRip;3517 #ifndef IEM_WITH_CODE_TLB3518 iemOpcodeFlushLight(pVCpu, cbInstr);3519 #endif3520 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3521 }3522 3523 3524 /**3525 * Implements a 64-bit relative call, no checking or clearing of flags.3526 *3527 * @param pVCpu The cross context virtual CPU structure of the calling thread.3528 * @param cbInstr The instruction length.3529 * @param offDisp The 64-bit displacement.3530 */3531 DECL_FORCE_INLINE(VBOXSTRICTRC)3532 iemRegRip64RelativeCallS64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT3533 {3534 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3535 uint64_t const uNewRip = uOldRip + (int64_t)offDisp;3536 if (IEM_IS_CANONICAL(uNewRip))3537 { /* likely */ }3538 else3539 return iemRaiseNotCanonical(pVCpu);3540 3541 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3542 if (rcStrict == VINF_SUCCESS)3543 { /* likely */ }3544 else3545 return rcStrict;3546 3547 pVCpu->cpum.GstCtx.rip = uNewRip;3548 #ifndef IEM_WITH_CODE_TLB3549 iemOpcodeFlushLight(pVCpu, cbInstr);3550 #endif3551 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3552 }3553 3554 3555 /**3556 * Implements a 64-bit relative call.3557 *3558 * @param pVCpu The cross context virtual CPU structure of the calling thread.3559 * @param cbInstr The instruction length.3560 * @param offDisp The 64-bit displacement.3561 */3562 DECL_FORCE_INLINE(VBOXSTRICTRC)3563 iemRegRip64RelativeCallS64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT3564 {3565 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3566 uint64_t const uNewRip = uOldRip + (int64_t)offDisp;3567 if (IEM_IS_CANONICAL(uNewRip))3568 { /* likely */ }3569 else3570 return iemRaiseNotCanonical(pVCpu);3571 3572 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3573 if (rcStrict == VINF_SUCCESS)3574 { /* likely */ }3575 else3576 return rcStrict;3577 3578 pVCpu->cpum.GstCtx.rip = uNewRip;3579 #ifndef IEM_WITH_CODE_TLB3580 iemOpcodeFlushLight(pVCpu, cbInstr);3581 #endif3582 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3583 }3584 3585 3586 /**3587 * Implements an 16-bit indirect call, no checking or clearing of3588 * flags.3589 *3590 * @param pVCpu The cross context virtual CPU structure of the calling thread.3591 * @param cbInstr The instruction length.3592 * @param uNewRip The new RIP value.3593 */3594 DECL_FORCE_INLINE(VBOXSTRICTRC)3595 iemRegIp16IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3596 {3597 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3598 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3599 { /* likely */ }3600 else3601 return iemRaiseGeneralProtectionFault0(pVCpu);3602 3603 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3604 if (rcStrict == VINF_SUCCESS)3605 { /* likely */ }3606 else3607 return rcStrict;3608 3609 pVCpu->cpum.GstCtx.rip = uNewRip;3610 #ifndef IEM_WITH_CODE_TLB3611 iemOpcodeFlushLight(pVCpu, cbInstr);3612 #endif3613 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3614 }3615 3616 3617 /**3618 * Implements an 16-bit indirect call, no checking or clearing of3619 * flags.3620 *3621 * @param pVCpu The cross context virtual CPU structure of the calling thread.3622 * @param cbInstr The instruction length.3623 * @param uNewRip The new RIP value.3624 */3625 DECL_FORCE_INLINE(VBOXSTRICTRC)3626 iemRegEip32IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3627 {3628 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3629 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3630 { /* likely */ }3631 else3632 return iemRaiseGeneralProtectionFault0(pVCpu);3633 3634 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3635 if (rcStrict == VINF_SUCCESS)3636 { /* likely */ }3637 else3638 return rcStrict;3639 3640 pVCpu->cpum.GstCtx.rip = uNewRip;3641 #ifndef IEM_WITH_CODE_TLB3642 iemOpcodeFlushLight(pVCpu, cbInstr);3643 #endif3644 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3645 }3646 3647 3648 /**3649 * Implements an 16-bit indirect call.3650 *3651 * @param pVCpu The cross context virtual CPU structure of the calling thread.3652 * @param cbInstr The instruction length.3653 * @param uNewRip The new RIP value.3654 */3655 DECL_FORCE_INLINE(VBOXSTRICTRC)3656 iemRegIp16IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3657 {3658 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3659 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3660 { /* likely */ }3661 else3662 return iemRaiseGeneralProtectionFault0(pVCpu);3663 3664 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3665 if (rcStrict == VINF_SUCCESS)3666 { /* likely */ }3667 else3668 return rcStrict;3669 3670 pVCpu->cpum.GstCtx.rip = uNewRip;3671 #ifndef IEM_WITH_CODE_TLB3672 iemOpcodeFlushLight(pVCpu, cbInstr);3673 #endif3674 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3675 }3676 3677 3678 /**3679 * Implements an 16-bit indirect call.3680 *3681 * @param pVCpu The cross context virtual CPU structure of the calling thread.3682 * @param cbInstr The instruction length.3683 * @param uNewRip The new RIP value.3684 */3685 DECL_FORCE_INLINE(VBOXSTRICTRC)3686 iemRegEip32IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT3687 {3688 uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;3689 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3690 { /* likely */ }3691 else3692 return iemRaiseGeneralProtectionFault0(pVCpu);3693 3694 VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);3695 if (rcStrict == VINF_SUCCESS)3696 { /* likely */ }3697 else3698 return rcStrict;3699 3700 pVCpu->cpum.GstCtx.rip = uNewRip;3701 #ifndef IEM_WITH_CODE_TLB3702 iemOpcodeFlushLight(pVCpu, cbInstr);3703 #endif3704 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3705 }3706 3707 3708 /**3709 * Implements an 32-bit indirect call, no checking or clearing of3710 * flags.3711 *3712 * @param pVCpu The cross context virtual CPU structure of the calling thread.3713 * @param cbInstr The instruction length.3714 * @param uNewRip The new RIP value.3715 */3716 DECL_FORCE_INLINE(VBOXSTRICTRC)3717 iemRegEip32IndirectCallU32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT3718 {3719 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3720 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3721 { /* likely */ }3722 else3723 return iemRaiseGeneralProtectionFault0(pVCpu);3724 3725 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3726 if (rcStrict == VINF_SUCCESS)3727 { /* likely */ }3728 else3729 return rcStrict;3730 3731 pVCpu->cpum.GstCtx.rip = uNewRip;3732 #ifndef IEM_WITH_CODE_TLB3733 iemOpcodeFlushLight(pVCpu, cbInstr);3734 #endif3735 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3736 }3737 3738 3739 /**3740 * Implements an 32-bit indirect call.3741 *3742 * @param pVCpu The cross context virtual CPU structure of the calling thread.3743 * @param cbInstr The instruction length.3744 * @param uNewRip The new RIP value.3745 */3746 DECL_FORCE_INLINE(VBOXSTRICTRC)3747 iemRegEip32IndirectCallU32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT3748 {3749 uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;3750 if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)3751 { /* likely */ }3752 else3753 return iemRaiseGeneralProtectionFault0(pVCpu);3754 3755 VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);3756 if (rcStrict == VINF_SUCCESS)3757 { /* likely */ }3758 else3759 return rcStrict;3760 3761 pVCpu->cpum.GstCtx.rip = uNewRip;3762 #ifndef IEM_WITH_CODE_TLB3763 iemOpcodeFlushLight(pVCpu, cbInstr);3764 #endif3765 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3766 }3767 3768 3769 /**3770 * Implements an 64-bit indirect call, no checking or clearing of3771 * flags.3772 *3773 * @param pVCpu The cross context virtual CPU structure of the calling thread.3774 * @param cbInstr The instruction length.3775 * @param uNewRip The new RIP value.3776 */3777 DECL_FORCE_INLINE(VBOXSTRICTRC)3778 iemRegRip64IndirectCallU64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT3779 {3780 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3781 if (IEM_IS_CANONICAL(uNewRip))3782 { /* likely */ }3783 else3784 return iemRaiseGeneralProtectionFault0(pVCpu);3785 3786 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3787 if (rcStrict == VINF_SUCCESS)3788 { /* likely */ }3789 else3790 return rcStrict;3791 3792 pVCpu->cpum.GstCtx.rip = uNewRip;3793 #ifndef IEM_WITH_CODE_TLB3794 iemOpcodeFlushLight(pVCpu, cbInstr);3795 #endif3796 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);3797 }3798 3799 3800 /**3801 * Implements an 64-bit indirect call.3802 *3803 * @param pVCpu The cross context virtual CPU structure of the calling thread.3804 * @param cbInstr The instruction length.3805 * @param uNewRip The new RIP value.3806 */3807 DECL_FORCE_INLINE(VBOXSTRICTRC)3808 iemRegRip64IndirectCallU64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT3809 {3810 uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;3811 if (IEM_IS_CANONICAL(uNewRip))3812 { /* likely */ }3813 else3814 return iemRaiseGeneralProtectionFault0(pVCpu);3815 3816 VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);3817 if (rcStrict == VINF_SUCCESS)3818 { /* likely */ }3819 else3820 return rcStrict;3821 3822 pVCpu->cpum.GstCtx.rip = uNewRip;3823 #ifndef IEM_WITH_CODE_TLB3824 iemOpcodeFlushLight(pVCpu, cbInstr);3825 #endif3826 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);3827 }3828 3829 3830 3831 /**3832 * Adds to the stack pointer.3833 *3834 * @param pVCpu The cross context virtual CPU structure of the calling thread.3835 * @param cbToAdd The number of bytes to add (8-bit!).3836 */3837 DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT3838 {3839 if (IEM_IS_64BIT_CODE(pVCpu))3840 pVCpu->cpum.GstCtx.rsp += cbToAdd;3841 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3842 pVCpu->cpum.GstCtx.esp += cbToAdd;3843 else3844 pVCpu->cpum.GstCtx.sp += cbToAdd;3845 }3846 3847 3848 /**3849 * Subtracts from the stack pointer.3850 *3851 * @param pVCpu The cross context virtual CPU structure of the calling thread.3852 * @param cbToSub The number of bytes to subtract (8-bit!).3853 */3854 DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT3855 {3856 if (IEM_IS_64BIT_CODE(pVCpu))3857 pVCpu->cpum.GstCtx.rsp -= cbToSub;3858 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3859 pVCpu->cpum.GstCtx.esp -= cbToSub;3860 else3861 pVCpu->cpum.GstCtx.sp -= cbToSub;3862 }3863 3864 3865 /**3866 * Adds to the temporary stack pointer.3867 *3868 * @param pVCpu The cross context virtual CPU structure of the calling thread.3869 * @param pTmpRsp The temporary SP/ESP/RSP to update.3870 * @param cbToAdd The number of bytes to add (16-bit).3871 */3872 DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT3873 {3874 if (IEM_IS_64BIT_CODE(pVCpu))3875 pTmpRsp->u += cbToAdd;3876 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3877 pTmpRsp->DWords.dw0 += cbToAdd;3878 else3879 pTmpRsp->Words.w0 += cbToAdd;3880 }3881 3882 3883 /**3884 * Subtracts from the temporary stack pointer.3885 *3886 * @param pVCpu The cross context virtual CPU structure of the calling thread.3887 * @param pTmpRsp The temporary SP/ESP/RSP to update.3888 * @param cbToSub The number of bytes to subtract.3889 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is3890 * expecting that.3891 */3892 DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT3893 {3894 if (IEM_IS_64BIT_CODE(pVCpu))3895 pTmpRsp->u -= cbToSub;3896 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3897 pTmpRsp->DWords.dw0 -= cbToSub;3898 else3899 pTmpRsp->Words.w0 -= cbToSub;3900 }3901 3902 3903 /**3904 * Calculates the effective stack address for a push of the specified size as3905 * well as the new RSP value (upper bits may be masked).3906 *3907 * @returns Effective stack addressf for the push.3908 * @param pVCpu The cross context virtual CPU structure of the calling thread.3909 * @param cbItem The size of the stack item to pop.3910 * @param puNewRsp Where to return the new RSP value.3911 */3912 DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT3913 {3914 RTUINT64U uTmpRsp;3915 RTGCPTR GCPtrTop;3916 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;3917 3918 if (IEM_IS_64BIT_CODE(pVCpu))3919 GCPtrTop = uTmpRsp.u -= cbItem;3920 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3921 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;3922 else3923 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;3924 *puNewRsp = uTmpRsp.u;3925 return GCPtrTop;3926 }3927 3928 3929 /**3930 * Gets the current stack pointer and calculates the value after a pop of the3931 * specified size.3932 *3933 * @returns Current stack pointer.3934 * @param pVCpu The cross context virtual CPU structure of the calling thread.3935 * @param cbItem The size of the stack item to pop.3936 * @param puNewRsp Where to return the new RSP value.3937 */3938 DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT3939 {3940 RTUINT64U uTmpRsp;3941 RTGCPTR GCPtrTop;3942 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;3943 3944 if (IEM_IS_64BIT_CODE(pVCpu))3945 {3946 GCPtrTop = uTmpRsp.u;3947 uTmpRsp.u += cbItem;3948 }3949 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3950 {3951 GCPtrTop = uTmpRsp.DWords.dw0;3952 uTmpRsp.DWords.dw0 += cbItem;3953 }3954 else3955 {3956 GCPtrTop = uTmpRsp.Words.w0;3957 uTmpRsp.Words.w0 += cbItem;3958 }3959 *puNewRsp = uTmpRsp.u;3960 return GCPtrTop;3961 }3962 3963 3964 /**3965 * Calculates the effective stack address for a push of the specified size as3966 * well as the new temporary RSP value (upper bits may be masked).3967 *3968 * @returns Effective stack addressf for the push.3969 * @param pVCpu The cross context virtual CPU structure of the calling thread.3970 * @param pTmpRsp The temporary stack pointer. This is updated.3971 * @param cbItem The size of the stack item to pop.3972 */3973 DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT3974 {3975 RTGCPTR GCPtrTop;3976 3977 if (IEM_IS_64BIT_CODE(pVCpu))3978 GCPtrTop = pTmpRsp->u -= cbItem;3979 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)3980 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;3981 else3982 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;3983 return GCPtrTop;3984 }3985 3986 3987 /**3988 * Gets the effective stack address for a pop of the specified size and3989 * calculates and updates the temporary RSP.3990 *3991 * @returns Current stack pointer.3992 * @param pVCpu The cross context virtual CPU structure of the calling thread.3993 * @param pTmpRsp The temporary stack pointer. This is updated.3994 * @param cbItem The size of the stack item to pop.3995 */3996 DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT3997 {3998 RTGCPTR GCPtrTop;3999 if (IEM_IS_64BIT_CODE(pVCpu))4000 {4001 GCPtrTop = pTmpRsp->u;4002 pTmpRsp->u += cbItem;4003 }4004 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)4005 {4006 GCPtrTop = pTmpRsp->DWords.dw0;4007 pTmpRsp->DWords.dw0 += cbItem;4008 }4009 else4010 {4011 GCPtrTop = pTmpRsp->Words.w0;4012 pTmpRsp->Words.w0 += cbItem;4013 }4014 return GCPtrTop;4015 }4016 4017 4018 /** Common body for iemRegRipNearReturnAndFinishClearingRF()4019 * and iemRegRipNearReturnAndFinishNoFlags(). */4020 template<bool a_fWithFlags>4021 DECL_FORCE_INLINE(VBOXSTRICTRC)4022 iemRegRipNearReturnCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4023 {4024 /* Fetch the new RIP from the stack. */4025 VBOXSTRICTRC rcStrict;4026 RTUINT64U NewRip;4027 RTUINT64U NewRsp;4028 NewRsp.u = pVCpu->cpum.GstCtx.rsp;4029 switch (enmEffOpSize)4030 {4031 case IEMMODE_16BIT:4032 NewRip.u = 0;4033 rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);4034 break;4035 case IEMMODE_32BIT:4036 NewRip.u = 0;4037 rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);4038 break;4039 case IEMMODE_64BIT:4040 rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);4041 break;4042 IEM_NOT_REACHED_DEFAULT_CASE_RET();4043 }4044 if (rcStrict != VINF_SUCCESS)4045 return rcStrict;4046 4047 /* Check the new ew RIP before loading it. */4048 /** @todo Should test this as the intel+amd pseudo code doesn't mention half4049 * of it. The canonical test is performed here and for call. */4050 if (enmEffOpSize != IEMMODE_64BIT)4051 {4052 if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit))4053 { /* likely */ }4054 else4055 {4056 Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));4057 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);4058 }4059 }4060 else4061 {4062 if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u)))4063 { /* likely */ }4064 else4065 {4066 Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));4067 return iemRaiseNotCanonical(pVCpu);4068 }4069 }4070 4071 /* Apply cbPop */4072 if (cbPop)4073 iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);4074 4075 /* Commit it. */4076 pVCpu->cpum.GstCtx.rip = NewRip.u;4077 pVCpu->cpum.GstCtx.rsp = NewRsp.u;4078 4079 /* Flush the prefetch buffer. */4080 #ifndef IEM_WITH_CODE_TLB4081 iemOpcodeFlushLight(pVCpu, cbInstr);4082 #endif4083 RT_NOREF(cbInstr);4084 4085 4086 if (a_fWithFlags)4087 return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);4088 return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);4089 }4090 4091 4092 /**4093 * Implements retn and retn imm16.4094 *4095 * @param pVCpu The cross context virtual CPU structure of the4096 * calling thread.4097 * @param cbInstr The current instruction length.4098 * @param enmEffOpSize The effective operand size. This is constant.4099 * @param cbPop The amount of arguments to pop from the stack4100 * (bytes). This can be constant (zero).4101 */4102 DECL_FORCE_INLINE(VBOXSTRICTRC)4103 iemRegRipNearReturnAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4104 {4105 return iemRegRipNearReturnCommon<true /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize);4106 }4107 4108 4109 /**4110 * Implements retn and retn imm16, no checking or clearing of4111 * flags.4112 *4113 * @param pVCpu The cross context virtual CPU structure of the4114 * calling thread.4115 * @param cbInstr The current instruction length.4116 * @param enmEffOpSize The effective operand size. This is constant.4117 * @param cbPop The amount of arguments to pop from the stack4118 * (bytes). This can be constant (zero).4119 */4120 DECL_FORCE_INLINE(VBOXSTRICTRC)4121 iemRegRipNearReturnAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT4122 {4123 return iemRegRipNearReturnCommon<false /*a_fWithFlags*/>(pVCpu, cbInstr, cbPop, enmEffOpSize);4124 }4125 4126 /** @} */4127 4128 4129 /** @name FPU access and helpers.4130 *4131 * @{4132 */4133 4134 4135 /**4136 * Hook for preparing to use the host FPU.4137 *4138 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4139 *4140 * @param pVCpu The cross context virtual CPU structure of the calling thread.4141 */4142 DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT4143 {4144 #ifdef IN_RING34145 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4146 #else4147 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);4148 #endif4149 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4150 }4151 4152 4153 /**4154 * Hook for preparing to use the host FPU for SSE.4155 *4156 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4157 *4158 * @param pVCpu The cross context virtual CPU structure of the calling thread.4159 */4160 DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT4161 {4162 iemFpuPrepareUsage(pVCpu);4163 }4164 4165 4166 /**4167 * Hook for preparing to use the host FPU for AVX.4168 *4169 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4170 *4171 * @param pVCpu The cross context virtual CPU structure of the calling thread.4172 */4173 DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT4174 {4175 iemFpuPrepareUsage(pVCpu);4176 }4177 4178 4179 /**4180 * Hook for actualizing the guest FPU state before the interpreter reads it.4181 *4182 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4183 *4184 * @param pVCpu The cross context virtual CPU structure of the calling thread.4185 */4186 DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4187 {4188 #ifdef IN_RING34189 NOREF(pVCpu);4190 #else4191 CPUMRZFpuStateActualizeForRead(pVCpu);4192 #endif4193 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4194 }4195 4196 4197 /**4198 * Hook for actualizing the guest FPU state before the interpreter changes it.4199 *4200 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4201 *4202 * @param pVCpu The cross context virtual CPU structure of the calling thread.4203 */4204 DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4205 {4206 #ifdef IN_RING34207 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4208 #else4209 CPUMRZFpuStateActualizeForChange(pVCpu);4210 #endif4211 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4212 }4213 4214 4215 /**4216 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read4217 * only.4218 *4219 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4220 *4221 * @param pVCpu The cross context virtual CPU structure of the calling thread.4222 */4223 DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4224 {4225 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)4226 NOREF(pVCpu);4227 #else4228 CPUMRZFpuStateActualizeSseForRead(pVCpu);4229 #endif4230 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4231 }4232 4233 4234 /**4235 * Hook for actualizing the guest XMM0..15 and MXCSR register state for4236 * read+write.4237 *4238 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4239 *4240 * @param pVCpu The cross context virtual CPU structure of the calling thread.4241 */4242 DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4243 {4244 #if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)4245 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4246 #else4247 CPUMRZFpuStateActualizeForChange(pVCpu);4248 #endif4249 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4250 4251 /* Make sure any changes are loaded the next time around. */4252 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;4253 }4254 4255 4256 /**4257 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read4258 * only.4259 *4260 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4261 *4262 * @param pVCpu The cross context virtual CPU structure of the calling thread.4263 */4264 DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT4265 {4266 #ifdef IN_RING34267 NOREF(pVCpu);4268 #else4269 CPUMRZFpuStateActualizeAvxForRead(pVCpu);4270 #endif4271 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4272 }4273 4274 4275 /**4276 * Hook for actualizing the guest YMM0..15 and MXCSR register state for4277 * read+write.4278 *4279 * This is necessary in ring-0 and raw-mode context (nop in ring-3).4280 *4281 * @param pVCpu The cross context virtual CPU structure of the calling thread.4282 */4283 DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT4284 {4285 #ifdef IN_RING34286 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);4287 #else4288 CPUMRZFpuStateActualizeForChange(pVCpu);4289 #endif4290 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);4291 4292 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */4293 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;4294 }4295 4296 4297 /**4298 * Stores a QNaN value into a FPU register.4299 *4300 * @param pReg Pointer to the register.4301 */4302 DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT4303 {4304 pReg->au32[0] = UINT32_C(0x00000000);4305 pReg->au32[1] = UINT32_C(0xc0000000);4306 pReg->au16[4] = UINT16_C(0xffff);4307 }4308 4309 4310 /**4311 * Updates the FOP, FPU.CS and FPUIP registers, extended version.4312 *4313 * @param pVCpu The cross context virtual CPU structure of the calling thread.4314 * @param pFpuCtx The FPU context.4315 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).4316 */4317 DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT4318 {4319 Assert(uFpuOpcode != UINT16_MAX);4320 pFpuCtx->FOP = uFpuOpcode;4321 /** @todo x87.CS and FPUIP needs to be kept seperately. */4322 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))4323 {4324 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled4325 * happens in real mode here based on the fnsave and fnstenv images. */4326 pFpuCtx->CS = 0;4327 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);4328 }4329 else if (!IEM_IS_LONG_MODE(pVCpu))4330 {4331 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;4332 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;4333 }4334 else4335 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;4336 }4337 4338 4339 /**4340 * Marks the specified stack register as free (for FFREE).4341 *4342 * @param pVCpu The cross context virtual CPU structure of the calling thread.4343 * @param iStReg The register to free.4344 */4345 DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT4346 {4347 Assert(iStReg < 8);4348 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4349 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4350 pFpuCtx->FTW &= ~RT_BIT(iReg);4351 }4352 4353 4354 /**4355 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.4356 *4357 * @param pVCpu The cross context virtual CPU structure of the calling thread.4358 */4359 DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT4360 {4361 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4362 uint16_t uFsw = pFpuCtx->FSW;4363 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;4364 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;4365 uFsw &= ~X86_FSW_TOP_MASK;4366 uFsw |= uTop;4367 pFpuCtx->FSW = uFsw;4368 }4369 4370 4371 /**4372 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.4373 *4374 * @param pVCpu The cross context virtual CPU structure of the calling thread.4375 */4376 DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT4377 {4378 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4379 uint16_t uFsw = pFpuCtx->FSW;4380 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;4381 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;4382 uFsw &= ~X86_FSW_TOP_MASK;4383 uFsw |= uTop;4384 pFpuCtx->FSW = uFsw;4385 }4386 4387 4388 4389 4390 DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT4391 {4392 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4393 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4394 if (pFpuCtx->FTW & RT_BIT(iReg))4395 return VINF_SUCCESS;4396 return VERR_NOT_FOUND;4397 }4398 4399 4400 DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT4401 {4402 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4403 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;4404 if (pFpuCtx->FTW & RT_BIT(iReg))4405 {4406 *ppRef = &pFpuCtx->aRegs[iStReg].r80;4407 return VINF_SUCCESS;4408 }4409 return VERR_NOT_FOUND;4410 }4411 4412 4413 DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,4414 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT4415 {4416 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4417 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4418 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;4419 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;4420 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))4421 {4422 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;4423 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;4424 return VINF_SUCCESS;4425 }4426 return VERR_NOT_FOUND;4427 }4428 4429 4430 DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT4431 {4432 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;4433 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4434 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;4435 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;4436 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))4437 {4438 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;4439 return VINF_SUCCESS;4440 }4441 return VERR_NOT_FOUND;4442 }4443 4444 4445 /**4446 * Rotates the stack registers when setting new TOS.4447 *4448 * @param pFpuCtx The FPU context.4449 * @param iNewTop New TOS value.4450 * @remarks We only do this to speed up fxsave/fxrstor which4451 * arrange the FP registers in stack order.4452 * MUST be done before writing the new TOS (FSW).4453 */4454 DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT4455 {4456 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4457 RTFLOAT80U ar80Temp[8];4458 4459 if (iOldTop == iNewTop)4460 return;4461 4462 /* Unscrew the stack and get it into 'native' order. */4463 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;4464 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;4465 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;4466 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;4467 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;4468 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;4469 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;4470 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;4471 4472 /* Now rotate the stack to the new position. */4473 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];4474 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];4475 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];4476 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];4477 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];4478 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];4479 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];4480 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];4481 }4482 4483 4484 /**4485 * Updates the FPU exception status after FCW is changed.4486 *4487 * @param pFpuCtx The FPU context.4488 */4489 DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT4490 {4491 uint16_t u16Fsw = pFpuCtx->FSW;4492 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))4493 u16Fsw |= X86_FSW_ES | X86_FSW_B;4494 else4495 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);4496 pFpuCtx->FSW = u16Fsw;4497 }4498 4499 4500 /**4501 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.4502 *4503 * @returns The full FTW.4504 * @param pFpuCtx The FPU context.4505 */4506 DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT4507 {4508 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;4509 uint16_t u16Ftw = 0;4510 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);4511 for (unsigned iSt = 0; iSt < 8; iSt++)4512 {4513 unsigned const iReg = (iSt + iTop) & 7;4514 if (!(u8Ftw & RT_BIT(iReg)))4515 u16Ftw |= 3 << (iReg * 2); /* empty */4516 else4517 {4518 uint16_t uTag;4519 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;4520 if (pr80Reg->s.uExponent == 0x7fff)4521 uTag = 2; /* Exponent is all 1's => Special. */4522 else if (pr80Reg->s.uExponent == 0x0000)4523 {4524 if (pr80Reg->s.uMantissa == 0x0000)4525 uTag = 1; /* All bits are zero => Zero. */4526 else4527 uTag = 2; /* Must be special. */4528 }4529 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */4530 uTag = 0; /* Valid. */4531 else4532 uTag = 2; /* Must be special. */4533 4534 u16Ftw |= uTag << (iReg * 2);4535 }4536 }4537 4538 return u16Ftw;4539 }4540 4541 4542 /**4543 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).4544 *4545 * @returns The compressed FTW.4546 * @param u16FullFtw The full FTW to convert.4547 */4548 DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT4549 {4550 uint8_t u8Ftw = 0;4551 for (unsigned i = 0; i < 8; i++)4552 {4553 if ((u16FullFtw & 3) != 3 /*empty*/)4554 u8Ftw |= RT_BIT(i);4555 u16FullFtw >>= 2;4556 }4557 4558 return u8Ftw;4559 }4560 4561 /** @} */4562 4563 4564 /** @name Memory access.4565 *4566 * @{4567 */4568 4569 4570 /**4571 * Checks whether alignment checks are enabled or not.4572 *4573 * @returns true if enabled, false if not.4574 * @param pVCpu The cross context virtual CPU structure of the calling thread.4575 */4576 DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT4577 {4578 #if 04579 AssertCompile(X86_CR0_AM == X86_EFL_AC);4580 return IEM_GET_CPL(pVCpu) == 34581 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);4582 #else4583 return RT_BOOL(pVCpu->iem.s.fExec & IEM_F_X86_AC);4584 #endif4585 }4586 4587 /**4588 * Checks if the given segment can be written to, raise the appropriate4589 * exception if not.4590 *4591 * @returns VBox strict status code.4592 *4593 * @param pVCpu The cross context virtual CPU structure of the calling thread.4594 * @param pHid Pointer to the hidden register.4595 * @param iSegReg The register number.4596 * @param pu64BaseAddr Where to return the base address to use for the4597 * segment. (In 64-bit code it may differ from the4598 * base in the hidden segment.)4599 */4600 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,4601 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT4602 {4603 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4604 4605 if (IEM_IS_64BIT_CODE(pVCpu))4606 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;4607 else4608 {4609 if (!pHid->Attr.n.u1Present)4610 {4611 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);4612 AssertRelease(uSel == 0);4613 LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));4614 return iemRaiseGeneralProtectionFault0(pVCpu);4615 }4616 4617 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)4618 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )4619 && !IEM_IS_64BIT_CODE(pVCpu) )4620 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4621 *pu64BaseAddr = pHid->u64Base;4622 }4623 return VINF_SUCCESS;4624 }4625 4626 4627 /**4628 * Checks if the given segment can be read from, raise the appropriate4629 * exception if not.4630 *4631 * @returns VBox strict status code.4632 *4633 * @param pVCpu The cross context virtual CPU structure of the calling thread.4634 * @param pHid Pointer to the hidden register.4635 * @param iSegReg The register number.4636 * @param pu64BaseAddr Where to return the base address to use for the4637 * segment. (In 64-bit code it may differ from the4638 * base in the hidden segment.)4639 */4640 DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,4641 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT4642 {4643 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4644 4645 if (IEM_IS_64BIT_CODE(pVCpu))4646 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;4647 else4648 {4649 if (!pHid->Attr.n.u1Present)4650 {4651 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);4652 AssertRelease(uSel == 0);4653 LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));4654 return iemRaiseGeneralProtectionFault0(pVCpu);4655 }4656 4657 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)4658 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4659 *pu64BaseAddr = pHid->u64Base;4660 }4661 return VINF_SUCCESS;4662 }4663 4664 293 4665 294 /** … … 4717 346 } 4718 347 4719 #ifdef IEM_WITH_SETJMP4720 4721 /** @todo slim this down */4722 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,4723 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP4724 {4725 Assert(cbMem >= 1);4726 Assert(iSegReg < X86_SREG_COUNT);4727 4728 /*4729 * 64-bit mode is simpler.4730 */4731 if (IEM_IS_64BIT_CODE(pVCpu))4732 {4733 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)4734 {4735 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4736 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);4737 GCPtrMem += pSel->u64Base;4738 }4739 4740 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))4741 return GCPtrMem;4742 iemRaiseGeneralProtectionFault0Jmp(pVCpu);4743 }4744 /*4745 * 16-bit and 32-bit segmentation.4746 */4747 else if (iSegReg != UINT8_MAX)4748 {4749 /** @todo Does this apply to segments with 4G-1 limit? */4750 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4751 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))4752 {4753 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4754 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);4755 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE4756 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */4757 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */4758 | X86_SEL_TYPE_CODE))4759 {4760 case X86DESCATTR_P: /* readonly data, expand up */4761 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */4762 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */4763 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */4764 /* expand up */4765 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))4766 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4767 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",4768 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));4769 break;4770 4771 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */4772 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */4773 /* expand down */4774 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit4775 && ( pSel->Attr.n.u1DefBig4776 || GCPtrLast32 <= UINT32_C(0xffff)) ))4777 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4778 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",4779 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));4780 break;4781 4782 default:4783 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));4784 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4785 break;4786 }4787 }4788 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));4789 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);4790 }4791 /*4792 * 32-bit flat address.4793 */4794 else4795 return GCPtrMem;4796 }4797 4798 4799 /** @todo slim this down */4800 DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,4801 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP4802 {4803 Assert(cbMem >= 1);4804 Assert(iSegReg < X86_SREG_COUNT);4805 4806 /*4807 * 64-bit mode is simpler.4808 */4809 if (IEM_IS_64BIT_CODE(pVCpu))4810 {4811 if (iSegReg >= X86_SREG_FS)4812 {4813 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4814 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);4815 GCPtrMem += pSel->u64Base;4816 }4817 4818 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))4819 return GCPtrMem;4820 }4821 /*4822 * 16-bit and 32-bit segmentation.4823 */4824 else4825 {4826 Assert(GCPtrMem <= UINT32_MAX);4827 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));4828 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);4829 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE4830 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);4831 if ( fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE) /* data, expand up */4832 /** @todo explore exactly how the CS stuff works in real mode. See also4833 * http://www.rcollins.org/Productivity/DescriptorCache.html and4834 * http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */4835 || (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. */ /** @todo testcase! */4836 {4837 /* expand up */4838 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4839 if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit4840 && GCPtrLast32 >= (uint32_t)GCPtrMem))4841 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4842 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4843 }4844 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */4845 {4846 /* expand down - the uppger boundary is defined by the B bit, not G. */4847 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;4848 if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit4849 && (pSel->Attr.n.u1DefBig || GCPtrLast32 <= UINT32_C(0xffff))4850 && GCPtrLast32 >= (uint32_t)GCPtrMem))4851 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;4852 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4853 }4854 else4855 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);4856 }4857 iemRaiseGeneralProtectionFault0Jmp(pVCpu);4858 }4859 4860 #endif /* IEM_WITH_SETJMP */4861 4862 /**4863 * Fakes a long mode stack selector for SS = 0.4864 *4865 * @param pDescSs Where to return the fake stack descriptor.4866 * @param uDpl The DPL we want.4867 */4868 DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT4869 {4870 pDescSs->Long.au64[0] = 0;4871 pDescSs->Long.au64[1] = 0;4872 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;4873 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */4874 pDescSs->Long.Gen.u2Dpl = uDpl;4875 pDescSs->Long.Gen.u1Present = 1;4876 pDescSs->Long.Gen.u1Long = 1;4877 }4878 4879 348 4880 349 /* … … 4935 404 4936 405 4937 /*4938 * Instantiate R/W inline templates.4939 */4940 4941 /** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK4942 * Used to check if an unaligned access is if within the page and won't4943 * trigger an \#AC.4944 *4945 * This can also be used to deal with misaligned accesses on platforms that are4946 * senstive to such if desires.4947 */4948 #if 14949 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \4950 ( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \4951 && !((a_pVCpu)->iem.s.fExec & IEM_F_X86_AC) )4952 #else4953 # define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 04954 #endif4955 4956 #define TMPL_MEM_WITH_ATOMIC_MAPPING4957 4958 #define TMPL_MEM_TYPE uint8_t4959 #define TMPL_MEM_TYPE_ALIGN 04960 #define TMPL_MEM_TYPE_SIZE 14961 #define TMPL_MEM_FN_SUFF U84962 #define TMPL_MEM_FMT_TYPE "%#04x"4963 #define TMPL_MEM_FMT_DESC "byte"4964 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4965 4966 #define TMPL_MEM_WITH_STACK4967 4968 #define TMPL_MEM_TYPE uint16_t4969 #define TMPL_MEM_TYPE_ALIGN 14970 #define TMPL_MEM_TYPE_SIZE 24971 #define TMPL_MEM_FN_SUFF U164972 #define TMPL_MEM_FMT_TYPE "%#06x"4973 #define TMPL_MEM_FMT_DESC "word"4974 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4975 4976 #define TMPL_WITH_PUSH_SREG4977 #define TMPL_MEM_TYPE uint32_t4978 #define TMPL_MEM_TYPE_ALIGN 34979 #define TMPL_MEM_TYPE_SIZE 44980 #define TMPL_MEM_FN_SUFF U324981 #define TMPL_MEM_FMT_TYPE "%#010x"4982 #define TMPL_MEM_FMT_DESC "dword"4983 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4984 #undef TMPL_WITH_PUSH_SREG4985 4986 #define TMPL_MEM_TYPE uint64_t4987 #define TMPL_MEM_TYPE_ALIGN 74988 #define TMPL_MEM_TYPE_SIZE 84989 #define TMPL_MEM_FN_SUFF U644990 #define TMPL_MEM_FMT_TYPE "%#018RX64"4991 #define TMPL_MEM_FMT_DESC "qword"4992 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"4993 4994 #undef TMPL_MEM_WITH_STACK4995 #undef TMPL_MEM_WITH_ATOMIC_MAPPING4996 4997 #define TMPL_MEM_NO_MAPPING /* currently sticky */4998 4999 #define TMPL_MEM_NO_STORE5000 #define TMPL_MEM_TYPE uint32_t5001 #define TMPL_MEM_TYPE_ALIGN 05002 #define TMPL_MEM_TYPE_SIZE 45003 #define TMPL_MEM_FN_SUFF U32NoAc5004 #define TMPL_MEM_FMT_TYPE "%#010x"5005 #define TMPL_MEM_FMT_DESC "dword"5006 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5007 5008 #define TMPL_MEM_NO_STORE5009 #define TMPL_MEM_TYPE uint64_t5010 #define TMPL_MEM_TYPE_ALIGN 05011 #define TMPL_MEM_TYPE_SIZE 85012 #define TMPL_MEM_FN_SUFF U64NoAc5013 #define TMPL_MEM_FMT_TYPE "%#018RX64"5014 #define TMPL_MEM_FMT_DESC "qword"5015 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5016 5017 #define TMPL_MEM_NO_STORE5018 #define TMPL_MEM_TYPE uint64_t5019 #define TMPL_MEM_TYPE_ALIGN 155020 #define TMPL_MEM_TYPE_SIZE 85021 #define TMPL_MEM_FN_SUFF U64AlignedU1285022 #define TMPL_MEM_FMT_TYPE "%#018RX64"5023 #define TMPL_MEM_FMT_DESC "qword"5024 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5025 5026 #undef TMPL_MEM_NO_MAPPING5027 5028 #define TMPL_MEM_TYPE RTFLOAT80U5029 #define TMPL_MEM_TYPE_ALIGN 75030 #define TMPL_MEM_TYPE_SIZE 105031 #define TMPL_MEM_FN_SUFF R805032 #define TMPL_MEM_FMT_TYPE "%.10Rhxs"5033 #define TMPL_MEM_FMT_DESC "tword"5034 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5035 5036 #define TMPL_MEM_TYPE RTPBCD80U5037 #define TMPL_MEM_TYPE_ALIGN 7 /** @todo RTPBCD80U alignment testcase */5038 #define TMPL_MEM_TYPE_SIZE 105039 #define TMPL_MEM_FN_SUFF D805040 #define TMPL_MEM_FMT_TYPE "%.10Rhxs"5041 #define TMPL_MEM_FMT_DESC "tword"5042 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5043 5044 #define TMPL_MEM_WITH_ATOMIC_MAPPING5045 #define TMPL_MEM_TYPE RTUINT128U5046 #define TMPL_MEM_TYPE_ALIGN 155047 #define TMPL_MEM_TYPE_SIZE 165048 #define TMPL_MEM_FN_SUFF U1285049 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5050 #define TMPL_MEM_FMT_DESC "dqword"5051 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5052 #undef TMPL_MEM_WITH_ATOMIC_MAPPING5053 5054 #define TMPL_MEM_NO_MAPPING5055 #define TMPL_MEM_TYPE RTUINT128U5056 #define TMPL_MEM_TYPE_ALIGN 05057 #define TMPL_MEM_TYPE_SIZE 165058 #define TMPL_MEM_FN_SUFF U128NoAc5059 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5060 #define TMPL_MEM_FMT_DESC "dqword"5061 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5062 #undef TMPL_MEM_NO_MAPPING5063 5064 5065 /* Every template relying on unaligned accesses inside a page not being okay should go below. */5066 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK5067 #define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 05068 5069 #define TMPL_MEM_NO_MAPPING5070 #define TMPL_MEM_TYPE RTUINT128U5071 #define TMPL_MEM_TYPE_ALIGN 155072 #define TMPL_MEM_TYPE_SIZE 165073 #define TMPL_MEM_FN_SUFF U128AlignedSse5074 #define TMPL_MEM_FMT_TYPE "%.16Rhxs"5075 #define TMPL_MEM_FMT_DESC "dqword"5076 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5077 #undef TMPL_MEM_NO_MAPPING5078 5079 #define TMPL_MEM_NO_MAPPING5080 #define TMPL_MEM_TYPE RTUINT256U5081 #define TMPL_MEM_TYPE_ALIGN 05082 #define TMPL_MEM_TYPE_SIZE 325083 #define TMPL_MEM_FN_SUFF U256NoAc5084 #define TMPL_MEM_FMT_TYPE "%.32Rhxs"5085 #define TMPL_MEM_FMT_DESC "qqword"5086 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5087 #undef TMPL_MEM_NO_MAPPING5088 5089 #define TMPL_MEM_NO_MAPPING5090 #define TMPL_MEM_TYPE RTUINT256U5091 #define TMPL_MEM_TYPE_ALIGN 315092 #define TMPL_MEM_TYPE_SIZE 325093 #define TMPL_MEM_FN_SUFF U256AlignedAvx5094 #define TMPL_MEM_FMT_TYPE "%.32Rhxs"5095 #define TMPL_MEM_FMT_DESC "qqword"5096 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"5097 #undef TMPL_MEM_NO_MAPPING5098 5099 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK5100 5101 406 /** @} */ 5102 407 5103 5104 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX5105 5106 /**5107 * Gets CR0 fixed-0 bits in VMX operation.5108 *5109 * We do this rather than fetching what we report to the guest (in5110 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same5111 * values regardless of whether unrestricted-guest feature is available on the CPU.5112 *5113 * @returns CR0 fixed-0 bits.5114 * @param pVCpu The cross context virtual CPU structure.5115 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode5116 * must be returned. When @c false, the CR0 fixed-05117 * bits for VMX root mode is returned.5118 *5119 */5120 DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT5121 {5122 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));5123 5124 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;5125 if ( fVmxNonRootMode5126 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))5127 return VMX_V_CR0_FIXED0_UX;5128 return VMX_V_CR0_FIXED0;5129 }5130 5131 5132 # ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */5133 /**5134 * Sets virtual-APIC write emulation as pending.5135 *5136 * @param pVCpu The cross context virtual CPU structure.5137 * @param offApic The offset in the virtual-APIC page that was written.5138 */5139 DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT5140 {5141 Assert(offApic < XAPIC_OFF_END + 4);5142 5143 /*5144 * Record the currently updated APIC offset, as we need this later for figuring5145 * out whether to perform TPR, EOI or self-IPI virtualization as well as well5146 * as for supplying the exit qualification when causing an APIC-write VM-exit.5147 */5148 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;5149 5150 /*5151 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI5152 * virtualization or APIC-write emulation).5153 */5154 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))5155 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);5156 }5157 # endif /* XAPIC_OFF_END */5158 5159 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */5160 408 5161 409 #if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3)
Note:
See TracChangeset
for help on using the changeset viewer.
