VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 104149

Last change on this file since 104149 was 104135, checked in by vboxsync, 13 months ago

VMM/IEM: Rework MXCSR handling for AVX instructions. See r162550 for an explanation, bugref:10641
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 172.6 KB
Line 
1/* $Id: IEMMc.h 104135 2024-04-03 13:18:38Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - IEM_MC_XXX.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29#define VMM_INCLUDED_SRC_include_IEMMc_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @name "Microcode" macros.
36 *
37 * The idea is that we should be able to use the same code to interpret
38 * instructions as well as recompiler instructions. Thus this obfuscation.
39 *
40 * @{
41 */
42
43#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44#define IEM_MC_END() }
45
46/** Internal macro. */
47#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48 do \
49 { \
50 VBOXSTRICTRC rcStrict2 = a_Expr; \
51 if (rcStrict2 == VINF_SUCCESS) \
52 { /* likely */ } \
53 else \
54 return rcStrict2; \
55 } while (0)
56
57
58/** Dummy MC that prevents native recompilation. */
59#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61/** Advances RIP, finishes the instruction and returns.
62 * This may include raising debug exceptions and such. */
63#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66 return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68 * @note only usable in 16-bit op size mode. */
69#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70 return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73 return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76 return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79 return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82 return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
85#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
86 do { \
87 if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)))) \
88 { /* probable */ } \
89 else return iemRaiseDeviceNotAvailable(pVCpu); \
90 } while (0)
91#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
92 do { \
93 if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)))) \
94 { /* probable */ } \
95 else return iemRaiseDeviceNotAvailable(pVCpu); \
96 } while (0)
97#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
98 do { \
99 if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
100 { /* probable */ } \
101 else return iemRaiseMathFault(pVCpu); \
102 } while (0)
103#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
104 do { \
105 /* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
106 be reduced to a single compare branch in the more probably code path. */ \
107 if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) \
108 | (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
109 | (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
110 == (XSAVE_C_YMM | XSAVE_C_SSE | X86_CR4_OSXSAVE))) \
111 { /* probable */ } \
112 else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
113 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
114 return iemRaiseUndefinedOpcode(pVCpu); \
115 else \
116 return iemRaiseDeviceNotAvailable(pVCpu); \
117 } while (0)
118AssertCompile(!((XSAVE_C_YMM | XSAVE_C_SSE) & X86_CR4_OSXSAVE));
119AssertCompile(!((XSAVE_C_YMM | XSAVE_C_SSE) & X86_CR0_TS));
120AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
121#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
122 do { \
123 /* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
124 single compare branch in the more probable code path. */ \
125 if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
126 | (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
127 == X86_CR4_OSFXSR)) \
128 { /* likely */ } \
129 else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
130 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
131 return iemRaiseUndefinedOpcode(pVCpu); \
132 else \
133 return iemRaiseDeviceNotAvailable(pVCpu); \
134 } while (0)
135AssertCompile(!((X86_CR0_EM | X86_CR0_TS) & X86_CR4_OSFXSR));
136#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
137 do { \
138 /* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
139 single compare branch in the more probable code path. */ \
140 if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
141 | (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
142 { /* probable */ } \
143 else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
144 return iemRaiseUndefinedOpcode(pVCpu); \
145 else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
146 return iemRaiseDeviceNotAvailable(pVCpu); \
147 else \
148 return iemRaiseMathFault(pVCpu); \
149 } while (0)
150AssertCompile(!((X86_CR0_EM | X86_CR0_TS) & X86_FSW_ES));
151/** @todo recomp: this one is slightly problematic as the recompiler doesn't
152 * count the CPL into the TB key. However it is safe enough for now, as
153 * it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
154 * emitted for it. */
155#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
156 do { \
157 if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
158 else return iemRaiseGeneralProtectionFault0(pVCpu); \
159 } while (0)
160#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
161 do { \
162 if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
163 else return iemRaiseGeneralProtectionFault0(pVCpu); \
164 } while (0)
165#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
166 do { \
167 if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) | IEM_GET_CPU_MODE(pVCpu)) \
168 == (X86_CR4_FSGSBASE | IEMMODE_64BIT))) \
169 { /* probable */ } \
170 else return iemRaiseUndefinedOpcode(pVCpu); \
171 } while (0)
172AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
173#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
174 do { \
175 if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
176 else return iemRaiseGeneralProtectionFault0(pVCpu); \
177 } while (0)
178#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
179 do { \
180 if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
181 & (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
182 { /* probable */ } \
183 else \
184 { \
185 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
186 return iemRaiseSimdFpException(pVCpu); \
187 return iemRaiseUndefinedOpcode(pVCpu); \
188 } \
189 } while (0)
190#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
191 do { \
192 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
193 return iemRaiseSimdFpException(pVCpu); \
194 return iemRaiseUndefinedOpcode(pVCpu); \
195 } while (0)
196
197
198#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
199#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
200#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
201#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
202#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
203#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
204#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
205/** @note IEMAllInstPython.py duplicates the expansion. */
206#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
207 uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
208 uint32_t *a_pName = &a_Name
209/** @note IEMAllInstPython.py duplicates the expansion. */
210#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
211#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
212 do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
213#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
214 AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
215 ("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
216 pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
217 (a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
218 (pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
219 pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
220 Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
221 } while (0)
222#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
223#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
224
225/** ASSUMES the source variable not used after this statement. */
226#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
227
228#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
229#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
230#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
231#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
232#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
233#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
234#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
235#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
236#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
237#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
238#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
239#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
240#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
241#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
242#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
243#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
244#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
245#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
246 (a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
247 (a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
248 } while(0)
249#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
250 (a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
251 (a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
252 } while(0)
253#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
254 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
255 (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
256 } while (0)
257#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
258 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
259 (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
260 } while (0)
261#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
262 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
263 (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
264 } while (0)
265/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
266#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
267 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
268 (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
269 } while (0)
270#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
271 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
272 (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
273 } while (0)
274/** @note Not for IOPL or IF testing or modification. */
275#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
276#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
277#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
278#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
279#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
280
281#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
282#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
283#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
284#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
285#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
286#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
287#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
288#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
289#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
290#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
291 *iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
292 *iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
293 } while(0)
294#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
295 *iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
296 *iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
297 } while(0)
298#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
299
300/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
301#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
302 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
303 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
304 } while (0)
305#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
306 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
307 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
308 } while (0)
309#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
310 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
311
312
313#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
314#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
315#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
316#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
317/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
318 * Use IEM_MC_CLEAR_HIGH_GREG_U64! */
319#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
320#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
321#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
322#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
323#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
324#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
325#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
326#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
327/** @note Not for IOPL or IF testing or modification.
328 * @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
329#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
330#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
331#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
332
333#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
334#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
335 do { \
336 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
337 *pu32Reg += (a_u32Value); \
338 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
339 } while (0)
340#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
341
342#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
343#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
344 do { \
345 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
346 *pu32Reg -= (a_u8Const); \
347 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
348 } while (0)
349#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
350#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
351
352#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
353#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
354#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
355#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
356#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
357#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
358#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
359
360#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
361#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
362#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
363#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
364
365#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
366#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
367#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
368
369#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) |= (a_u8Mask); } while (0)
370#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) |= (a_u16Mask); } while (0)
371#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
372
373#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
374#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
375#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
376
377#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
378
379#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
380#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
381#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
382
383#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
384
385#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
386
387#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
388#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
389#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
390 do { \
391 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
392 *pu32Reg &= (a_u32Value); \
393 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
394 } while (0)
395#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
396
397#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
398#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
399#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
400 do { \
401 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
402 *pu32Reg |= (a_u32Value); \
403 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
404 } while (0)
405#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
406
407#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
408#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
409#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
410
411/** @note Not for IOPL or IF modification. */
412#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
413/** @note Not for IOPL or IF modification. */
414#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
415/** @note Not for IOPL or IF modification. */
416#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
417
418#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
419
420/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
421#define IEM_MC_FPU_TO_MMX_MODE() do { \
422 iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
423 pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
424 pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
425 } while (0)
426
427/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
428#define IEM_MC_FPU_FROM_MMX_MODE() do { \
429 iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
430 pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
431 pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
432 } while (0)
433
434#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
435 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
436#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
437 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
438#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
439 do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
440#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
441 do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
442#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
443 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
444 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
445 } while (0)
446#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
447 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
448 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
449 } while (0)
450#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
451 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
452 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
453 } while (0)
454#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
455 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
456 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
457 } while (0)
458#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
459 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
460 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
461 } while (0)
462#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
463 (a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
464#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
465 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
466#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
467 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
468#define IEM_MC_MODIFIED_MREG(a_iMReg) \
469 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
470#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
471 do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
472
473#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
474 do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
475 if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
476 if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
477 if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
478 } while (0)
479#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
480 do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
481 (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
482 } while (0)
483#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
484 do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
485 (a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
486 } while (0)
487#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
488 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
489#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
490 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
491#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
492 do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
493#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
494 do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
495#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
496 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
497 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
498 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
499 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
500 } while (0)
501#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
502 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
503 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
504 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
505 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
506 } while (0)
507#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
508 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
509 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
510 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
511 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
512 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
513 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
514 } while (0)
515#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
516 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
517 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
518 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
519 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
520 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
521 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
522 } while (0)
523#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
524 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
525 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
526 } while (0)
527#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
528 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
529 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
530 } while (0)
531#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
532 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
533#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
534 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
535#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
536 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
537#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
538 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
539#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
540 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
541#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
542 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
543
544#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
545 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
546 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
547 } while (0)
548
549#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
550 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
551#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
552 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
553#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
554 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
555#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
556 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
557 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
558 } while (0)
559
560#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
561 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
562 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
563 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
564 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
565 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
566 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
567 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
568 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
569 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
570 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
571 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
572 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
573 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
574 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
575 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
576 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
577 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
578 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
579 } while (0)
580#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
581 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
582 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
583 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
584 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
585 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
586 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
587 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
588 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
589 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
590 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
591 } while (0)
592#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
593 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
594 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
595 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
596 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
597 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
598 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
599 } while (0)
600#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
601 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
602 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
603 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
604 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
605 } while (0)
606
607#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
608 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
609#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
610 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
611#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
612 (a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
613#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
614 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
615#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
616 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
617#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
618 (a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
619#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
620 (a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
621#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
622 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
623 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
624 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
625 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
626 } while (0)
627
628#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
629 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
630 (a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
631 } while (0)
632#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
633 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
634 (a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
635 } while (0)
636#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
637 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
638 if ((a_iDQword) == 0) \
639 { \
640 (a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
641 (a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
642 } \
643 else \
644 { \
645 (a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
646 (a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
647 } \
648 } while (0)
649#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
650 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
651 (a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
652 (a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
653 (a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
654 (a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
655 } while (0)
656
657#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
658 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
659 if ((a_iDQword) == 0) \
660 { \
661 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
662 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
663 } \
664 else \
665 { \
666 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
667 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
668 } \
669 } while (0)
670
671#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
672#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
673 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
674 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
675 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
676 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
677 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
678 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
679 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
680 } while (0)
681#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
682 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
683 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
684 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
685 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
686 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
687 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
688 } while (0)
689#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
690 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
691 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
692 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
693 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
694 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
695 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
696 } while (0)
697#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
698 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
699 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
700 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
701 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
702 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
703 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
704 } while (0)
705#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
706 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
707 if ((a_iDwDst) < 4) \
708 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
709 else \
710 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
711 } while (0)
712#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
713 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
714 if ((a_iQwDst) < 2) \
715 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
716 else \
717 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
718 } while (0)
719#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
720 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
721 if ((a_iQword) < 2) \
722 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
723 else \
724 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
725 } while (0)
726
727#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
728 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
729 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
730 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
731 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
732 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
733 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
734 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
735 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
736 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
737 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
738 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
739 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
740 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
741 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
742 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
743 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
744 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
745 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
746 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
747 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
748 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
749 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
750 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
751 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
752 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
753 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
754 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
755 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
756 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
757 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
758 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
759 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
760 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
761 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
762 } while (0)
763#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
764 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
765 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
766 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
767 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
768 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
769 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
770 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
771 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
772 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
773 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
774 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
775 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
776 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
777 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
778 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
779 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
780 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
781 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
782 } while (0)
783#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
784 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
785 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
786 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
787 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
788 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
789 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
790 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
791 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
792 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
793 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
794 } while (0)
795#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
796 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
797 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
798 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
799 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
800 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
801 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
802 } while (0)
803#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
804 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
805 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
806 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
807 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
808 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
809 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
810 } while (0)
811
812#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
813 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
814#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
815 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
816#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
817 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
818#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
819 do { uintptr_t const iYRegTmp = (a_iYReg); \
820 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
821 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
822 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
823 } while (0)
824
825#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
826 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
827 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
828 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
829 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
830 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
831 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
832 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
833 } while (0)
834#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
835 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
836 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
837 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
838 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
839 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
840 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
841 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
842 } while (0)
843#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
844 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
845 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
846 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
847 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
848 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
849 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
850 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
851 } while (0)
852
853#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
854 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
855 uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
856 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
857 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
858 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
859 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
860 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
861 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
862 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
863 } while (0)
864#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
865 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
866 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
867 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
868 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
869 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
870 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
871 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
872 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
873 } while (0)
874#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
875 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
876 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
877 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
878 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
879 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
880 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
881 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
882 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
883 } while (0)
884#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
885 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
886 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
887 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
888 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
889 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
890 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
891 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
892 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
893 } while (0)
894#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
895 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
896 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
897 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
898 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
899 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
900 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
901 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
902 } while (0)
903#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
904 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
905 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
906 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
907 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
908 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
909 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
910 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
911 } while (0)
912
913#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
914 do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
915
916#ifndef IEM_WITH_SETJMP
917# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
918 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
919# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
920 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
921# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
922 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
923#else
924# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
925 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
926# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
927 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
928# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
929 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
930
931# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
932 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
933# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
934 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
935# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
936 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
937#endif
938
939#ifndef IEM_WITH_SETJMP
940# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
941 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
942# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
943 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
944# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
945 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
946#else
947# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
948 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
949# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
950 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
951# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
952 ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
953
954# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
955 ((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
956# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
957 ((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
958# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
959 ((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
960#endif
961
962#ifndef IEM_WITH_SETJMP
963# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
964 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
965# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
966 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
967# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
968 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
969#else
970# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
971 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
972# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
973 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
974# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
975 ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
976
977# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
978 ((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
979# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
980 ((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
981# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
982 ((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
983#endif
984
985#ifndef IEM_WITH_SETJMP
986# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
987 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
988# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
989 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
990# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
991 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
992# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
993 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
994#else
995# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
996 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
997# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
998 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
999# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1000 ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1001# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1002 ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1003
1004# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1005 ((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1006# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1007 ((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1008# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1009 ((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1010# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1011 ((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1012#endif
1013
1014#ifndef IEM_WITH_SETJMP
1015# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1016 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1017# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1018 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1019# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1020 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1021# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1022 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1023#else
1024# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1025 ((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1026# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1027 ((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1028# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1029 iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1030# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1031 iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1032
1033# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1034 ((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1035# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1036 ((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1037# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1038 iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1039# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1040 iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1041#endif
1042
1043#ifndef IEM_WITH_SETJMP
1044# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1045 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1046# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1047 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1048# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1049 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1050
1051# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1052 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1053# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1054 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1055# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1056 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
1057# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1058 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
1059
1060# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1061 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1062 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1063 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1064 } while (0)
1065
1066# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1067 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1068 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1069 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1070 } while (0)
1071
1072# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1073 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1074 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1075 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1076 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1077 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1078 } while (0)
1079
1080# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1081 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1082 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1083 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1084 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1085 } while (0)
1086
1087# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1088 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1089 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1090 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1091 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1092 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1093 } while (0)
1094# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1095 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1096 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1097 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1098 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1099 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1100 } while (0)
1101
1102#else
1103# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1104 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1105# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1106 iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1107# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1108 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1109
1110# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1111 iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1112# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1113 iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1114# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1115 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1116# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1117 (a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1118# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1119 (a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1120
1121# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1122 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1123# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1124 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1125# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1126 iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1127
1128# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1129 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1130# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1131 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1132# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1133 iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1134# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
1135 (a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
1136# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
1137 (a_XmmDst).au64[(a_iQWord)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem))
1138
1139# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1140 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1141 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1142 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1143 } while (0)
1144# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1145 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1146 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1147 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1148 } while (0)
1149
1150# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1151 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1152 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1153 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1154 } while (0)
1155# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1156 iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1157 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1158 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1159 } while (0)
1160
1161# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1162 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1163 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1164 (a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1165 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1166 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1167 } while (0)
1168# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1169 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1170 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1171 (a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1172 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1173 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1174 } while (0)
1175
1176# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1177 (a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1178 (a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1179 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1180 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1181 } while (0)
1182# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1183 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1184 (a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1185 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1186 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1187 } while (0)
1188
1189
1190# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1191 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1192 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1193 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1194 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1195 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1196 } while (0)
1197# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1198 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1199 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1200 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1201 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1202 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1203 } while (0)
1204
1205# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1206 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1207 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1208 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1209 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1210 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1211 } while (0)
1212# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1213 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1214 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1215 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1216 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1217 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1218 } while (0)
1219
1220#endif
1221
1222#ifndef IEM_WITH_SETJMP
1223# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1224 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1225# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1226 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1227# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1228 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1229
1230# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1231 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1232# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1233 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1234# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1235 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1236#else
1237# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1238 iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1239# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1240 iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1241# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1242 iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1243
1244# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1245 iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1246# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1247 iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1248# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1249 iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1250
1251# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1252 iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1253# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1254 iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1255# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1256 iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1257
1258# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1259 iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1260# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1261 iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1262# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1263 iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1264#endif
1265
1266
1267
1268#ifndef IEM_WITH_SETJMP
1269# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1270 do { \
1271 uint8_t u8Tmp; \
1272 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1273 (a_u16Dst) = u8Tmp; \
1274 } while (0)
1275# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1276 do { \
1277 uint8_t u8Tmp; \
1278 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1279 (a_u32Dst) = u8Tmp; \
1280 } while (0)
1281# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1282 do { \
1283 uint8_t u8Tmp; \
1284 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1285 (a_u64Dst) = u8Tmp; \
1286 } while (0)
1287# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1288 do { \
1289 uint16_t u16Tmp; \
1290 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1291 (a_u32Dst) = u16Tmp; \
1292 } while (0)
1293# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1294 do { \
1295 uint16_t u16Tmp; \
1296 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1297 (a_u64Dst) = u16Tmp; \
1298 } while (0)
1299# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1300 do { \
1301 uint32_t u32Tmp; \
1302 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1303 (a_u64Dst) = u32Tmp; \
1304 } while (0)
1305#else /* IEM_WITH_SETJMP */
1306# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1307 ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1308# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1309 ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1310# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1311 ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1312# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1313 ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1314# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1315 ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1316# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1317 ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1318
1319# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1320 ((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1321# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1322 ((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1323# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1324 ((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1325# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1326 ((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1327# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1328 ((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1329# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1330 ((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1331#endif /* IEM_WITH_SETJMP */
1332
1333#ifndef IEM_WITH_SETJMP
1334# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1335 do { \
1336 uint8_t u8Tmp; \
1337 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1338 (a_u16Dst) = (int8_t)u8Tmp; \
1339 } while (0)
1340# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1341 do { \
1342 uint8_t u8Tmp; \
1343 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1344 (a_u32Dst) = (int8_t)u8Tmp; \
1345 } while (0)
1346# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1347 do { \
1348 uint8_t u8Tmp; \
1349 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1350 (a_u64Dst) = (int8_t)u8Tmp; \
1351 } while (0)
1352# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1353 do { \
1354 uint16_t u16Tmp; \
1355 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1356 (a_u32Dst) = (int16_t)u16Tmp; \
1357 } while (0)
1358# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1359 do { \
1360 uint16_t u16Tmp; \
1361 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1362 (a_u64Dst) = (int16_t)u16Tmp; \
1363 } while (0)
1364# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1365 do { \
1366 uint32_t u32Tmp; \
1367 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1368 (a_u64Dst) = (int32_t)u32Tmp; \
1369 } while (0)
1370#else /* IEM_WITH_SETJMP */
1371# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1372 ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1373# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1374 ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1375# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1376 ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1377# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1378 ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1379# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1380 ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1381# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1382 ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1383
1384# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1385 ((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1386# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1387 ((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1388# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1389 ((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1390# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1391 ((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1392# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1393 ((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1394# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1395 ((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1396#endif /* IEM_WITH_SETJMP */
1397
1398#ifndef IEM_WITH_SETJMP
1399# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1400 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1401# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1402 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1403# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1404 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1405# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1406 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1407#else
1408# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1409 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1410# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1411 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1412# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1413 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1414# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1415 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1416
1417# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1418 iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1419# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1420 iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1421# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1422 iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1423# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1424 iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1425#endif
1426
1427#ifndef IEM_WITH_SETJMP
1428# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1429 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1430# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1431 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1432# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1433 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1434# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1435 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1436#else
1437# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1438 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1439# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1440 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1441# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1442 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1443# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1444 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1445
1446# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1447 iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1448# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1449 iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1450# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1451 iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1452# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1453 iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1454#endif
1455
1456#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1457#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1458#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1459#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1460#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1461#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1462#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1463 do { \
1464 (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1465 (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1466 } while (0)
1467#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1468 do { \
1469 (a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1470 (a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1471 } while (0)
1472
1473#ifndef IEM_WITH_SETJMP
1474# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1475 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1476# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1477 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1478# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1479 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1480#else
1481# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1482 iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1483# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1484 iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1485# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1486 iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1487
1488# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1489 iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1490# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1491 iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1492# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1493 iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1494#endif
1495
1496#ifndef IEM_WITH_SETJMP
1497# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1498 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1499# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1500 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1501# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1502 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1503#else
1504# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1505 iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1506# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1507 iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1508# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1509 iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1510
1511# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1512 iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1513# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1514 iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1515# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1516 iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1517#endif
1518
1519/* Regular stack push and pop: */
1520#ifndef IEM_WITH_SETJMP
1521# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1522# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1523# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1524# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1525
1526# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1527# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1528# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1529#else
1530# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1531# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1532# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1533# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1534
1535# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1536# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1537# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1538#endif
1539
1540/* 32-bit flat stack push and pop: */
1541#ifndef IEM_WITH_SETJMP
1542# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1543# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1544# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1545
1546# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1547# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1548#else
1549# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1550# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1551# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1552
1553# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1554# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1555#endif
1556
1557/* 64-bit flat stack push and pop: */
1558#ifndef IEM_WITH_SETJMP
1559# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1560# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1561
1562# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1563# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1564#else
1565# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1566# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1567
1568# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1569# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1570#endif
1571
1572
1573/* 8-bit */
1574
1575/**
1576 * Maps guest memory for byte atomic read+write direct (or bounce) buffer
1577 * acccess, for atomic operations.
1578 *
1579 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1580 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1581 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1582 * @param[in] a_GCPtrMem The memory address.
1583 * @remarks Will return/long jump on errors.
1584 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1585 */
1586#ifndef IEM_WITH_SETJMP
1587# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1588 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1589 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1590#else
1591# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1592 (a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1593#endif
1594
1595/**
1596 * Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1597 *
1598 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1599 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1600 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1601 * @param[in] a_GCPtrMem The memory address.
1602 * @remarks Will return/long jump on errors.
1603 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1604 */
1605#ifndef IEM_WITH_SETJMP
1606# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1607 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1608 (a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1609#else
1610# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1611 (a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1612#endif
1613
1614/**
1615 * Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1616 *
1617 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1618 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1619 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1620 * @param[in] a_GCPtrMem The memory address.
1621 * @remarks Will return/long jump on errors.
1622 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1623 */
1624#ifndef IEM_WITH_SETJMP
1625# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1626 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1627 (a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1628#else
1629# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1630 (a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1631#endif
1632
1633/**
1634 * Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1635 *
1636 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1637 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1638 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1639 * @param[in] a_GCPtrMem The memory address.
1640 * @remarks Will return/long jump on errors.
1641 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1642 */
1643#ifndef IEM_WITH_SETJMP
1644# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1645 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1646 (a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1647#else
1648# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1649 (a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1650#endif
1651
1652/**
1653 * Maps guest memory for byte atomic read+write direct (or bounce) buffer
1654 * acccess, flat address variant.
1655 *
1656 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1657 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1658 * @param[in] a_GCPtrMem The memory address.
1659 * @remarks Will return/long jump on errors.
1660 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1661 */
1662#ifndef IEM_WITH_SETJMP
1663# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1664 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1665 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1666#else
1667# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1668 (a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1669#endif
1670
1671/**
1672 * Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1673 * address variant.
1674 *
1675 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1676 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1677 * @param[in] a_GCPtrMem The memory address.
1678 * @remarks Will return/long jump on errors.
1679 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1680 */
1681#ifndef IEM_WITH_SETJMP
1682# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1683 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1684 (a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1685#else
1686# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1687 (a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1688#endif
1689
1690/**
1691 * Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1692 * address variant.
1693 *
1694 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1695 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1696 * @param[in] a_GCPtrMem The memory address.
1697 * @remarks Will return/long jump on errors.
1698 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1699 */
1700#ifndef IEM_WITH_SETJMP
1701# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1702 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1703 (a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1704#else
1705# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1706 (a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1707#endif
1708
1709/**
1710 * Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1711 * address variant.
1712 *
1713 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1714 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1715 * @param[in] a_GCPtrMem The memory address.
1716 * @remarks Will return/long jump on errors.
1717 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1718 */
1719#ifndef IEM_WITH_SETJMP
1720# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1721 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1722 (a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1723#else
1724# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1725 (a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1726#endif
1727
1728
1729/* 16-bit */
1730
1731/**
1732 * Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1733 *
1734 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1735 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1736 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1737 * @param[in] a_GCPtrMem The memory address.
1738 * @remarks Will return/long jump on errors.
1739 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1740 */
1741#ifndef IEM_WITH_SETJMP
1742# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1743 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1744 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1745#else
1746# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1747 (a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1748#endif
1749
1750/**
1751 * Maps guest memory for word read+write direct (or bounce) buffer acccess.
1752 *
1753 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1754 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1755 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1756 * @param[in] a_GCPtrMem The memory address.
1757 * @remarks Will return/long jump on errors.
1758 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1759 */
1760#ifndef IEM_WITH_SETJMP
1761# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1762 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1763 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1764#else
1765# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1766 (a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1767#endif
1768
1769/**
1770 * Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1771 *
1772 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1773 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1774 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1775 * @param[in] a_GCPtrMem The memory address.
1776 * @remarks Will return/long jump on errors.
1777 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1778 */
1779#ifndef IEM_WITH_SETJMP
1780# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1781 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1782 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1783#else
1784# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1785 (a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1786#endif
1787
1788/**
1789 * Maps guest memory for word readonly direct (or bounce) buffer acccess.
1790 *
1791 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1792 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1793 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1794 * @param[in] a_GCPtrMem The memory address.
1795 * @remarks Will return/long jump on errors.
1796 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1797 */
1798#ifndef IEM_WITH_SETJMP
1799# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1800 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1801 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1802#else
1803# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1804 (a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1805#endif
1806
1807/**
1808 * Maps guest memory for word atomic read+write direct (or bounce) buffer
1809 * acccess, flat address variant.
1810 *
1811 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1812 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1813 * @param[in] a_GCPtrMem The memory address.
1814 * @remarks Will return/long jump on errors.
1815 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1816 */
1817#ifndef IEM_WITH_SETJMP
1818# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1819 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1820 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1821#else
1822# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1823 (a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1824#endif
1825
1826/**
1827 * Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1828 * address variant.
1829 *
1830 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1831 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1832 * @param[in] a_GCPtrMem The memory address.
1833 * @remarks Will return/long jump on errors.
1834 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1835 */
1836#ifndef IEM_WITH_SETJMP
1837# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1838 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1839 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1840#else
1841# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1842 (a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1843#endif
1844
1845/**
1846 * Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1847 * address variant.
1848 *
1849 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1850 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1851 * @param[in] a_GCPtrMem The memory address.
1852 * @remarks Will return/long jump on errors.
1853 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1854 */
1855#ifndef IEM_WITH_SETJMP
1856# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1857 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1858 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1859#else
1860# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1861 (a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1862#endif
1863
1864/**
1865 * Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1866 * address variant.
1867 *
1868 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1869 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1870 * @param[in] a_GCPtrMem The memory address.
1871 * @remarks Will return/long jump on errors.
1872 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1873 */
1874#ifndef IEM_WITH_SETJMP
1875# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1876 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1877 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1878#else
1879# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1880 (a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1881#endif
1882
1883/** int16_t alias. */
1884#ifndef IEM_WITH_SETJMP
1885# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1886 IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1887#else
1888# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1889 (a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1890#endif
1891
1892/** Flat int16_t alias. */
1893#ifndef IEM_WITH_SETJMP
1894# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1895 IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1896#else
1897# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1898 (a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1899#endif
1900
1901
1902/* 32-bit */
1903
1904/**
1905 * Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1906 *
1907 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1908 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1909 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1910 * @param[in] a_GCPtrMem The memory address.
1911 * @remarks Will return/long jump on errors.
1912 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1913 */
1914#ifndef IEM_WITH_SETJMP
1915# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1916 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1917 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1918#else
1919# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1920 (a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1921#endif
1922
1923/**
1924 * Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1925 *
1926 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1927 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1928 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1929 * @param[in] a_GCPtrMem The memory address.
1930 * @remarks Will return/long jump on errors.
1931 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1932 */
1933#ifndef IEM_WITH_SETJMP
1934# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1935 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1936 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
1937#else
1938# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1939 (a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1940#endif
1941
1942/**
1943 * Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1944 *
1945 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1946 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1947 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1948 * @param[in] a_GCPtrMem The memory address.
1949 * @remarks Will return/long jump on errors.
1950 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1951 */
1952#ifndef IEM_WITH_SETJMP
1953# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1954 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1955 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
1956#else
1957# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1958 (a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1959#endif
1960
1961/**
1962 * Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1963 *
1964 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1965 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1966 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1967 * @param[in] a_GCPtrMem The memory address.
1968 * @remarks Will return/long jump on errors.
1969 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1970 */
1971#ifndef IEM_WITH_SETJMP
1972# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1973 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1974 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
1975#else
1976# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1977 (a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1978#endif
1979
1980/**
1981 * Maps guest memory for dword atomic read+write direct (or bounce) buffer
1982 * acccess, flat address variant.
1983 *
1984 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1985 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1986 * @param[in] a_GCPtrMem The memory address.
1987 * @remarks Will return/long jump on errors.
1988 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1989 */
1990#ifndef IEM_WITH_SETJMP
1991# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1992 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1993 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1994#else
1995# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1996 (a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1997#endif
1998
1999/**
2000 * Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2001 * flat address variant.
2002 *
2003 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
2004 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2005 * @param[in] a_GCPtrMem The memory address.
2006 * @remarks Will return/long jump on errors.
2007 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2008 */
2009#ifndef IEM_WITH_SETJMP
2010# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2011 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2012 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2013#else
2014# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2015 (a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2016#endif
2017
2018/**
2019 * Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2020 * address variant.
2021 *
2022 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
2023 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2024 * @param[in] a_GCPtrMem The memory address.
2025 * @remarks Will return/long jump on errors.
2026 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2027 */
2028#ifndef IEM_WITH_SETJMP
2029# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2030 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2031 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2032#else
2033# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2034 (a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2035#endif
2036
2037/**
2038 * Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2039 * address variant.
2040 *
2041 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
2042 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2043 * @param[in] a_GCPtrMem The memory address.
2044 * @remarks Will return/long jump on errors.
2045 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2046 */
2047#ifndef IEM_WITH_SETJMP
2048# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2049 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2050 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2051#else
2052# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2053 (a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2054#endif
2055
2056/** int32_t alias. */
2057#ifndef IEM_WITH_SETJMP
2058# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2059 IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2060#else
2061# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2062 (a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2063#endif
2064
2065/** Flat int32_t alias. */
2066#ifndef IEM_WITH_SETJMP
2067# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2068 IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2069#else
2070# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2071 (a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2072#endif
2073
2074/** RTFLOAT32U alias. */
2075#ifndef IEM_WITH_SETJMP
2076# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2077 IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2078#else
2079# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2080 (a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2081#endif
2082
2083/** Flat RTFLOAT32U alias. */
2084#ifndef IEM_WITH_SETJMP
2085# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2086 IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2087#else
2088# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2089 (a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2090#endif
2091
2092
2093/* 64-bit */
2094
2095/**
2096 * Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2097 *
2098 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2099 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2100 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2101 * @param[in] a_GCPtrMem The memory address.
2102 * @remarks Will return/long jump on errors.
2103 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2104 */
2105#ifndef IEM_WITH_SETJMP
2106# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2107 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2108 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2109#else
2110# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2111 (a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2112#endif
2113
2114/**
2115 * Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2116 *
2117 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2118 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2119 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2120 * @param[in] a_GCPtrMem The memory address.
2121 * @remarks Will return/long jump on errors.
2122 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2123 */
2124#ifndef IEM_WITH_SETJMP
2125# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2126 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2127 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2128#else
2129# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2130 (a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2131#endif
2132
2133/**
2134 * Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2135 *
2136 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2137 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2138 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2139 * @param[in] a_GCPtrMem The memory address.
2140 * @remarks Will return/long jump on errors.
2141 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2142 */
2143#ifndef IEM_WITH_SETJMP
2144# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2145 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2146 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2147#else
2148# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2149 (a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2150#endif
2151
2152/**
2153 * Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2154 *
2155 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2156 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2157 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2158 * @param[in] a_GCPtrMem The memory address.
2159 * @remarks Will return/long jump on errors.
2160 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2161 */
2162#ifndef IEM_WITH_SETJMP
2163# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2164 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2165 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2166#else
2167# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2168 (a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2169#endif
2170
2171/**
2172 * Maps guest memory for qword atomic read+write direct (or bounce) buffer
2173 * acccess, flat address variant.
2174 *
2175 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2176 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2177 * @param[in] a_GCPtrMem The memory address.
2178 * @remarks Will return/long jump on errors.
2179 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2180 */
2181#ifndef IEM_WITH_SETJMP
2182# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2183 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2184 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2185#else
2186# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2187 (a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2188#endif
2189
2190/**
2191 * Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2192 * flat address variant.
2193 *
2194 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2195 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2196 * @param[in] a_GCPtrMem The memory address.
2197 * @remarks Will return/long jump on errors.
2198 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2199 */
2200#ifndef IEM_WITH_SETJMP
2201# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2202 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2203 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2204#else
2205# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2206 (a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2207#endif
2208
2209/**
2210 * Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2211 * address variant.
2212 *
2213 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2214 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2215 * @param[in] a_GCPtrMem The memory address.
2216 * @remarks Will return/long jump on errors.
2217 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2218 */
2219#ifndef IEM_WITH_SETJMP
2220# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2221 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2222 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2223#else
2224# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2225 (a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2226#endif
2227
2228/**
2229 * Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2230 * address variant.
2231 *
2232 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2233 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2234 * @param[in] a_GCPtrMem The memory address.
2235 * @remarks Will return/long jump on errors.
2236 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2237 */
2238#ifndef IEM_WITH_SETJMP
2239# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2240 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2241 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2242#else
2243# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2244 (a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2245#endif
2246
2247/** int64_t alias. */
2248#ifndef IEM_WITH_SETJMP
2249# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2250 IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2251#else
2252# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2253 (a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2254#endif
2255
2256/** Flat int64_t alias. */
2257#ifndef IEM_WITH_SETJMP
2258# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2259 IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2260#else
2261# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2262 (a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2263#endif
2264
2265/** RTFLOAT64U alias. */
2266#ifndef IEM_WITH_SETJMP
2267# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2268 IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2269#else
2270# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2271 (a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2272#endif
2273
2274/** Flat RTFLOAT64U alias. */
2275#ifndef IEM_WITH_SETJMP
2276# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2277 IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2278#else
2279# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2280 (a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2281#endif
2282
2283
2284/* 128-bit */
2285
2286/**
2287 * Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2288 *
2289 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2290 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2291 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2292 * @param[in] a_GCPtrMem The memory address.
2293 * @remarks Will return/long jump on errors.
2294 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2295 */
2296#ifndef IEM_WITH_SETJMP
2297# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2298 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2299 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2300#else
2301# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2302 (a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2303#endif
2304
2305/**
2306 * Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2307 *
2308 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2309 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2310 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2311 * @param[in] a_GCPtrMem The memory address.
2312 * @remarks Will return/long jump on errors.
2313 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2314 */
2315#ifndef IEM_WITH_SETJMP
2316# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2317 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2318 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2319#else
2320# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2321 (a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2322#endif
2323
2324/**
2325 * Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2326 *
2327 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2328 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2329 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2330 * @param[in] a_GCPtrMem The memory address.
2331 * @remarks Will return/long jump on errors.
2332 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2333 */
2334#ifndef IEM_WITH_SETJMP
2335# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2336 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2337 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2338#else
2339# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2340 (a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2341#endif
2342
2343/**
2344 * Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2345 *
2346 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2347 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2348 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2349 * @param[in] a_GCPtrMem The memory address.
2350 * @remarks Will return/long jump on errors.
2351 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2352 */
2353#ifndef IEM_WITH_SETJMP
2354# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2355 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2356 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2357#else
2358# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2359 (a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2360#endif
2361
2362/**
2363 * Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2364 * access, flat address variant.
2365 *
2366 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2367 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2368 * @param[in] a_GCPtrMem The memory address.
2369 * @remarks Will return/long jump on errors.
2370 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2371 */
2372#ifndef IEM_WITH_SETJMP
2373# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2374 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2375 (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2376#else
2377# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2378 (a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2379#endif
2380
2381/**
2382 * Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2383 * flat address variant.
2384 *
2385 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2386 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2387 * @param[in] a_GCPtrMem The memory address.
2388 * @remarks Will return/long jump on errors.
2389 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2390 */
2391#ifndef IEM_WITH_SETJMP
2392# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2393 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2394 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2395#else
2396# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2397 (a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2398#endif
2399
2400/**
2401 * Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2402 * flat address variant.
2403 *
2404 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2405 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2406 * @param[in] a_GCPtrMem The memory address.
2407 * @remarks Will return/long jump on errors.
2408 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2409 */
2410#ifndef IEM_WITH_SETJMP
2411# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2412 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2413 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2414#else
2415# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2416 (a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2417#endif
2418
2419/**
2420 * Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2421 * address variant.
2422 *
2423 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2424 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2425 * @param[in] a_GCPtrMem The memory address.
2426 * @remarks Will return/long jump on errors.
2427 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2428 */
2429#ifndef IEM_WITH_SETJMP
2430# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2431 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2432 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2433#else
2434# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2435 (a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2436#endif
2437
2438
2439/* misc */
2440
2441/**
2442 * Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2443 *
2444 * @param[out] a_pr80Mem Where to return the pointer to the mapping.
2445 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2446 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2447 * @param[in] a_GCPtrMem The memory address.
2448 * @remarks Will return/long jump on errors.
2449 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2450 */
2451#ifndef IEM_WITH_SETJMP
2452# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2453 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2454 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2455#else
2456# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2457 (a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2458#endif
2459
2460/**
2461 * Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2462 *
2463 * @param[out] a_pr80Mem Where to return the pointer to the mapping.
2464 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2465 * @param[in] a_GCPtrMem The memory address.
2466 * @remarks Will return/long jump on errors.
2467 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2468 */
2469#ifndef IEM_WITH_SETJMP
2470# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2471 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2472 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2473#else
2474# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2475 (a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2476#endif
2477
2478
2479/**
2480 * Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2481 *
2482 * @param[out] a_pd80Mem Where to return the pointer to the mapping.
2483 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2484 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2485 * @param[in] a_GCPtrMem The memory address.
2486 * @remarks Will return/long jump on errors.
2487 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2488 */
2489#ifndef IEM_WITH_SETJMP
2490# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2491 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2492 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2493#else
2494# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2495 (a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2496#endif
2497
2498/**
2499 * Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2500 *
2501 * @param[out] a_pd80Mem Where to return the pointer to the mapping.
2502 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2503 * @param[in] a_GCPtrMem The memory address.
2504 * @remarks Will return/long jump on errors.
2505 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2506 */
2507#ifndef IEM_WITH_SETJMP
2508# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2509 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2510 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2511#else
2512# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2513 (a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2514#endif
2515
2516
2517
2518/* commit + unmap */
2519
2520/** Commits the memory and unmaps guest memory previously mapped RW.
2521 * @remarks May return.
2522 * @note Implictly frees the a_bMapInfo variable.
2523 */
2524#ifndef IEM_WITH_SETJMP
2525# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2526#else
2527# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2528#endif
2529
2530/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2531 * @remarks May return.
2532 * @note Implictly frees the a_bMapInfo variable.
2533 */
2534#ifndef IEM_WITH_SETJMP
2535# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2536#else
2537# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2538#endif
2539
2540/** Commits the memory and unmaps guest memory previously mapped W.
2541 * @remarks May return.
2542 * @note Implictly frees the a_bMapInfo variable.
2543 */
2544#ifndef IEM_WITH_SETJMP
2545# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2546#else
2547# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2548#endif
2549
2550/** Commits the memory and unmaps guest memory previously mapped R.
2551 * @remarks May return.
2552 * @note Implictly frees the a_bMapInfo variable.
2553 */
2554#ifndef IEM_WITH_SETJMP
2555# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2556#else
2557# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2558#endif
2559
2560
2561/** Commits the memory and unmaps the guest memory unless the FPU status word
2562 * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2563 * that would cause FLD not to store.
2564 *
2565 * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2566 * store, while \#P will not.
2567 *
2568 * @remarks May in theory return - for now.
2569 * @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2570 */
2571#ifndef IEM_WITH_SETJMP
2572# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2573 if ( !(a_u16FSW & X86_FSW_ES) \
2574 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
2575 & ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2576 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2577 else \
2578 iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2579 } while (0)
2580#else
2581# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2582 if ( !(a_u16FSW & X86_FSW_ES) \
2583 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
2584 & ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2585 iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2586 else \
2587 iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2588 } while (0)
2589#endif
2590
2591/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2592 * @note Implictly frees the a_bMapInfo variable. */
2593#ifndef IEM_WITH_SETJMP
2594# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2595#else
2596# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2597#endif
2598
2599
2600
2601/** Calculate efficient address from R/M. */
2602#ifndef IEM_WITH_SETJMP
2603# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2604 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2605#else
2606# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2607 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2608#endif
2609
2610
2611/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2612#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2613#define IEM_MC_NATIVE_ELSE() } else {
2614#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2615
2616#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2617#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2618#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2619#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2620#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2621#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2622#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2623#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2624#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2625
2626/** This can be used to direct the register allocator when dealing with
2627 * x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2628#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2629
2630
2631#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2632#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2633#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2634#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2635#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2636#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
2637#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
2638
2639
2640/** @def IEM_MC_CALL_CIMPL_HLP_RET
2641 * Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2642 */
2643#ifdef VBOX_STRICT
2644#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2645 do { \
2646 uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2647 uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2648 uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2649 uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2650 uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2651 VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2652 if (rcStrictHlp == VINF_SUCCESS) \
2653 { \
2654 AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2655 || ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
2656 && uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2657 || ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2658 && uRipBefore == pVCpu->cpum.GstCtx.rip \
2659 && uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2660 ("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2661 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
2662 if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2663 { /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2664 else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2665 AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS | X86_EFL_RF)) \
2666 == (fEflBefore & ~(X86_EFL_STATUS_BITS | X86_EFL_RF)), \
2667 ("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2668 else \
2669 AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2670 == (fEflBefore & ~(X86_EFL_RF)), \
2671 ("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2672 if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2673 { \
2674 uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2675 AssertMsg( fExecBefore == fExecRecalc \
2676 /* in case ES, DS or SS was external initially (happens alot with HM): */ \
2677 || ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2678 && (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2679 ("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2680 } \
2681 } \
2682 return rcStrictHlp; \
2683 } while (0)
2684#else
2685# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2686#endif
2687
2688/**
2689 * Defers the rest of the instruction emulation to a C implementation routine
2690 * and returns, only taking the standard parameters.
2691 *
2692 * @param a_fFlags IEM_CIMPL_F_XXX.
2693 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2694 * in the native recompiler.
2695 * @param a_pfnCImpl The pointer to the C routine.
2696 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2697 */
2698#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2699 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2700
2701/**
2702 * Defers the rest of instruction emulation to a C implementation routine and
2703 * returns, taking one argument in addition to the standard ones.
2704 *
2705 * @param a_fFlags IEM_CIMPL_F_XXX.
2706 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2707 * in the native recompiler.
2708 * @param a_pfnCImpl The pointer to the C routine.
2709 * @param a0 The argument.
2710 */
2711#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2712 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2713
2714/**
2715 * Defers the rest of the instruction emulation to a C implementation routine
2716 * and returns, taking two arguments in addition to the standard ones.
2717 *
2718 * @param a_fFlags IEM_CIMPL_F_XXX.
2719 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2720 * in the native recompiler.
2721 * @param a_pfnCImpl The pointer to the C routine.
2722 * @param a0 The first extra argument.
2723 * @param a1 The second extra argument.
2724 */
2725#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2726 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2727
2728/**
2729 * Defers the rest of the instruction emulation to a C implementation routine
2730 * and returns, taking three arguments in addition to the standard ones.
2731 *
2732 * @param a_fFlags IEM_CIMPL_F_XXX.
2733 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2734 * in the native recompiler.
2735 * @param a_pfnCImpl The pointer to the C routine.
2736 * @param a0 The first extra argument.
2737 * @param a1 The second extra argument.
2738 * @param a2 The third extra argument.
2739 */
2740#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2741 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2742
2743/**
2744 * Defers the rest of the instruction emulation to a C implementation routine
2745 * and returns, taking four arguments in addition to the standard ones.
2746 *
2747 * @param a_fFlags IEM_CIMPL_F_XXX.
2748 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2749 * in the native recompiler.
2750 * @param a_pfnCImpl The pointer to the C routine.
2751 * @param a0 The first extra argument.
2752 * @param a1 The second extra argument.
2753 * @param a2 The third extra argument.
2754 * @param a3 The fourth extra argument.
2755 */
2756#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2757 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2758
2759/**
2760 * Defers the rest of the instruction emulation to a C implementation routine
2761 * and returns, taking two arguments in addition to the standard ones.
2762 *
2763 * @param a_fFlags IEM_CIMPL_F_XXX.
2764 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2765 * in the native recompiler.
2766 * @param a_pfnCImpl The pointer to the C routine.
2767 * @param a0 The first extra argument.
2768 * @param a1 The second extra argument.
2769 * @param a2 The third extra argument.
2770 * @param a3 The fourth extra argument.
2771 * @param a4 The fifth extra argument.
2772 */
2773#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2774 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2775
2776/**
2777 * Defers the entire instruction emulation to a C implementation routine and
2778 * returns, only taking the standard parameters.
2779 *
2780 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2781 *
2782 * @param a_fFlags IEM_CIMPL_F_XXX.
2783 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2784 * in the native recompiler.
2785 * @param a_pfnCImpl The pointer to the C routine.
2786 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2787 */
2788#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2789 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2790
2791/**
2792 * Defers the entire instruction emulation to a C implementation routine and
2793 * returns, taking one argument in addition to the standard ones.
2794 *
2795 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2796 *
2797 * @param a_fFlags IEM_CIMPL_F_XXX.
2798 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2799 * in the native recompiler.
2800 * @param a_pfnCImpl The pointer to the C routine.
2801 * @param a0 The argument.
2802 */
2803#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2804 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2805
2806/**
2807 * Defers the entire instruction emulation to a C implementation routine and
2808 * returns, taking two arguments in addition to the standard ones.
2809 *
2810 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2811 *
2812 * @param a_fFlags IEM_CIMPL_F_XXX.
2813 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2814 * in the native recompiler.
2815 * @param a_pfnCImpl The pointer to the C routine.
2816 * @param a0 The first extra argument.
2817 * @param a1 The second extra argument.
2818 */
2819#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2820 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2821
2822/**
2823 * Defers the entire instruction emulation to a C implementation routine and
2824 * returns, taking three arguments in addition to the standard ones.
2825 *
2826 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2827 *
2828 * @param a_fFlags IEM_CIMPL_F_XXX.
2829 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2830 * in the native recompiler.
2831 * @param a_pfnCImpl The pointer to the C routine.
2832 * @param a0 The first extra argument.
2833 * @param a1 The second extra argument.
2834 * @param a2 The third extra argument.
2835 */
2836#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2837 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2838
2839
2840/**
2841 * Calls a FPU assembly implementation taking one visible argument.
2842 *
2843 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2844 * @param a0 The first extra argument.
2845 */
2846#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2847 do { \
2848 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2849 } while (0)
2850
2851/**
2852 * Calls a FPU assembly implementation taking two visible arguments.
2853 *
2854 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2855 * @param a0 The first extra argument.
2856 * @param a1 The second extra argument.
2857 */
2858#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2859 do { \
2860 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2861 } while (0)
2862
2863/**
2864 * Calls a FPU assembly implementation taking three visible arguments.
2865 *
2866 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2867 * @param a0 The first extra argument.
2868 * @param a1 The second extra argument.
2869 * @param a2 The third extra argument.
2870 */
2871#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2872 do { \
2873 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2874 } while (0)
2875
2876#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2877 do { \
2878 (a_FpuData).FSW = (a_FSW); \
2879 (a_FpuData).r80Result = *(a_pr80Value); \
2880 } while (0)
2881
2882/** Pushes FPU result onto the stack. */
2883#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2884 iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2885/** Pushes FPU result onto the stack and sets the FPUDP. */
2886#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2887 iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2888
2889/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2890#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2891 iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2892
2893/** Stores FPU result in a stack register. */
2894#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2895 iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2896/** Stores FPU result in a stack register and pops the stack. */
2897#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2898 iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2899/** Stores FPU result in a stack register and sets the FPUDP. */
2900#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2901 iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2902/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2903 * stack. */
2904#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2905 iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2906
2907/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2908#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2909 iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2910/** Free a stack register (for FFREE and FFREEP). */
2911#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2912 iemFpuStackFree(pVCpu, a_iStReg)
2913/** Increment the FPU stack pointer. */
2914#define IEM_MC_FPU_STACK_INC_TOP() \
2915 iemFpuStackIncTop(pVCpu)
2916/** Decrement the FPU stack pointer. */
2917#define IEM_MC_FPU_STACK_DEC_TOP() \
2918 iemFpuStackDecTop(pVCpu)
2919
2920/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2921#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2922 iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2923/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2924#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2925 iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2926/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2927#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2928 iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2929/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2930#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2931 iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2932/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2933 * stack. */
2934#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2935 iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2936/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2937#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2938 iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2939
2940/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2941#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2942 iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2943/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2944 * stack. */
2945#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2946 iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2947/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2948 * FPUDS. */
2949#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2950 iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2951/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2952 * FPUDS. Pops stack. */
2953#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2954 iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2955/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2956 * stack twice. */
2957#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2958 iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2959/** Raises a FPU stack underflow exception for an instruction pushing a result
2960 * value onto the stack. Sets FPUIP, FPUCS and FOP. */
2961#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2962 iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2963/** Raises a FPU stack underflow exception for an instruction pushing a result
2964 * value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2965#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2966 iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2967
2968/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2969 * FPUIP, FPUCS and FOP. */
2970#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2971 iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2972/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2973 * FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2974#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2975 iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2976/** Prepares for using the FPU state.
2977 * Ensures that we can use the host FPU in the current context (RC+R0.
2978 * Ensures the guest FPU state in the CPUMCTX is up to date. */
2979#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2980/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2981#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2982/** Actualizes the guest FPU state so it can be accessed and modified. */
2983#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2984
2985/** Stores SSE SIMD result updating MXCSR. */
2986#define IEM_MC_STORE_SSE_RESULT(a_Res, a_iXmmReg) \
2987 do { \
2988 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; \
2989 if (( ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
2990 & (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0) \
2991 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXmmReg)] = (a_Res); \
2992 } while (0)
2993
2994/** Prepares for using the SSE state.
2995 * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2996 * Ensures the guest SSE state in the CPUMCTX is up to date. */
2997#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2998/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2999#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3000/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3001#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3002
3003/** Prepares for using the AVX state.
3004 * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3005 * Ensures the guest AVX state in the CPUMCTX is up to date.
3006 * @note This will include the AVX512 state too when support for it is added
3007 * due to the zero extending feature of VEX instruction. */
3008#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3009/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3010#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3011/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3012#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3013
3014/**
3015 * Calls a MMX assembly implementation taking two visible arguments.
3016 *
3017 * @param a_pfnAImpl Pointer to the assembly MMX routine.
3018 * @param a0 The first extra argument.
3019 * @param a1 The second extra argument.
3020 */
3021#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3022 do { \
3023 IEM_MC_PREPARE_FPU_USAGE(); \
3024 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3025 } while (0)
3026
3027/**
3028 * Calls a MMX assembly implementation taking three visible arguments.
3029 *
3030 * @param a_pfnAImpl Pointer to the assembly MMX routine.
3031 * @param a0 The first extra argument.
3032 * @param a1 The second extra argument.
3033 * @param a2 The third extra argument.
3034 */
3035#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3036 do { \
3037 IEM_MC_PREPARE_FPU_USAGE(); \
3038 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3039 } while (0)
3040
3041
3042/**
3043 * Calls a SSE assembly implementation taking two visible arguments.
3044 *
3045 * @param a_pfnAImpl Pointer to the assembly SSE routine.
3046 * @param a0 The first extra argument.
3047 * @param a1 The second extra argument.
3048 */
3049#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3050 do { \
3051 IEM_MC_PREPARE_SSE_USAGE(); \
3052 pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3053 (a0), (a1)); \
3054 } while (0)
3055
3056/**
3057 * Calls a SSE assembly implementation taking three visible arguments.
3058 *
3059 * @param a_pfnAImpl Pointer to the assembly SSE routine.
3060 * @param a0 The first extra argument.
3061 * @param a1 The second extra argument.
3062 * @param a2 The third extra argument.
3063 */
3064#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3065 do { \
3066 IEM_MC_PREPARE_SSE_USAGE(); \
3067 pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3068 (a0), (a1), (a2)); \
3069 } while (0)
3070
3071
3072/**
3073 * Calls a AVX assembly implementation taking two visible arguments.
3074 *
3075 * There is one implicit zero'th argument, a pointer to the extended state.
3076 *
3077 * @param a_pfnAImpl Pointer to the assembly AVX routine.
3078 * @param a1 The first extra argument.
3079 * @param a2 The second extra argument.
3080 */
3081#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3082 do { \
3083 IEM_MC_PREPARE_AVX_USAGE(); \
3084 pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3085 (a0), (a1)); \
3086 } while (0)
3087
3088/**
3089 * Calls a AVX assembly implementation taking three visible arguments.
3090 *
3091 * There is one implicit zero'th argument, a pointer to the extended state.
3092 *
3093 * @param a_pfnAImpl Pointer to the assembly AVX routine.
3094 * @param a1 The first extra argument.
3095 * @param a2 The second extra argument.
3096 * @param a3 The third extra argument.
3097 */
3098#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3099 do { \
3100 IEM_MC_PREPARE_AVX_USAGE(); \
3101 pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3102 (a0), (a1), (a2)); \
3103 } while (0)
3104
3105/** @note Not for IOPL or IF testing. */
3106#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3107/** @note Not for IOPL or IF testing. */
3108#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3109/** @note Not for IOPL or IF testing. */
3110#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3111/** @note Not for IOPL or IF testing. */
3112#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3113/** @note Not for IOPL or IF testing. */
3114#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3115 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3116 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3117/** @note Not for IOPL or IF testing. */
3118#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3119 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3120 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3121/** @note Not for IOPL or IF testing. */
3122#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3123 if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3124 || !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3125 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3126/** @note Not for IOPL or IF testing. */
3127#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3128 if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3129 && !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3130 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3131#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3132#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3133#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3134#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3135#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3136#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3137/** @note Not for IOPL or IF testing. */
3138#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3139 if ( pVCpu->cpum.GstCtx.cx != 1 \
3140 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3141/** @note Not for IOPL or IF testing. */
3142#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3143 if ( pVCpu->cpum.GstCtx.ecx != 1 \
3144 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3145/** @note Not for IOPL or IF testing. */
3146#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3147 if ( pVCpu->cpum.GstCtx.rcx != 1 \
3148 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3149/** @note Not for IOPL or IF testing. */
3150#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3151 if ( pVCpu->cpum.GstCtx.cx != 1 \
3152 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3153/** @note Not for IOPL or IF testing. */
3154#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3155 if ( pVCpu->cpum.GstCtx.ecx != 1 \
3156 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3157/** @note Not for IOPL or IF testing. */
3158#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3159 if ( pVCpu->cpum.GstCtx.rcx != 1 \
3160 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3161#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3162#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3163
3164#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3165 do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3166#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3167 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3168#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3169 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3170#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3171 if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3172#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3173 if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3174#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3175 if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3176#define IEM_MC_IF_FCW_IM() \
3177 if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3178#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
3179 if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3180 & (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
3181
3182#define IEM_MC_ELSE() } else {
3183#define IEM_MC_ENDIF() } do {} while (0)
3184
3185
3186/** Recompiler debugging: Flush guest register shadow copies. */
3187#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3188
3189/** @} */
3190
3191#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3192
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette