VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 102876

Last change on this file since 102876 was 102876, checked in by vboxsync, 13 months ago

VMM/IEM: Call different threaded functions for each branch in a conditional jump (jcc, loop, loopcc) so we can quit immediately when taking a different branch from what we did during compilation. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 156.9 KB
Line 
1/* $Id: IEMMc.h 102876 2024-01-15 14:26:27Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - IEM_MC_XXX.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29#define VMM_INCLUDED_SRC_include_IEMMc_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @name "Microcode" macros.
36 *
37 * The idea is that we should be able to use the same code to interpret
38 * instructions as well as recompiler instructions. Thus this obfuscation.
39 *
40 * @{
41 */
42
43#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) {
44#define IEM_MC_END() }
45
46/** Internal macro. */
47#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48 do \
49 { \
50 VBOXSTRICTRC rcStrict2 = a_Expr; \
51 if (rcStrict2 == VINF_SUCCESS) \
52 { /* likely */ } \
53 else \
54 return rcStrict2; \
55 } while (0)
56
57
58/** Dummy MC that prevents native recompilation. */
59#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61/** Advances RIP, finishes the instruction and returns.
62 * This may include raising debug exceptions and such. */
63#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66 return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68 * @note only usable in 16-bit op size mode. */
69#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70 return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73 return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76 return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79 return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82 return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
85#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
86 do { \
87 if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)))) \
88 { /* probable */ } \
89 else return iemRaiseDeviceNotAvailable(pVCpu); \
90 } while (0)
91#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
92 do { \
93 if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)))) \
94 { /* probable */ } \
95 else return iemRaiseDeviceNotAvailable(pVCpu); \
96 } while (0)
97#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
98 do { \
99 if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
100 { /* probable */ } \
101 else return iemRaiseMathFault(pVCpu); \
102 } while (0)
103#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
104 do { \
105 /* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
106 be reduced to a single compare branch in the more probably code path. */ \
107 if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) \
108 | (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
109 | (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
110 == (XSAVE_C_YMM | XSAVE_C_SSE | X86_CR4_OSXSAVE))) \
111 { /* probable */ } \
112 else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
113 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
114 return iemRaiseUndefinedOpcode(pVCpu); \
115 else \
116 return iemRaiseDeviceNotAvailable(pVCpu); \
117 } while (0)
118AssertCompile(!((XSAVE_C_YMM | XSAVE_C_SSE) & X86_CR4_OSXSAVE));
119AssertCompile(!((XSAVE_C_YMM | XSAVE_C_SSE) & X86_CR0_TS));
120AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
121#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
122 do { \
123 /* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
124 single compare branch in the more probable code path. */ \
125 if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
126 | (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
127 == X86_CR4_OSFXSR)) \
128 { /* likely */ } \
129 else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
130 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
131 return iemRaiseUndefinedOpcode(pVCpu); \
132 else \
133 return iemRaiseDeviceNotAvailable(pVCpu); \
134 } while (0)
135AssertCompile(!((X86_CR0_EM | X86_CR0_TS) & X86_CR4_OSFXSR));
136#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
137 do { \
138 /* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
139 single compare branch in the more probable code path. */ \
140 if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
141 | (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
142 { /* probable */ } \
143 else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
144 return iemRaiseUndefinedOpcode(pVCpu); \
145 else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
146 return iemRaiseDeviceNotAvailable(pVCpu); \
147 else \
148 return iemRaiseMathFault(pVCpu); \
149 } while (0)
150AssertCompile(!((X86_CR0_EM | X86_CR0_TS) & X86_FSW_ES));
151/** @todo recomp: this one is slightly problematic as the recompiler doesn't
152 * count the CPL into the TB key. However it is safe enough for now, as
153 * it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
154 * emitted for it. */
155#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
156 do { \
157 if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
158 else return iemRaiseGeneralProtectionFault0(pVCpu); \
159 } while (0)
160#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
161 do { \
162 if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
163 else return iemRaiseGeneralProtectionFault0(pVCpu); \
164 } while (0)
165#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
166 do { \
167 if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) | IEM_GET_CPU_MODE(pVCpu)) \
168 == (X86_CR4_FSGSBASE | IEMMODE_64BIT))) \
169 { /* probable */ } \
170 else return iemRaiseUndefinedOpcode(pVCpu); \
171 } while (0)
172AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
173#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
174 do { \
175 if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
176 else return iemRaiseGeneralProtectionFault0(pVCpu); \
177 } while (0)
178#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
179 do { \
180 if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
181 & (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
182 { /* probable */ } \
183 else \
184 { \
185 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
186 return iemRaiseSimdFpException(pVCpu); \
187 return iemRaiseUndefinedOpcode(pVCpu); \
188 } \
189 } while (0)
190#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
191 do { \
192 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
193 return iemRaiseSimdFpException(pVCpu); \
194 return iemRaiseUndefinedOpcode(pVCpu); \
195 } while (0)
196
197
198#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
199#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
200#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
201/** @deprecated Use IEM_MC_ARG_LOCAL_REF instead! */
202#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
203#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
204#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
205#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
206/** @note IEMAllInstPython.py duplicates the expansion. */
207#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
208 uint32_t a_Name; \
209 uint32_t *a_pName = &a_Name
210#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
211 do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
212
213/** ASSUMES the source variable not used after this statement. */
214#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
215
216#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
217#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
218#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
219#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
220#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
221#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
222#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
223#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
224#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
225#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
226#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
227#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
228#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
229#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
230#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
231#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
232#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
233#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
234 (a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
235 (a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
236 } while(0)
237#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
238 (a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
239 (a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
240 } while(0)
241#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
242 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
243 (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
244 } while (0)
245#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
246 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
247 (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
248 } while (0)
249#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
250 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
251 (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
252 } while (0)
253/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
254#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
255 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
256 (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
257 } while (0)
258#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
259 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
260 (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
261 } while (0)
262/** @note Not for IOPL or IF testing or modification. */
263#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
264#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
265#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
266#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
267
268#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
269#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
270#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
271#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
272#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
273#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
274#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
275#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
276#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
277#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
278 *iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
279 *iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
280 } while(0)
281#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
282 *iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
283 *iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
284 } while(0)
285#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
286
287/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
288#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
289 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
290 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
291 } while (0)
292#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
293 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
294 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
295 } while (0)
296#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
297 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
298
299
300#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
301#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
302#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
303#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
304/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
305 * Use IEM_MC_CLEAR_HIGH_GREG_U64! */
306#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
307#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
308#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
309#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
310#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
311#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
312#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
313#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
314/** @note Not for IOPL or IF testing or modification.
315 * @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
316#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
317#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
318
319#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
320#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
321 do { \
322 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
323 *pu32Reg += (a_u32Value); \
324 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
325 } while (0)
326#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
327
328#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
329#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
330 do { \
331 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
332 *pu32Reg -= (a_u8Const); \
333 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
334 } while (0)
335#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
336#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
337
338#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
339#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
340#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
341#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
342#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
343#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
344#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
345
346#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
347#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
348#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
349#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
350
351#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
352#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
353#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
354
355#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) |= (a_u8Mask); } while (0)
356#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) |= (a_u16Mask); } while (0)
357#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
358
359#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
360#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
361#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
362
363#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
364
365#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
366#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
367#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
368
369#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
370
371#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
372
373#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
374#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
375#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
376 do { \
377 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
378 *pu32Reg &= (a_u32Value); \
379 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
380 } while (0)
381#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
382
383#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
384#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
385#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
386 do { \
387 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
388 *pu32Reg |= (a_u32Value); \
389 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
390 } while (0)
391#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
392
393#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
394#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
395#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
396
397/** @note Not for IOPL or IF modification. */
398#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
399/** @note Not for IOPL or IF modification. */
400#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
401/** @note Not for IOPL or IF modification. */
402#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
403
404#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
405
406/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
407#define IEM_MC_FPU_TO_MMX_MODE() do { \
408 iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
409 pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
410 pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
411 } while (0)
412
413/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
414#define IEM_MC_FPU_FROM_MMX_MODE() do { \
415 iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
416 pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
417 pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
418 } while (0)
419
420#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
421 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
422#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
423 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
424#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
425 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
426 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
427 } while (0)
428#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
429 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
430 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
431 } while (0)
432#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
433 (a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
434#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
435 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
436#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
437 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
438#define IEM_MC_MODIFIED_MREG(a_iMReg) \
439 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
440#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
441 do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
442
443#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
444 do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
445 if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
446 if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
447 if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
448 } while (0)
449#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
450 do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
451 (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
452 } while (0)
453#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
454 do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
455 (a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
456 } while (0)
457#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
458 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
459#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
460 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
461#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
462 do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
463#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
464 do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iByte)]; } while (0)
465#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
466 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
467 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
468 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
469 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
470 } while (0)
471#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
472 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
473 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
474 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
475 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
476 } while (0)
477#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
478 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
479 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
480 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
481 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
482 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
483 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
484 } while (0)
485#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
486 do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
487 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
488 (a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
489 (a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
490 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
491 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
492 } while (0)
493#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
494 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
495 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
496 } while (0)
497#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
498 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
499 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
500 } while (0)
501#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
502 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
503#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
504 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
505#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
506 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
507#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
508 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
509#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
510 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
511#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
512 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
513
514#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
515 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
516 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
517 } while (0)
518
519#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
520 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
521#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
522 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
523#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
524 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
525#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
526 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
527 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
528 } while (0)
529#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
530 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
531
532#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
533 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
534 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
535 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
536 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
537 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
538 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
539 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
540 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
541 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
542 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
543 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
544 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
545 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
546 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
547 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
548 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
549 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
550 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
551 } while (0)
552#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
553 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
554 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
555 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
556 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
557 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
558 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
559 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
560 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
561 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
562 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
563 } while (0)
564#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
565 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
566 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
567 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
568 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
569 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
570 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
571 } while (0)
572#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
573 do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
574 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
575 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
576 IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
577 } while (0)
578
579#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
580 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
581#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
582 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
583#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
584 (a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
585#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
586 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
587#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
588 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
589#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
590 (a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
591#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
592 (a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
593#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
594 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
595 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
596 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
597 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
598 } while (0)
599
600#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
601 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
602 (a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
603 } while (0)
604#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
605 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
606 (a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
607 } while (0)
608#define IEM_MC_FETCH_YREG_2ND_U64(a_u64Dst, a_iYRegSrc) \
609 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
610 (a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
611 } while (0)
612#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
613 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
614 (a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
615 (a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
616 } while (0)
617#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
618 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
619 (a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
620 (a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
621 (a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
622 (a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
623 } while (0)
624
625#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
626 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
627 if ((a_iDQword) == 0) \
628 { \
629 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
630 pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
631 } \
632 else \
633 { \
634 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
635 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
636 } \
637 } while (0)
638
639#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
640#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
641 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
642 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
643 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
644 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
645 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
646 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
647 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
648 } while (0)
649#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
650 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
651 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
652 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
653 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
654 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
655 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
656 } while (0)
657#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
658 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
659 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
660 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
661 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
662 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
663 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
664 } while (0)
665#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
666 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
667 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
668 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
669 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
670 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
671 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
672 } while (0)
673
674#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
675 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
676 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
677 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
678 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
679 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
680 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
681 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
682 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
683 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
684 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
685 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
686 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
687 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
688 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
689 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
690 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
691 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
692 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
693 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
694 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
695 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
696 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
697 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
698 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
699 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
700 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
701 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
702 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
703 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
704 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
705 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
706 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
707 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
708 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
709 } while (0)
710#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
711 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
712 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
713 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
714 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
715 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
716 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
717 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
718 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
719 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
720 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
721 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
722 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
723 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
724 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
725 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
726 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
727 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
728 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
729 } while (0)
730#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
731 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
732 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
733 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
734 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
735 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
736 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
737 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
738 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
739 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
740 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
741 } while (0)
742#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
743 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
744 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
745 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
746 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
747 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
748 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
749 } while (0)
750#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
751 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
752 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
753 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
754 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
755 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
756 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
757 } while (0)
758
759#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
760 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
761#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
762 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
763#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
764 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
765#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
766 do { uintptr_t const iYRegTmp = (a_iYReg); \
767 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
768 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
769 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
770 } while (0)
771
772#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
773 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
774 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
775 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
776 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
777 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
778 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
779 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
780 } while (0)
781#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
782 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
783 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
784 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
785 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
786 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
787 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
788 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
789 } while (0)
790#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
791 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
792 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
793 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
794 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
795 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
796 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
797 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
798 } while (0)
799
800#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
801 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
802 uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
803 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
804 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
805 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
806 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
807 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
808 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
809 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
810 } while (0)
811#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
812 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
813 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
814 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
815 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
816 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
817 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
818 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
819 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
820 } while (0)
821#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
822 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
823 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
824 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
825 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
826 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
827 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
828 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
829 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
830 } while (0)
831#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
832 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
833 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
834 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
835 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
836 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
837 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
838 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
839 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
840 } while (0)
841#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
842 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
843 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
844 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
845 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
846 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
847 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
848 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
849 } while (0)
850#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
851 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
852 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
853 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
854 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
855 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
856 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
857 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
858 } while (0)
859
860#ifndef IEM_WITH_SETJMP
861# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
862 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
863# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
864 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
865# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
866 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
867#else
868# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
869 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
870# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
871 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
872# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
873 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
874
875# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
876 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
877# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
878 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
879# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
880 ((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
881#endif
882
883#ifndef IEM_WITH_SETJMP
884# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
885 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
886# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
887 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
888# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
889 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
890#else
891# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
892 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
893# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
894 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
895# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
896 ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
897
898# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
899 ((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
900# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
901 ((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
902# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
903 ((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
904#endif
905
906#ifndef IEM_WITH_SETJMP
907# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
908 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
909# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
910 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
911# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
912 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
913#else
914# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
915 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
916# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
917 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
918# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
919 ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
920
921# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
922 ((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
923# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
924 ((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
925# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
926 ((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
927#endif
928
929#ifndef IEM_WITH_SETJMP
930# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
931 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
932# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
933 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
934# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
935 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
936# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
937 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
938#else
939# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
940 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
941# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
942 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
943# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
944 ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
945# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
946 ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
947
948# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
949 ((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
950# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
951 ((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
952# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
953 ((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
954# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
955 ((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
956#endif
957
958#ifndef IEM_WITH_SETJMP
959# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
960 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
961# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
962 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
963# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
964 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
965# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
966 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
967#else
968# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
969 ((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
970# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
971 ((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
972# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
973 iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
974# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
975 iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
976
977# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
978 ((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
979# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
980 ((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
981# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
982 iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
983# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
984 iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
985#endif
986
987#ifndef IEM_WITH_SETJMP
988# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
989 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
990# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
991 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
992# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
993 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
994
995# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
996 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
997# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
998 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
999# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1000 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1001# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1002 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
1003# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1004 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
1005
1006# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1007 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1008 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1009 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1010 } while (0)
1011
1012# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1013 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1014 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1015 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1016 } while (0)
1017
1018# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1019 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1020 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1021 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1022 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1023 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1024 } while (0)
1025
1026# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1027 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1028 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1029 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1030 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1031 } while (0)
1032
1033# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1034 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1035 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1036 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1037 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1038 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1039 } while (0)
1040# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1041 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1042 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1043 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1044 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1045 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1046 } while (0)
1047
1048#else
1049# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1050 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1051# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1052 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1053# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1054 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1055
1056# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1057 iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1058# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1059 iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1060# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1061 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1062# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1063 (a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1064# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1065 (a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1066
1067# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1068 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1069# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1070 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1071# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1072 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
1073
1074# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1075 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1076# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1077 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1078# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1079 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
1080# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
1081 (a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
1082# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
1083 (a_XmmDst).au64[(a_iQWord)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem))
1084
1085# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1086 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1087 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1088 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1089 } while (0)
1090# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1091 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1092 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1093 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1094 } while (0)
1095
1096# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1097 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1098 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1099 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1100 } while (0)
1101# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1102 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, UINT8_MAX, (a_GCPtrMem2)); \
1103 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1104 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1105 } while (0)
1106
1107# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1108 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1109 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1110 (a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1111 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1112 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1113 } while (0)
1114# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1115 (a_Dst).uSrc2.uXmm.au64[0] = 0; \
1116 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1117 (a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1118 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1119 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1120 } while (0)
1121
1122# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1123 (a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1124 (a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1125 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1126 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1127 } while (0)
1128# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1129 (a_Dst).uSrc2.uXmm.au64[1] = 0; \
1130 (a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1131 (a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1132 (a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1133 } while (0)
1134
1135
1136# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1137 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1138 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1139 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1140 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1141 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1142 } while (0)
1143# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1144 iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1145 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1146 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1147 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1148 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1149 } while (0)
1150
1151# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1152 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1153 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1154 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1155 (a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1156 (a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1157 } while (0)
1158# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1159 iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1160 (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1161 (a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1162 (a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1163 (a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1164 } while (0)
1165
1166#endif
1167
1168#ifndef IEM_WITH_SETJMP
1169# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1170 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1171# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1172 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1173# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1174 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1175
1176# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1177 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1178# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1179 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1180# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1181 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1182#else
1183# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1184 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1185# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1186 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1187# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1188 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1189
1190# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1191 iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1192# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1193 iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1194# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1195 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1196
1197# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1198 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1199# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1200 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1201# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1202 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1203
1204# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1205 iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1206# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1207 iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1208# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1209 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1210#endif
1211
1212
1213
1214#ifndef IEM_WITH_SETJMP
1215# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1216 do { \
1217 uint8_t u8Tmp; \
1218 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1219 (a_u16Dst) = u8Tmp; \
1220 } while (0)
1221# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1222 do { \
1223 uint8_t u8Tmp; \
1224 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1225 (a_u32Dst) = u8Tmp; \
1226 } while (0)
1227# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1228 do { \
1229 uint8_t u8Tmp; \
1230 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1231 (a_u64Dst) = u8Tmp; \
1232 } while (0)
1233# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1234 do { \
1235 uint16_t u16Tmp; \
1236 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1237 (a_u32Dst) = u16Tmp; \
1238 } while (0)
1239# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1240 do { \
1241 uint16_t u16Tmp; \
1242 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1243 (a_u64Dst) = u16Tmp; \
1244 } while (0)
1245# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1246 do { \
1247 uint32_t u32Tmp; \
1248 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1249 (a_u64Dst) = u32Tmp; \
1250 } while (0)
1251#else /* IEM_WITH_SETJMP */
1252# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1253 ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1254# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1255 ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1256# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1257 ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1258# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1259 ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1260# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1261 ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1262# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1263 ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1264
1265# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1266 ((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1267# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1268 ((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1269# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1270 ((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1271# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1272 ((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1273# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1274 ((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1275# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1276 ((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1277#endif /* IEM_WITH_SETJMP */
1278
1279#ifndef IEM_WITH_SETJMP
1280# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1281 do { \
1282 uint8_t u8Tmp; \
1283 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1284 (a_u16Dst) = (int8_t)u8Tmp; \
1285 } while (0)
1286# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1287 do { \
1288 uint8_t u8Tmp; \
1289 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1290 (a_u32Dst) = (int8_t)u8Tmp; \
1291 } while (0)
1292# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1293 do { \
1294 uint8_t u8Tmp; \
1295 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1296 (a_u64Dst) = (int8_t)u8Tmp; \
1297 } while (0)
1298# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1299 do { \
1300 uint16_t u16Tmp; \
1301 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1302 (a_u32Dst) = (int16_t)u16Tmp; \
1303 } while (0)
1304# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1305 do { \
1306 uint16_t u16Tmp; \
1307 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1308 (a_u64Dst) = (int16_t)u16Tmp; \
1309 } while (0)
1310# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1311 do { \
1312 uint32_t u32Tmp; \
1313 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1314 (a_u64Dst) = (int32_t)u32Tmp; \
1315 } while (0)
1316#else /* IEM_WITH_SETJMP */
1317# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1318 ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1319# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1320 ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1321# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1322 ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1323# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1324 ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1325# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1326 ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1327# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1328 ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1329
1330# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1331 ((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1332# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1333 ((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1334# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1335 ((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1336# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1337 ((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1338# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1339 ((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1340# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1341 ((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1342#endif /* IEM_WITH_SETJMP */
1343
1344#ifndef IEM_WITH_SETJMP
1345# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1346 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1347# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1348 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1349# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1350 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1351# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1352 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1353#else
1354# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1355 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1356# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1357 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1358# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1359 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1360# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1361 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1362
1363# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1364 iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1365# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1366 iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1367# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1368 iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1369# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1370 iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1371#endif
1372
1373#ifndef IEM_WITH_SETJMP
1374# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1375 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1376# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1377 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1378# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1379 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1380# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1381 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1382#else
1383# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1384 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1385# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1386 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1387# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1388 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1389# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1390 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1391
1392# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1393 iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1394# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1395 iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1396# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1397 iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1398# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1399 iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1400#endif
1401
1402#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1403#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1404#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1405#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1406#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1407#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1408#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1409 do { \
1410 (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1411 (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1412 } while (0)
1413#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1414 do { \
1415 (a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1416 (a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1417 } while (0)
1418
1419#ifndef IEM_WITH_SETJMP
1420# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1421 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1422# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1423 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1424#else
1425# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1426 iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1427# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1428 iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1429
1430# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1431 iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1432# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1433 iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1434#endif
1435
1436#ifndef IEM_WITH_SETJMP
1437# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1438 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1439# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1440 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1441#else
1442# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1443 iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1444# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1445 iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1446
1447# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1448 iemMemStoreDataU256Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1449# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1450 iemMemStoreDataU256AlignedAvxJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1451#endif
1452
1453/* Regular stack push and pop: */
1454#ifndef IEM_WITH_SETJMP
1455# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1456# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1457# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1458# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1459
1460# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1461# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1462# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1463#else
1464# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1465# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1466# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1467# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1468
1469# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1470# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1471# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1472#endif
1473
1474/* 32-bit flat stack push and pop: */
1475#ifndef IEM_WITH_SETJMP
1476# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1477# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1478# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1479
1480# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1481# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1482#else
1483# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1484# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1485# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1486
1487# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1488# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1489#endif
1490
1491/* 64-bit flat stack push and pop: */
1492#ifndef IEM_WITH_SETJMP
1493# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1494# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1495
1496# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1497# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1498#else
1499# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1500# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1501
1502# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1503# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1504#endif
1505
1506
1507/* 8-bit */
1508
1509/**
1510 * Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1511 *
1512 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1513 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1514 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1515 * @param[in] a_GCPtrMem The memory address.
1516 * @remarks Will return/long jump on errors.
1517 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1518 */
1519#ifndef IEM_WITH_SETJMP
1520# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1521 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1522 (a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1523#else
1524# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1525 (a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1526#endif
1527
1528/**
1529 * Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1530 *
1531 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1532 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1533 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1534 * @param[in] a_GCPtrMem The memory address.
1535 * @remarks Will return/long jump on errors.
1536 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1537 */
1538#ifndef IEM_WITH_SETJMP
1539# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1540 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1541 (a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1542#else
1543# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1544 (a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1545#endif
1546
1547/**
1548 * Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1549 *
1550 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1551 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1552 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1553 * @param[in] a_GCPtrMem The memory address.
1554 * @remarks Will return/long jump on errors.
1555 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1556 */
1557#ifndef IEM_WITH_SETJMP
1558# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1559 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1560 (a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1561#else
1562# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1563 (a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1564#endif
1565
1566/**
1567 * Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1568 * address variant.
1569 *
1570 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1571 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1572 * @param[in] a_GCPtrMem The memory address.
1573 * @remarks Will return/long jump on errors.
1574 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1575 */
1576#ifndef IEM_WITH_SETJMP
1577# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1578 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1579 (a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1580#else
1581# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1582 (a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1583#endif
1584
1585/**
1586 * Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1587 * address variant.
1588 *
1589 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1590 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1591 * @param[in] a_GCPtrMem The memory address.
1592 * @remarks Will return/long jump on errors.
1593 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1594 */
1595#ifndef IEM_WITH_SETJMP
1596# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1597 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1598 (a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1599#else
1600# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1601 (a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1602#endif
1603
1604/**
1605 * Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1606 * address variant.
1607 *
1608 * @param[out] a_pu8Mem Where to return the pointer to the mapping.
1609 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1610 * @param[in] a_GCPtrMem The memory address.
1611 * @remarks Will return/long jump on errors.
1612 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1613 */
1614#ifndef IEM_WITH_SETJMP
1615# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1616 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1617 (a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1618#else
1619# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1620 (a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1621#endif
1622
1623
1624/* 16-bit */
1625
1626/**
1627 * Maps guest memory for word read+write direct (or bounce) buffer acccess.
1628 *
1629 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1630 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1631 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1632 * @param[in] a_GCPtrMem The memory address.
1633 * @remarks Will return/long jump on errors.
1634 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1635 */
1636#ifndef IEM_WITH_SETJMP
1637# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1638 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1639 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1640#else
1641# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1642 (a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1643#endif
1644
1645/**
1646 * Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1647 *
1648 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1649 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1650 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1651 * @param[in] a_GCPtrMem The memory address.
1652 * @remarks Will return/long jump on errors.
1653 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1654 */
1655#ifndef IEM_WITH_SETJMP
1656# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1657 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1658 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1659#else
1660# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1661 (a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1662#endif
1663
1664/**
1665 * Maps guest memory for word readonly direct (or bounce) buffer acccess.
1666 *
1667 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1668 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1669 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1670 * @param[in] a_GCPtrMem The memory address.
1671 * @remarks Will return/long jump on errors.
1672 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1673 */
1674#ifndef IEM_WITH_SETJMP
1675# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1676 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1677 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1678#else
1679# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1680 (a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1681#endif
1682
1683/**
1684 * Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1685 * address variant.
1686 *
1687 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1688 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1689 * @param[in] a_GCPtrMem The memory address.
1690 * @remarks Will return/long jump on errors.
1691 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1692 */
1693#ifndef IEM_WITH_SETJMP
1694# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1695 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1696 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1697#else
1698# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1699 (a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1700#endif
1701
1702/**
1703 * Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1704 * address variant.
1705 *
1706 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1707 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1708 * @param[in] a_GCPtrMem The memory address.
1709 * @remarks Will return/long jump on errors.
1710 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1711 */
1712#ifndef IEM_WITH_SETJMP
1713# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1714 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1715 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1716#else
1717# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1718 (a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1719#endif
1720
1721/**
1722 * Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1723 * address variant.
1724 *
1725 * @param[out] a_pu16Mem Where to return the pointer to the mapping.
1726 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1727 * @param[in] a_GCPtrMem The memory address.
1728 * @remarks Will return/long jump on errors.
1729 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1730 */
1731#ifndef IEM_WITH_SETJMP
1732# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1733 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1734 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1735#else
1736# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1737 (a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1738#endif
1739
1740/** int16_t alias. */
1741#ifndef IEM_WITH_SETJMP
1742# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1743 IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1744#else
1745# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1746 (a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1747#endif
1748
1749/** Flat int16_t alias. */
1750#ifndef IEM_WITH_SETJMP
1751# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1752 IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1753#else
1754# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1755 (a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1756#endif
1757
1758
1759/* 32-bit */
1760
1761/**
1762 * Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1763 *
1764 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1765 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1766 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1767 * @param[in] a_GCPtrMem The memory address.
1768 * @remarks Will return/long jump on errors.
1769 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1770 */
1771#ifndef IEM_WITH_SETJMP
1772# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1773 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1774 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
1775#else
1776# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1777 (a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1778#endif
1779
1780/**
1781 * Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1782 *
1783 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1784 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1785 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1786 * @param[in] a_GCPtrMem The memory address.
1787 * @remarks Will return/long jump on errors.
1788 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1789 */
1790#ifndef IEM_WITH_SETJMP
1791# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1792 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1793 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
1794#else
1795# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1796 (a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1797#endif
1798
1799/**
1800 * Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1801 *
1802 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1803 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1804 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1805 * @param[in] a_GCPtrMem The memory address.
1806 * @remarks Will return/long jump on errors.
1807 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1808 */
1809#ifndef IEM_WITH_SETJMP
1810# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1811 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1812 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
1813#else
1814# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1815 (a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1816#endif
1817
1818/**
1819 * Maps guest memory for dword read+write direct (or bounce) buffer acccess,
1820 * flat address variant.
1821 *
1822 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1823 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1824 * @param[in] a_GCPtrMem The memory address.
1825 * @remarks Will return/long jump on errors.
1826 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1827 */
1828#ifndef IEM_WITH_SETJMP
1829# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1830 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1831 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
1832#else
1833# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1834 (a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1835#endif
1836
1837/**
1838 * Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
1839 * address variant.
1840 *
1841 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1842 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1843 * @param[in] a_GCPtrMem The memory address.
1844 * @remarks Will return/long jump on errors.
1845 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1846 */
1847#ifndef IEM_WITH_SETJMP
1848# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1849 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1850 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
1851#else
1852# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1853 (a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1854#endif
1855
1856/**
1857 * Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
1858 * address variant.
1859 *
1860 * @param[out] a_pu32Mem Where to return the pointer to the mapping.
1861 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1862 * @param[in] a_GCPtrMem The memory address.
1863 * @remarks Will return/long jump on errors.
1864 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1865 */
1866#ifndef IEM_WITH_SETJMP
1867# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1868 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1869 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
1870#else
1871# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1872 (a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1873#endif
1874
1875/** int32_t alias. */
1876#ifndef IEM_WITH_SETJMP
1877# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1878 IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1879#else
1880# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1881 (a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1882#endif
1883
1884/** Flat int32_t alias. */
1885#ifndef IEM_WITH_SETJMP
1886# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
1887 IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
1888#else
1889# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
1890 (a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1891#endif
1892
1893/** RTFLOAT32U alias. */
1894#ifndef IEM_WITH_SETJMP
1895# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1896 IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1897#else
1898# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1899 (a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1900#endif
1901
1902/** Flat RTFLOAT32U alias. */
1903#ifndef IEM_WITH_SETJMP
1904# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
1905 IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
1906#else
1907# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
1908 (a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1909#endif
1910
1911
1912/* 64-bit */
1913
1914/**
1915 * Maps guest memory for qword read+write direct (or bounce) buffer acccess.
1916 *
1917 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
1918 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1919 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1920 * @param[in] a_GCPtrMem The memory address.
1921 * @remarks Will return/long jump on errors.
1922 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1923 */
1924#ifndef IEM_WITH_SETJMP
1925# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1926 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
1927 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
1928#else
1929# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1930 (a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1931#endif
1932
1933/**
1934 * Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
1935 *
1936 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
1937 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1938 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1939 * @param[in] a_GCPtrMem The memory address.
1940 * @remarks Will return/long jump on errors.
1941 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1942 */
1943#ifndef IEM_WITH_SETJMP
1944# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1945 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
1946 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
1947#else
1948# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1949 (a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1950#endif
1951
1952/**
1953 * Maps guest memory for qword readonly direct (or bounce) buffer acccess.
1954 *
1955 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
1956 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1957 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1958 * @param[in] a_GCPtrMem The memory address.
1959 * @remarks Will return/long jump on errors.
1960 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1961 */
1962#ifndef IEM_WITH_SETJMP
1963# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1964 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
1965 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
1966#else
1967# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1968 (a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1969#endif
1970
1971/**
1972 * Maps guest memory for qword read+write direct (or bounce) buffer acccess,
1973 * flat address variant.
1974 *
1975 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
1976 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1977 * @param[in] a_GCPtrMem The memory address.
1978 * @remarks Will return/long jump on errors.
1979 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1980 */
1981#ifndef IEM_WITH_SETJMP
1982# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1983 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
1984 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
1985#else
1986# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1987 (a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1988#endif
1989
1990/**
1991 * Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
1992 * address variant.
1993 *
1994 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
1995 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1996 * @param[in] a_GCPtrMem The memory address.
1997 * @remarks Will return/long jump on errors.
1998 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1999 */
2000#ifndef IEM_WITH_SETJMP
2001# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2002 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2003 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2004#else
2005# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2006 (a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2007#endif
2008
2009/**
2010 * Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2011 * address variant.
2012 *
2013 * @param[out] a_pu64Mem Where to return the pointer to the mapping.
2014 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2015 * @param[in] a_GCPtrMem The memory address.
2016 * @remarks Will return/long jump on errors.
2017 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2018 */
2019#ifndef IEM_WITH_SETJMP
2020# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2021 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2022 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2023#else
2024# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2025 (a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2026#endif
2027
2028/** int64_t alias. */
2029#ifndef IEM_WITH_SETJMP
2030# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2031 IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2032#else
2033# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2034 (a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2035#endif
2036
2037/** Flat int64_t alias. */
2038#ifndef IEM_WITH_SETJMP
2039# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2040 IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2041#else
2042# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2043 (a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2044#endif
2045
2046/** RTFLOAT64U alias. */
2047#ifndef IEM_WITH_SETJMP
2048# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2049 IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2050#else
2051# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2052 (a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2053#endif
2054
2055/** Flat RTFLOAT64U alias. */
2056#ifndef IEM_WITH_SETJMP
2057# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2058 IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2059#else
2060# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2061 (a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2062#endif
2063
2064
2065/* 128-bit */
2066
2067/**
2068 * Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2069 *
2070 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2071 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2072 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2073 * @param[in] a_GCPtrMem The memory address.
2074 * @remarks Will return/long jump on errors.
2075 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2076 */
2077#ifndef IEM_WITH_SETJMP
2078# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2079 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2080 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2081#else
2082# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2083 (a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2084#endif
2085
2086/**
2087 * Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2088 *
2089 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2090 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2091 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2092 * @param[in] a_GCPtrMem The memory address.
2093 * @remarks Will return/long jump on errors.
2094 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2095 */
2096#ifndef IEM_WITH_SETJMP
2097# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2098 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2099 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2100#else
2101# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2102 (a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2103#endif
2104
2105/**
2106 * Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2107 *
2108 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2109 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2110 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2111 * @param[in] a_GCPtrMem The memory address.
2112 * @remarks Will return/long jump on errors.
2113 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2114 */
2115#ifndef IEM_WITH_SETJMP
2116# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2117 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2118 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2119#else
2120# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2121 (a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2122#endif
2123
2124/**
2125 * Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2126 * flat address variant.
2127 *
2128 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2129 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2130 * @param[in] a_GCPtrMem The memory address.
2131 * @remarks Will return/long jump on errors.
2132 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2133 */
2134#ifndef IEM_WITH_SETJMP
2135# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2136 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2137 (a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2138#else
2139# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2140 (a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2141#endif
2142
2143/**
2144 * Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2145 * flat address variant.
2146 *
2147 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2148 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2149 * @param[in] a_GCPtrMem The memory address.
2150 * @remarks Will return/long jump on errors.
2151 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2152 */
2153#ifndef IEM_WITH_SETJMP
2154# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2155 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2156 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2157#else
2158# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2159 (a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2160#endif
2161
2162/**
2163 * Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2164 * address variant.
2165 *
2166 * @param[out] a_pu128Mem Where to return the pointer to the mapping.
2167 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2168 * @param[in] a_GCPtrMem The memory address.
2169 * @remarks Will return/long jump on errors.
2170 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2171 */
2172#ifndef IEM_WITH_SETJMP
2173# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2174 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2175 (a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2176#else
2177# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2178 (a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2179#endif
2180
2181
2182/* misc */
2183
2184/**
2185 * Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2186 *
2187 * @param[out] a_pr80Mem Where to return the pointer to the mapping.
2188 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2189 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2190 * @param[in] a_GCPtrMem The memory address.
2191 * @remarks Will return/long jump on errors.
2192 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2193 */
2194#ifndef IEM_WITH_SETJMP
2195# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2196 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2197 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2198#else
2199# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2200 (a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2201#endif
2202
2203/**
2204 * Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2205 *
2206 * @param[out] a_pr80Mem Where to return the pointer to the mapping.
2207 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2208 * @param[in] a_GCPtrMem The memory address.
2209 * @remarks Will return/long jump on errors.
2210 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2211 */
2212#ifndef IEM_WITH_SETJMP
2213# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2214 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2215 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2216#else
2217# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2218 (a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2219#endif
2220
2221
2222/**
2223 * Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2224 *
2225 * @param[out] a_pd80Mem Where to return the pointer to the mapping.
2226 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2227 * @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2228 * @param[in] a_GCPtrMem The memory address.
2229 * @remarks Will return/long jump on errors.
2230 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2231 */
2232#ifndef IEM_WITH_SETJMP
2233# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2234 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2235 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2236#else
2237# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2238 (a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2239#endif
2240
2241/**
2242 * Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2243 *
2244 * @param[out] a_pd80Mem Where to return the pointer to the mapping.
2245 * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2246 * @param[in] a_GCPtrMem The memory address.
2247 * @remarks Will return/long jump on errors.
2248 * @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2249 */
2250#ifndef IEM_WITH_SETJMP
2251# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2252 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2253 (a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2254#else
2255# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2256 (a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2257#endif
2258
2259
2260
2261/* commit + unmap */
2262
2263/** Commits the memory and unmaps guest memory previously mapped RW.
2264 * @remarks May return.
2265 * @note Implictly frees the a_bMapInfo variable.
2266 */
2267#ifndef IEM_WITH_SETJMP
2268# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2269#else
2270# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2271#endif
2272
2273/** Commits the memory and unmaps guest memory previously mapped W.
2274 * @remarks May return.
2275 * @note Implictly frees the a_bMapInfo variable.
2276 */
2277#ifndef IEM_WITH_SETJMP
2278# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2279#else
2280# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2281#endif
2282
2283/** Commits the memory and unmaps guest memory previously mapped R.
2284 * @remarks May return.
2285 * @note Implictly frees the a_bMapInfo variable.
2286 */
2287#ifndef IEM_WITH_SETJMP
2288# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2289#else
2290# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2291#endif
2292
2293
2294/** Commits the memory and unmaps the guest memory unless the FPU status word
2295 * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2296 * that would cause FLD not to store.
2297 *
2298 * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2299 * store, while \#P will not.
2300 *
2301 * @remarks May in theory return - for now.
2302 * @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2303 */
2304#ifndef IEM_WITH_SETJMP
2305# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2306 if ( !(a_u16FSW & X86_FSW_ES) \
2307 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
2308 & ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2309 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2310 else \
2311 iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2312 } while (0)
2313#else
2314# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2315 if ( !(a_u16FSW & X86_FSW_ES) \
2316 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
2317 & ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2318 iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2319 else \
2320 iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2321 } while (0)
2322#endif
2323
2324/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2325 * @note Implictly frees the a_bMapInfo variable. */
2326#ifndef IEM_WITH_SETJMP
2327# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2328#else
2329# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2330#endif
2331
2332
2333
2334/** Calculate efficient address from R/M. */
2335#ifndef IEM_WITH_SETJMP
2336# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2337 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2338#else
2339# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2340 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2341#endif
2342
2343#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2344#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2345#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2346#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2347#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2348#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
2349#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
2350
2351
2352/** @def IEM_MC_CALL_CIMPL_HLP_RET
2353 * Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2354 */
2355#ifdef VBOX_STRICT
2356#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2357 do { \
2358 uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2359 uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2360 uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2361 uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2362 uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2363 VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2364 if (rcStrictHlp == VINF_SUCCESS) \
2365 { \
2366 AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2367 || ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
2368 && uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2369 || ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2370 && uRipBefore == pVCpu->cpum.GstCtx.rip \
2371 && uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2372 ("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2373 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
2374 if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2375 { /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2376 else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2377 AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS | X86_EFL_RF)) \
2378 == (fEflBefore & ~(X86_EFL_STATUS_BITS | X86_EFL_RF)), \
2379 ("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2380 else \
2381 AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2382 == (fEflBefore & ~(X86_EFL_RF)), \
2383 ("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2384 if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2385 { \
2386 uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2387 AssertMsg( fExecBefore == fExecRecalc \
2388 /* in case ES, DS or SS was external initially (happens alot with HM): */ \
2389 || ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2390 && (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2391 ("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2392 } \
2393 } \
2394 return rcStrictHlp; \
2395 } while (0)
2396#else
2397# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2398#endif
2399
2400/**
2401 * Defers the rest of the instruction emulation to a C implementation routine
2402 * and returns, only taking the standard parameters.
2403 *
2404 * @param a_fFlags IEM_CIMPL_F_XXX.
2405 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2406 * in the native recompiler.
2407 * @param a_pfnCImpl The pointer to the C routine.
2408 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2409 */
2410#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2411 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2412
2413/**
2414 * Defers the rest of instruction emulation to a C implementation routine and
2415 * returns, taking one argument in addition to the standard ones.
2416 *
2417 * @param a_fFlags IEM_CIMPL_F_XXX.
2418 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2419 * in the native recompiler.
2420 * @param a_pfnCImpl The pointer to the C routine.
2421 * @param a0 The argument.
2422 */
2423#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2424 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2425
2426/**
2427 * Defers the rest of the instruction emulation to a C implementation routine
2428 * and returns, taking two arguments in addition to the standard ones.
2429 *
2430 * @param a_fFlags IEM_CIMPL_F_XXX.
2431 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2432 * in the native recompiler.
2433 * @param a_pfnCImpl The pointer to the C routine.
2434 * @param a0 The first extra argument.
2435 * @param a1 The second extra argument.
2436 */
2437#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2438 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2439
2440/**
2441 * Defers the rest of the instruction emulation to a C implementation routine
2442 * and returns, taking three arguments in addition to the standard ones.
2443 *
2444 * @param a_fFlags IEM_CIMPL_F_XXX.
2445 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2446 * in the native recompiler.
2447 * @param a_pfnCImpl The pointer to the C routine.
2448 * @param a0 The first extra argument.
2449 * @param a1 The second extra argument.
2450 * @param a2 The third extra argument.
2451 */
2452#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2453 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2454
2455/**
2456 * Defers the rest of the instruction emulation to a C implementation routine
2457 * and returns, taking four arguments in addition to the standard ones.
2458 *
2459 * @param a_fFlags IEM_CIMPL_F_XXX.
2460 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2461 * in the native recompiler.
2462 * @param a_pfnCImpl The pointer to the C routine.
2463 * @param a0 The first extra argument.
2464 * @param a1 The second extra argument.
2465 * @param a2 The third extra argument.
2466 * @param a3 The fourth extra argument.
2467 */
2468#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2469 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2470
2471/**
2472 * Defers the rest of the instruction emulation to a C implementation routine
2473 * and returns, taking two arguments in addition to the standard ones.
2474 *
2475 * @param a_fFlags IEM_CIMPL_F_XXX.
2476 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2477 * in the native recompiler.
2478 * @param a_pfnCImpl The pointer to the C routine.
2479 * @param a0 The first extra argument.
2480 * @param a1 The second extra argument.
2481 * @param a2 The third extra argument.
2482 * @param a3 The fourth extra argument.
2483 * @param a4 The fifth extra argument.
2484 */
2485#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2486 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2487
2488/**
2489 * Defers the entire instruction emulation to a C implementation routine and
2490 * returns, only taking the standard parameters.
2491 *
2492 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2493 *
2494 * @param a_fFlags IEM_CIMPL_F_XXX.
2495 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2496 * in the native recompiler.
2497 * @param a_pfnCImpl The pointer to the C routine.
2498 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2499 */
2500#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2501 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2502
2503/**
2504 * Defers the entire instruction emulation to a C implementation routine and
2505 * returns, taking one argument in addition to the standard ones.
2506 *
2507 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2508 *
2509 * @param a_fFlags IEM_CIMPL_F_XXX.
2510 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2511 * in the native recompiler.
2512 * @param a_pfnCImpl The pointer to the C routine.
2513 * @param a0 The argument.
2514 */
2515#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2516 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2517
2518/**
2519 * Defers the entire instruction emulation to a C implementation routine and
2520 * returns, taking two arguments in addition to the standard ones.
2521 *
2522 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2523 *
2524 * @param a_fFlags IEM_CIMPL_F_XXX.
2525 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2526 * in the native recompiler.
2527 * @param a_pfnCImpl The pointer to the C routine.
2528 * @param a0 The first extra argument.
2529 * @param a1 The second extra argument.
2530 */
2531#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2532 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2533
2534/**
2535 * Defers the entire instruction emulation to a C implementation routine and
2536 * returns, taking three arguments in addition to the standard ones.
2537 *
2538 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2539 *
2540 * @param a_fFlags IEM_CIMPL_F_XXX.
2541 * @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2542 * in the native recompiler.
2543 * @param a_pfnCImpl The pointer to the C routine.
2544 * @param a0 The first extra argument.
2545 * @param a1 The second extra argument.
2546 * @param a2 The third extra argument.
2547 */
2548#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2549 IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2550
2551
2552/**
2553 * Calls a FPU assembly implementation taking one visible argument.
2554 *
2555 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2556 * @param a0 The first extra argument.
2557 */
2558#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2559 do { \
2560 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2561 } while (0)
2562
2563/**
2564 * Calls a FPU assembly implementation taking two visible arguments.
2565 *
2566 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2567 * @param a0 The first extra argument.
2568 * @param a1 The second extra argument.
2569 */
2570#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2571 do { \
2572 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2573 } while (0)
2574
2575/**
2576 * Calls a FPU assembly implementation taking three visible arguments.
2577 *
2578 * @param a_pfnAImpl Pointer to the assembly FPU routine.
2579 * @param a0 The first extra argument.
2580 * @param a1 The second extra argument.
2581 * @param a2 The third extra argument.
2582 */
2583#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2584 do { \
2585 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2586 } while (0)
2587
2588#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2589 do { \
2590 (a_FpuData).FSW = (a_FSW); \
2591 (a_FpuData).r80Result = *(a_pr80Value); \
2592 } while (0)
2593
2594/** Pushes FPU result onto the stack. */
2595#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2596 iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2597/** Pushes FPU result onto the stack and sets the FPUDP. */
2598#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2599 iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2600
2601/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2602#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2603 iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2604
2605/** Stores FPU result in a stack register. */
2606#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2607 iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2608/** Stores FPU result in a stack register and pops the stack. */
2609#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2610 iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2611/** Stores FPU result in a stack register and sets the FPUDP. */
2612#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2613 iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2614/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2615 * stack. */
2616#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2617 iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2618
2619/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2620#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2621 iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2622/** Free a stack register (for FFREE and FFREEP). */
2623#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2624 iemFpuStackFree(pVCpu, a_iStReg)
2625/** Increment the FPU stack pointer. */
2626#define IEM_MC_FPU_STACK_INC_TOP() \
2627 iemFpuStackIncTop(pVCpu)
2628/** Decrement the FPU stack pointer. */
2629#define IEM_MC_FPU_STACK_DEC_TOP() \
2630 iemFpuStackDecTop(pVCpu)
2631
2632/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2633#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2634 iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2635/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2636#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2637 iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2638/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2639#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2640 iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2641/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2642#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2643 iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2644/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2645 * stack. */
2646#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2647 iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2648/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2649#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2650 iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2651
2652/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2653#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2654 iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2655/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2656 * stack. */
2657#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2658 iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2659/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2660 * FPUDS. */
2661#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2662 iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2663/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2664 * FPUDS. Pops stack. */
2665#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2666 iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2667/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2668 * stack twice. */
2669#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2670 iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2671/** Raises a FPU stack underflow exception for an instruction pushing a result
2672 * value onto the stack. Sets FPUIP, FPUCS and FOP. */
2673#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2674 iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2675/** Raises a FPU stack underflow exception for an instruction pushing a result
2676 * value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2677#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2678 iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2679
2680/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2681 * FPUIP, FPUCS and FOP. */
2682#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2683 iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2684/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2685 * FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2686#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2687 iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2688/** Prepares for using the FPU state.
2689 * Ensures that we can use the host FPU in the current context (RC+R0.
2690 * Ensures the guest FPU state in the CPUMCTX is up to date. */
2691#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2692/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2693#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2694/** Actualizes the guest FPU state so it can be accessed and modified. */
2695#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2696
2697/** Stores SSE SIMD result updating MXCSR. */
2698#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
2699 iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
2700/** Updates MXCSR. */
2701#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
2702 iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
2703
2704/** Prepares for using the SSE state.
2705 * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2706 * Ensures the guest SSE state in the CPUMCTX is up to date. */
2707#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2708/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2709#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
2710/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
2711#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
2712
2713/** Prepares for using the AVX state.
2714 * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
2715 * Ensures the guest AVX state in the CPUMCTX is up to date.
2716 * @note This will include the AVX512 state too when support for it is added
2717 * due to the zero extending feature of VEX instruction. */
2718#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
2719/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2720#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
2721/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
2722#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
2723
2724/**
2725 * Calls a MMX assembly implementation taking two visible arguments.
2726 *
2727 * @param a_pfnAImpl Pointer to the assembly MMX routine.
2728 * @param a0 The first extra argument.
2729 * @param a1 The second extra argument.
2730 */
2731#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
2732 do { \
2733 IEM_MC_PREPARE_FPU_USAGE(); \
2734 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2735 } while (0)
2736
2737/**
2738 * Calls a MMX assembly implementation taking three visible arguments.
2739 *
2740 * @param a_pfnAImpl Pointer to the assembly MMX routine.
2741 * @param a0 The first extra argument.
2742 * @param a1 The second extra argument.
2743 * @param a2 The third extra argument.
2744 */
2745#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2746 do { \
2747 IEM_MC_PREPARE_FPU_USAGE(); \
2748 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2749 } while (0)
2750
2751
2752/**
2753 * Calls a SSE assembly implementation taking two visible arguments.
2754 *
2755 * @param a_pfnAImpl Pointer to the assembly SSE routine.
2756 * @param a0 The first extra argument.
2757 * @param a1 The second extra argument.
2758 */
2759#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
2760 do { \
2761 IEM_MC_PREPARE_SSE_USAGE(); \
2762 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2763 } while (0)
2764
2765/**
2766 * Calls a SSE assembly implementation taking three visible arguments.
2767 *
2768 * @param a_pfnAImpl Pointer to the assembly SSE routine.
2769 * @param a0 The first extra argument.
2770 * @param a1 The second extra argument.
2771 * @param a2 The third extra argument.
2772 */
2773#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2774 do { \
2775 IEM_MC_PREPARE_SSE_USAGE(); \
2776 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2777 } while (0)
2778
2779
2780/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
2781 * IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ...
2782 * @note IEMAllInstPython.py duplicates the expansion. */
2783#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
2784 IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
2785
2786/**
2787 * Calls a AVX assembly implementation taking two visible arguments.
2788 *
2789 * There is one implicit zero'th argument, a pointer to the extended state.
2790 *
2791 * @param a_pfnAImpl Pointer to the assembly AVX routine.
2792 * @param a1 The first extra argument.
2793 * @param a2 The second extra argument.
2794 */
2795#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
2796 do { \
2797 IEM_MC_PREPARE_AVX_USAGE(); \
2798 a_pfnAImpl(pXState, (a1), (a2)); \
2799 } while (0)
2800
2801/**
2802 * Calls a AVX assembly implementation taking three visible arguments.
2803 *
2804 * There is one implicit zero'th argument, a pointer to the extended state.
2805 *
2806 * @param a_pfnAImpl Pointer to the assembly AVX routine.
2807 * @param a1 The first extra argument.
2808 * @param a2 The second extra argument.
2809 * @param a3 The third extra argument.
2810 */
2811#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
2812 do { \
2813 IEM_MC_PREPARE_AVX_USAGE(); \
2814 a_pfnAImpl(pXState, (a1), (a2), (a3)); \
2815 } while (0)
2816
2817/** @note Not for IOPL or IF testing. */
2818#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
2819/** @note Not for IOPL or IF testing. */
2820#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
2821/** @note Not for IOPL or IF testing. */
2822#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
2823/** @note Not for IOPL or IF testing. */
2824#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
2825/** @note Not for IOPL or IF testing. */
2826#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
2827 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2828 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2829/** @note Not for IOPL or IF testing. */
2830#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
2831 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2832 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2833/** @note Not for IOPL or IF testing. */
2834#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
2835 if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2836 || !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2837 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2838/** @note Not for IOPL or IF testing. */
2839#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
2840 if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2841 && !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2842 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2843#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
2844#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
2845#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
2846#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
2847#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
2848#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
2849/** @note Not for IOPL or IF testing. */
2850#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
2851 if ( pVCpu->cpum.GstCtx.cx != 1 \
2852 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2853/** @note Not for IOPL or IF testing. */
2854#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
2855 if ( pVCpu->cpum.GstCtx.ecx != 1 \
2856 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2857/** @note Not for IOPL or IF testing. */
2858#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
2859 if ( pVCpu->cpum.GstCtx.rcx != 1 \
2860 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2861/** @note Not for IOPL or IF testing. */
2862#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
2863 if ( pVCpu->cpum.GstCtx.cx != 1 \
2864 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2865/** @note Not for IOPL or IF testing. */
2866#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
2867 if ( pVCpu->cpum.GstCtx.ecx != 1 \
2868 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2869/** @note Not for IOPL or IF testing. */
2870#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
2871 if ( pVCpu->cpum.GstCtx.rcx != 1 \
2872 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2873#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
2874#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
2875
2876#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
2877 do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
2878#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2879 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2880#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
2881 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
2882#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2883 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2884#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
2885 if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
2886#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
2887 if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
2888#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
2889 if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
2890#define IEM_MC_IF_FCW_IM() \
2891 if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
2892#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
2893 if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
2894 & (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
2895
2896#define IEM_MC_ELSE() } else {
2897#define IEM_MC_ENDIF() } do {} while (0)
2898
2899
2900/** Recompiler debugging: Flush guest register shadow copies. */
2901#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
2902
2903/** @} */
2904
2905#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
2906
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette