IEMMc.h@ 104016

Last change on this file since 104016 was 104016, checked in by vboxsync, 11 months ago
VMM/IEM: IEM_MC_LOCAL_EFLAGS. bugref:10376
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 169.5 KB

Line
1	/* $Id: IEMMc.h 104016 2024-03-23 22:46:33Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
85	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
86	do { \
87	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
88	{ /* probable */ } \
89	else return iemRaiseDeviceNotAvailable(pVCpu); \
90	} while (0)
91	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
92	do { \
93	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
94	{ /* probable */ } \
95	else return iemRaiseDeviceNotAvailable(pVCpu); \
96	} while (0)
97	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
98	do { \
99	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
100	{ /* probable */ } \
101	else return iemRaiseMathFault(pVCpu); \
102	} while (0)
103	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
104	do { \
105	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
106	be reduced to a single compare branch in the more probably code path. */ \
107	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
108	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
109	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
110	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
111	{ /* probable */ } \
112	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
113	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
114	return iemRaiseUndefinedOpcode(pVCpu); \
115	else \
116	return iemRaiseDeviceNotAvailable(pVCpu); \
117	} while (0)
118	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
119	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
120	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
121	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
122	do { \
123	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
124	single compare branch in the more probable code path. */ \
125	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
126	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
127	== X86_CR4_OSFXSR)) \
128	{ /* likely */ } \
129	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
130	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
131	return iemRaiseUndefinedOpcode(pVCpu); \
132	else \
133	return iemRaiseDeviceNotAvailable(pVCpu); \
134	} while (0)
135	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
136	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
137	do { \
138	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
139	single compare branch in the more probable code path. */ \
140	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
141	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
142	{ /* probable */ } \
143	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
144	return iemRaiseUndefinedOpcode(pVCpu); \
145	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
146	return iemRaiseDeviceNotAvailable(pVCpu); \
147	else \
148	return iemRaiseMathFault(pVCpu); \
149	} while (0)
150	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
151	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
152	* count the CPL into the TB key. However it is safe enough for now, as
153	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
154	* emitted for it. */
155	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
156	do { \
157	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
158	else return iemRaiseGeneralProtectionFault0(pVCpu); \
159	} while (0)
160	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
161	do { \
162	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
163	else return iemRaiseGeneralProtectionFault0(pVCpu); \
164	} while (0)
165	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
166	do { \
167	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
168	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
169	{ /* probable */ } \
170	else return iemRaiseUndefinedOpcode(pVCpu); \
171	} while (0)
172	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
173	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
174	do { \
175	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
176	else return iemRaiseGeneralProtectionFault0(pVCpu); \
177	} while (0)
178	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
179	do { \
180	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
181	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
182	{ /* probable */ } \
183	else \
184	{ \
185	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
186	return iemRaiseSimdFpException(pVCpu); \
187	return iemRaiseUndefinedOpcode(pVCpu); \
188	} \
189	} while (0)
190	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
191	do { \
192	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
193	return iemRaiseSimdFpException(pVCpu); \
194	return iemRaiseUndefinedOpcode(pVCpu); \
195	} while (0)
196
197
198	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
199	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
200	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
201	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
202	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
203	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
204	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
205	/** @note IEMAllInstPython.py duplicates the expansion. */
206	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
207	uint32_t a_Name; \
208	uint32_t *a_pName = &a_Name
209	/** @note IEMAllInstPython.py duplicates the expansion. */
210	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
211	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
212	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
213	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
214	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
215	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
216	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
217	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
218	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
219	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
220	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
221	} while (0)
222	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
223	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
224
225	/** ASSUMES the source variable not used after this statement. */
226	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
227
228	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
229	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
230	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
231	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
232	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
233	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
234	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
235	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
236	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
237	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
238	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
239	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
240	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
241	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
242	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
243	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
244	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
245	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
246	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
247	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
248	} while(0)
249	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
250	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
251	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
252	} while(0)
253	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
254	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
255	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
256	} while (0)
257	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
258	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
259	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
260	} while (0)
261	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
262	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
263	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
264	} while (0)
265	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
266	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
267	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
268	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
269	} while (0)
270	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
271	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
272	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
273	} while (0)
274	/** @note Not for IOPL or IF testing or modification. */
275	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
276	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
277	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
278	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
279	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
280
281	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
282	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
283	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
284	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
285	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
286	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
287	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
288	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
289	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
290	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
291	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
292	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
293	} while(0)
294	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
295	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
296	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
297	} while(0)
298	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
299
300	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
301	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
302	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
303	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
304	} while (0)
305	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
306	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
307	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
308	} while (0)
309	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
310	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
311
312
313	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
314	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
315	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
316	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
317	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
318	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
319	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
320	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
321	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
322	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
323	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
324	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
325	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
326	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
327	/** @note Not for IOPL or IF testing or modification.
328	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
329	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
330	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
331	#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
332
333	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
334	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
335	do { \
336	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
337	*pu32Reg += (a_u32Value); \
338	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
339	} while (0)
340	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
341
342	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
343	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
344	do { \
345	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
346	*pu32Reg -= (a_u8Const); \
347	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
348	} while (0)
349	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
350	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
351
352	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
353	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
354	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
355	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
356	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
357	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
358	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
359
360	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
361	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
362	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
363	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
364
365	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
366	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
367	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
368
369	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
370	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
371	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
372
373	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
374	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
375	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
376
377	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
378
379	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
380	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
381	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
382
383	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
384
385	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
386
387	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
388	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
389	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
390	do { \
391	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
392	*pu32Reg &= (a_u32Value); \
393	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
394	} while (0)
395	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
396
397	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
398	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
399	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
400	do { \
401	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
402	*pu32Reg \|= (a_u32Value); \
403	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
404	} while (0)
405	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
406
407	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
408	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
409	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
410
411	/** @note Not for IOPL or IF modification. */
412	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
413	/** @note Not for IOPL or IF modification. */
414	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
415	/** @note Not for IOPL or IF modification. */
416	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
417
418	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
419
420	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
421	#define IEM_MC_FPU_TO_MMX_MODE() do { \
422	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
423	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
424	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
425	} while (0)
426
427	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
428	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
429	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
430	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
431	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
432	} while (0)
433
434	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
435	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
436	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
437	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
438	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
439	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
440	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
441	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
442	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
443	} while (0)
444	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
445	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
446	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
447	} while (0)
448	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
449	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
450	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
451	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
452	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
453	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
454	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
455	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
456	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
457	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
458
459	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
460	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
461	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
462	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
463	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
464	} while (0)
465	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
466	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
467	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
468	} while (0)
469	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
470	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
471	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
472	} while (0)
473	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
474	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
475	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
476	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
477	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
478	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
479	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
480	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
481	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
482	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
483	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
484	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
485	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
486	} while (0)
487	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
488	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
489	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
490	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
491	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
492	} while (0)
493	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
494	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
495	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
496	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
497	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
498	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
499	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
500	} while (0)
501	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
502	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
503	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
504	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
505	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
506	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
507	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
508	} while (0)
509	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
510	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
511	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
512	} while (0)
513	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
514	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
515	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
516	} while (0)
517	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
518	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
519	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
520	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
521	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
522	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
523	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
524	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
525	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
526	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
527	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
528	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
529
530	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
531	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
532	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
533	} while (0)
534
535	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
536	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
537	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
538	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
539	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
540	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
541	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
542	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
543	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
544	} while (0)
545
546	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
547	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
548	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
549	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
550	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
551	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
552	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
553	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
554	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
555	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
556	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
557	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
558	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
559	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
560	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
561	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
562	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
563	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
564	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
565	} while (0)
566	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
567	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
568	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
569	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
570	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
571	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
572	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
573	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
574	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
575	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
576	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
577	} while (0)
578	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
579	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
580	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
581	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
582	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
583	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
584	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
585	} while (0)
586	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
587	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
588	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
590	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
591	} while (0)
592
593	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
594	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
595	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
596	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
597	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
598	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
599	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
600	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
601	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
602	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
603	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
604	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
605	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
606	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
607	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
608	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
609	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
611	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
612	} while (0)
613
614	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
615	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
616	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
617	} while (0)
618	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
619	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
620	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
621	} while (0)
622	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
623	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
624	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
625	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
626	} while (0)
627	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
628	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
629	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
630	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
631	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
632	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
633	} while (0)
634
635	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
636	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
637	if ((a_iDQword) == 0) \
638	{ \
639	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
640	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
641	} \
642	else \
643	{ \
644	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
645	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
646	} \
647	} while (0)
648
649	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
650	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
651	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
652	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
653	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
654	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
655	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
656	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
657	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
658	} while (0)
659	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
660	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
661	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
662	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
663	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
664	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
665	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
666	} while (0)
667	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
668	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
669	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
670	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
671	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
672	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
673	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
674	} while (0)
675	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
676	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
677	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
678	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
679	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
680	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
681	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
682	} while (0)
683
684	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
685	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
686	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
687	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
688	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
689	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
690	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
691	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
692	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
693	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
694	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
695	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
696	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
697	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
698	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
699	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
700	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
701	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
702	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
703	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
704	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
705	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
706	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
707	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
708	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
709	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
710	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
711	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
712	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
713	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
714	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
715	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
716	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
717	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
718	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
719	} while (0)
720	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
721	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
722	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
723	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
724	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
725	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
726	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
727	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
728	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
729	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
730	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
731	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
732	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
733	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
734	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
735	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
736	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
737	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
738	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
739	} while (0)
740	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
741	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
742	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
743	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
744	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
745	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
746	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
747	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
749	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
750	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
751	} while (0)
752	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
753	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
754	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
755	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
756	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
757	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
758	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
759	} while (0)
760	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
761	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
762	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
763	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
764	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
765	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
766	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
767	} while (0)
768
769	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
770	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
771	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
772	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
773	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
774	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
775	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
776	do { uintptr_t const iYRegTmp = (a_iYReg); \
777	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
778	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
779	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
780	} while (0)
781
782	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
783	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
784	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
785	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
787	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
788	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
789	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
790	} while (0)
791	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
792	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
793	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
796	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
797	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
798	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
799	} while (0)
800	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
801	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
802	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
803	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
804	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
807	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
808	} while (0)
809
810	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
811	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
812	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
813	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
814	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
815	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
816	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
817	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
818	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
819	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
820	} while (0)
821	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
822	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
823	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
824	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
825	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
826	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
827	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
828	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
829	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
830	} while (0)
831	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
832	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
833	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
834	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
835	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
836	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
837	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
838	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
839	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
840	} while (0)
841	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
842	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
843	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
844	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
845	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
846	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
847	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
848	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
849	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
850	} while (0)
851	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
852	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
853	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
854	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
855	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
856	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
857	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
858	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
859	} while (0)
860	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
861	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
862	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
863	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
864	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
865	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
866	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
867	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
868	} while (0)
869
870	#ifndef IEM_WITH_SETJMP
871	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
872	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
873	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
874	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
875	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
876	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
877	#else
878	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
879	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
880	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
881	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
882	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
883	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
884
885	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
886	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
887	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
888	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
889	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
890	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
891	#endif
892
893	#ifndef IEM_WITH_SETJMP
894	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
895	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
896	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
897	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
898	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
899	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
900	#else
901	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
902	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
903	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
904	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
905	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
906	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
907
908	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
909	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
910	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
911	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
912	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
913	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
914	#endif
915
916	#ifndef IEM_WITH_SETJMP
917	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
918	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
919	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
920	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
921	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
922	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
923	#else
924	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
925	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
926	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
927	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
928	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
929	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
930
931	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
932	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
933	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
934	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
935	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
936	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
937	#endif
938
939	#ifndef IEM_WITH_SETJMP
940	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
941	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
942	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
943	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
944	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
945	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
946	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
947	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
948	#else
949	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
950	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
951	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
952	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
953	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
954	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
955	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
956	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
957
958	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
959	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
960	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
961	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
962	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
963	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
964	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
965	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
966	#endif
967
968	#ifndef IEM_WITH_SETJMP
969	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
970	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
971	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
972	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
973	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
974	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
975	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
976	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
977	#else
978	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
979	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
980	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
981	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
982	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
983	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
984	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
985	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
986
987	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
988	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
989	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
990	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
991	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
992	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
993	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
994	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
995	#endif
996
997	#ifndef IEM_WITH_SETJMP
998	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
999	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1000	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1001	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1002	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1003	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1004
1005	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1006	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1007	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1008	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1009	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1010	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
1011	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1012	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
1013
1014	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1015	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1016	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1017	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1018	} while (0)
1019
1020	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1021	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1022	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1023	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1024	} while (0)
1025
1026	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1027	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1028	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1029	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1030	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1031	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1032	} while (0)
1033
1034	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1035	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1036	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1037	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1038	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1039	} while (0)
1040
1041	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1042	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1043	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1044	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1045	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1046	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1047	} while (0)
1048	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1049	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1050	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1051	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1052	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1053	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1054	} while (0)
1055
1056	#else
1057	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1058	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1059	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1060	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1061	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1062	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1063
1064	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1065	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1066	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1067	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1068	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1069	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1070	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
1071	(a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1072	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
1073	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
1074
1075	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1076	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1077	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1078	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1079	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1080	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1081
1082	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1083	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1084	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1085	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1086	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1087	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1088	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
1089	(a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
1090	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
1091	(a_XmmDst).au64[(a_iQWord)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem))
1092
1093	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1094	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1095	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1096	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1097	} while (0)
1098	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1099	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1100	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1101	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1102	} while (0)
1103
1104	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1105	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1106	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1107	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1108	} while (0)
1109	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1110	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1111	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1112	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1113	} while (0)
1114
1115	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1116	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1117	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1118	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1119	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1120	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1121	} while (0)
1122	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1123	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1124	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1125	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1126	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1127	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1128	} while (0)
1129
1130	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1131	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1132	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1133	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1134	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1135	} while (0)
1136	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1137	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1138	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1139	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1140	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1141	} while (0)
1142
1143
1144	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1145	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1146	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1147	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1148	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1149	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1150	} while (0)
1151	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1152	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1153	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1154	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1155	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1156	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1157	} while (0)
1158
1159	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1160	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1161	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1162	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1163	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1164	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1165	} while (0)
1166	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1167	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1168	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1169	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1170	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1171	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1172	} while (0)
1173
1174	#endif
1175
1176	#ifndef IEM_WITH_SETJMP
1177	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1178	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1179	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1180	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1181	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1182	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1183
1184	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1185	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1186	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1187	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1188	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1189	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1190	#else
1191	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1192	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1193	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1194	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1195	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1196	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1197
1198	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1199	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1200	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1201	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1202	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1203	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1204
1205	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1206	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1207	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1208	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1209	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1210	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1211
1212	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1213	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1214	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1215	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1216	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1217	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1218	#endif
1219
1220
1221
1222	#ifndef IEM_WITH_SETJMP
1223	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1224	do { \
1225	uint8_t u8Tmp; \
1226	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1227	(a_u16Dst) = u8Tmp; \
1228	} while (0)
1229	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1230	do { \
1231	uint8_t u8Tmp; \
1232	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1233	(a_u32Dst) = u8Tmp; \
1234	} while (0)
1235	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1236	do { \
1237	uint8_t u8Tmp; \
1238	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1239	(a_u64Dst) = u8Tmp; \
1240	} while (0)
1241	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1242	do { \
1243	uint16_t u16Tmp; \
1244	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1245	(a_u32Dst) = u16Tmp; \
1246	} while (0)
1247	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1248	do { \
1249	uint16_t u16Tmp; \
1250	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1251	(a_u64Dst) = u16Tmp; \
1252	} while (0)
1253	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1254	do { \
1255	uint32_t u32Tmp; \
1256	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1257	(a_u64Dst) = u32Tmp; \
1258	} while (0)
1259	#else /* IEM_WITH_SETJMP */
1260	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1261	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1262	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1263	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1264	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1265	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1266	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1267	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1268	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1269	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1270	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1271	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1272
1273	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1274	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1275	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1276	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1277	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1278	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1279	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1280	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1281	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1282	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1283	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1284	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1285	#endif /* IEM_WITH_SETJMP */
1286
1287	#ifndef IEM_WITH_SETJMP
1288	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1289	do { \
1290	uint8_t u8Tmp; \
1291	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1292	(a_u16Dst) = (int8_t)u8Tmp; \
1293	} while (0)
1294	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1295	do { \
1296	uint8_t u8Tmp; \
1297	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1298	(a_u32Dst) = (int8_t)u8Tmp; \
1299	} while (0)
1300	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1301	do { \
1302	uint8_t u8Tmp; \
1303	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1304	(a_u64Dst) = (int8_t)u8Tmp; \
1305	} while (0)
1306	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1307	do { \
1308	uint16_t u16Tmp; \
1309	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1310	(a_u32Dst) = (int16_t)u16Tmp; \
1311	} while (0)
1312	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1313	do { \
1314	uint16_t u16Tmp; \
1315	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1316	(a_u64Dst) = (int16_t)u16Tmp; \
1317	} while (0)
1318	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1319	do { \
1320	uint32_t u32Tmp; \
1321	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1322	(a_u64Dst) = (int32_t)u32Tmp; \
1323	} while (0)
1324	#else /* IEM_WITH_SETJMP */
1325	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1326	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1327	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1328	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1329	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1330	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1331	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1332	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1333	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1334	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1335	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1336	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1337
1338	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1339	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1340	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1341	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1342	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1343	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1344	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1345	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1346	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1347	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1348	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1349	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1350	#endif /* IEM_WITH_SETJMP */
1351
1352	#ifndef IEM_WITH_SETJMP
1353	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1354	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1355	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1356	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1357	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1358	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1359	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1360	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1361	#else
1362	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1363	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1364	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1365	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1366	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1367	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1368	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1369	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1370
1371	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1372	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1373	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1374	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1375	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1376	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1377	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1378	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1379	#endif
1380
1381	#ifndef IEM_WITH_SETJMP
1382	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1383	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1384	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1385	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1386	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1387	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1388	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1389	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1390	#else
1391	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1392	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1393	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1394	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1395	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1396	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1397	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1398	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1399
1400	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1401	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1402	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1403	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1404	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1405	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1406	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1407	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1408	#endif
1409
1410	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1411	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1412	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1413	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1414	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1415	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1416	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1417	do { \
1418	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1419	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1420	} while (0)
1421	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1422	do { \
1423	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1424	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1425	} while (0)
1426
1427	#ifndef IEM_WITH_SETJMP
1428	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1429	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1430	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1431	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1432	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1433	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1434	#else
1435	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1436	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1437	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1438	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1439	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1440	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1441
1442	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1443	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1444	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1445	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1446	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1447	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1448	#endif
1449
1450	#ifndef IEM_WITH_SETJMP
1451	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1452	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1453	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1454	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1455	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1456	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1457	#else
1458	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1459	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1460	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1461	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1462	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1463	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1464
1465	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1466	iemMemStoreDataU256Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1467	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1468	iemMemStoreDataU256NoAcJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1469	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1470	iemMemStoreDataU256AlignedAvxJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1471	#endif
1472
1473	/* Regular stack push and pop: */
1474	#ifndef IEM_WITH_SETJMP
1475	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1476	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1477	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1478	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1479
1480	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1481	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1482	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1483	#else
1484	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1485	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1486	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1487	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1488
1489	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1490	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1491	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1492	#endif
1493
1494	/* 32-bit flat stack push and pop: */
1495	#ifndef IEM_WITH_SETJMP
1496	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1497	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1498	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1499
1500	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1501	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1502	#else
1503	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1504	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1505	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1506
1507	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1508	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1509	#endif
1510
1511	/* 64-bit flat stack push and pop: */
1512	#ifndef IEM_WITH_SETJMP
1513	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1514	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1515
1516	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1517	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1518	#else
1519	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1520	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1521
1522	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1523	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1524	#endif
1525
1526
1527	/* 8-bit */
1528
1529	/**
1530	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1531	* acccess, for atomic operations.
1532	*
1533	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1534	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1535	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1536	* @param[in] a_GCPtrMem The memory address.
1537	* @remarks Will return/long jump on errors.
1538	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1539	*/
1540	#ifndef IEM_WITH_SETJMP
1541	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1542	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1543	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1544	#else
1545	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1546	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1547	#endif
1548
1549	/**
1550	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1551	*
1552	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1553	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1554	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1555	* @param[in] a_GCPtrMem The memory address.
1556	* @remarks Will return/long jump on errors.
1557	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1558	*/
1559	#ifndef IEM_WITH_SETJMP
1560	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1561	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1562	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1563	#else
1564	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1565	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1566	#endif
1567
1568	/**
1569	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1570	*
1571	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1572	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1573	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1574	* @param[in] a_GCPtrMem The memory address.
1575	* @remarks Will return/long jump on errors.
1576	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1577	*/
1578	#ifndef IEM_WITH_SETJMP
1579	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1580	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1581	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1582	#else
1583	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1584	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1585	#endif
1586
1587	/**
1588	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1589	*
1590	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1591	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1592	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1593	* @param[in] a_GCPtrMem The memory address.
1594	* @remarks Will return/long jump on errors.
1595	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1596	*/
1597	#ifndef IEM_WITH_SETJMP
1598	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1599	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1600	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1601	#else
1602	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1603	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1604	#endif
1605
1606	/**
1607	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1608	* acccess, flat address variant.
1609	*
1610	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1611	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1612	* @param[in] a_GCPtrMem The memory address.
1613	* @remarks Will return/long jump on errors.
1614	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1615	*/
1616	#ifndef IEM_WITH_SETJMP
1617	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1618	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1619	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1620	#else
1621	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1622	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1623	#endif
1624
1625	/**
1626	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1627	* address variant.
1628	*
1629	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1630	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1631	* @param[in] a_GCPtrMem The memory address.
1632	* @remarks Will return/long jump on errors.
1633	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1634	*/
1635	#ifndef IEM_WITH_SETJMP
1636	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1637	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1638	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1639	#else
1640	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1641	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1642	#endif
1643
1644	/**
1645	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1646	* address variant.
1647	*
1648	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1649	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1650	* @param[in] a_GCPtrMem The memory address.
1651	* @remarks Will return/long jump on errors.
1652	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1653	*/
1654	#ifndef IEM_WITH_SETJMP
1655	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1656	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1657	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1658	#else
1659	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1660	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1661	#endif
1662
1663	/**
1664	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1665	* address variant.
1666	*
1667	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1668	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1669	* @param[in] a_GCPtrMem The memory address.
1670	* @remarks Will return/long jump on errors.
1671	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1672	*/
1673	#ifndef IEM_WITH_SETJMP
1674	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1675	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1676	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1677	#else
1678	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1679	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1680	#endif
1681
1682
1683	/* 16-bit */
1684
1685	/**
1686	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1687	*
1688	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1689	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1690	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1691	* @param[in] a_GCPtrMem The memory address.
1692	* @remarks Will return/long jump on errors.
1693	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1694	*/
1695	#ifndef IEM_WITH_SETJMP
1696	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1697	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1698	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1699	#else
1700	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1701	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1702	#endif
1703
1704	/**
1705	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1706	*
1707	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1708	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1709	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1710	* @param[in] a_GCPtrMem The memory address.
1711	* @remarks Will return/long jump on errors.
1712	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1713	*/
1714	#ifndef IEM_WITH_SETJMP
1715	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1716	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1717	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1718	#else
1719	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1720	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1721	#endif
1722
1723	/**
1724	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1725	*
1726	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1727	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1728	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1729	* @param[in] a_GCPtrMem The memory address.
1730	* @remarks Will return/long jump on errors.
1731	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1732	*/
1733	#ifndef IEM_WITH_SETJMP
1734	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1735	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1736	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1737	#else
1738	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1739	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1740	#endif
1741
1742	/**
1743	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1744	*
1745	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1746	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1747	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1748	* @param[in] a_GCPtrMem The memory address.
1749	* @remarks Will return/long jump on errors.
1750	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1751	*/
1752	#ifndef IEM_WITH_SETJMP
1753	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1754	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1755	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1756	#else
1757	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1758	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1759	#endif
1760
1761	/**
1762	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1763	* acccess, flat address variant.
1764	*
1765	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1766	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1767	* @param[in] a_GCPtrMem The memory address.
1768	* @remarks Will return/long jump on errors.
1769	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1770	*/
1771	#ifndef IEM_WITH_SETJMP
1772	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1773	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1774	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1775	#else
1776	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1777	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1778	#endif
1779
1780	/**
1781	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1782	* address variant.
1783	*
1784	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1785	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1786	* @param[in] a_GCPtrMem The memory address.
1787	* @remarks Will return/long jump on errors.
1788	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1789	*/
1790	#ifndef IEM_WITH_SETJMP
1791	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1792	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1793	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1794	#else
1795	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1796	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1797	#endif
1798
1799	/**
1800	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1801	* address variant.
1802	*
1803	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1804	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1805	* @param[in] a_GCPtrMem The memory address.
1806	* @remarks Will return/long jump on errors.
1807	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1808	*/
1809	#ifndef IEM_WITH_SETJMP
1810	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1811	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1812	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1813	#else
1814	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1815	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1816	#endif
1817
1818	/**
1819	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1820	* address variant.
1821	*
1822	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1823	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1824	* @param[in] a_GCPtrMem The memory address.
1825	* @remarks Will return/long jump on errors.
1826	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1827	*/
1828	#ifndef IEM_WITH_SETJMP
1829	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1830	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1831	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1832	#else
1833	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1834	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1835	#endif
1836
1837	/** int16_t alias. */
1838	#ifndef IEM_WITH_SETJMP
1839	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1840	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1841	#else
1842	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1843	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1844	#endif
1845
1846	/** Flat int16_t alias. */
1847	#ifndef IEM_WITH_SETJMP
1848	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1849	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1850	#else
1851	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1852	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1853	#endif
1854
1855
1856	/* 32-bit */
1857
1858	/**
1859	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1860	*
1861	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1862	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1863	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1864	* @param[in] a_GCPtrMem The memory address.
1865	* @remarks Will return/long jump on errors.
1866	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1867	*/
1868	#ifndef IEM_WITH_SETJMP
1869	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1870	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1871	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1872	#else
1873	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1874	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1875	#endif
1876
1877	/**
1878	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1879	*
1880	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1881	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1882	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1883	* @param[in] a_GCPtrMem The memory address.
1884	* @remarks Will return/long jump on errors.
1885	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1886	*/
1887	#ifndef IEM_WITH_SETJMP
1888	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1889	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1890	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
1891	#else
1892	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1893	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1894	#endif
1895
1896	/**
1897	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1898	*
1899	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1900	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1901	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1902	* @param[in] a_GCPtrMem The memory address.
1903	* @remarks Will return/long jump on errors.
1904	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1905	*/
1906	#ifndef IEM_WITH_SETJMP
1907	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1908	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1909	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
1910	#else
1911	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1912	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1913	#endif
1914
1915	/**
1916	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1917	*
1918	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1919	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1920	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1921	* @param[in] a_GCPtrMem The memory address.
1922	* @remarks Will return/long jump on errors.
1923	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1924	*/
1925	#ifndef IEM_WITH_SETJMP
1926	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1927	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1928	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
1929	#else
1930	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1931	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1932	#endif
1933
1934	/**
1935	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
1936	* acccess, flat address variant.
1937	*
1938	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1939	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1940	* @param[in] a_GCPtrMem The memory address.
1941	* @remarks Will return/long jump on errors.
1942	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1943	*/
1944	#ifndef IEM_WITH_SETJMP
1945	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1946	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1947	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1948	#else
1949	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1950	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1951	#endif
1952
1953	/**
1954	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
1955	* flat address variant.
1956	*
1957	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1958	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1959	* @param[in] a_GCPtrMem The memory address.
1960	* @remarks Will return/long jump on errors.
1961	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1962	*/
1963	#ifndef IEM_WITH_SETJMP
1964	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1965	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1966	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
1967	#else
1968	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1969	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1970	#endif
1971
1972	/**
1973	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
1974	* address variant.
1975	*
1976	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1977	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1978	* @param[in] a_GCPtrMem The memory address.
1979	* @remarks Will return/long jump on errors.
1980	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1981	*/
1982	#ifndef IEM_WITH_SETJMP
1983	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1984	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
1985	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
1986	#else
1987	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1988	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1989	#endif
1990
1991	/**
1992	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
1993	* address variant.
1994	*
1995	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1996	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1997	* @param[in] a_GCPtrMem The memory address.
1998	* @remarks Will return/long jump on errors.
1999	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2000	*/
2001	#ifndef IEM_WITH_SETJMP
2002	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2003	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2004	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2005	#else
2006	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2007	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2008	#endif
2009
2010	/** int32_t alias. */
2011	#ifndef IEM_WITH_SETJMP
2012	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2013	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2014	#else
2015	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2016	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2017	#endif
2018
2019	/** Flat int32_t alias. */
2020	#ifndef IEM_WITH_SETJMP
2021	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2022	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2023	#else
2024	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2025	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2026	#endif
2027
2028	/** RTFLOAT32U alias. */
2029	#ifndef IEM_WITH_SETJMP
2030	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2031	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2032	#else
2033	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2034	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2035	#endif
2036
2037	/** Flat RTFLOAT32U alias. */
2038	#ifndef IEM_WITH_SETJMP
2039	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2040	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2041	#else
2042	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2043	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2044	#endif
2045
2046
2047	/* 64-bit */
2048
2049	/**
2050	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2051	*
2052	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2053	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2054	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2055	* @param[in] a_GCPtrMem The memory address.
2056	* @remarks Will return/long jump on errors.
2057	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2058	*/
2059	#ifndef IEM_WITH_SETJMP
2060	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2061	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2062	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2063	#else
2064	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2065	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2066	#endif
2067
2068	/**
2069	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2070	*
2071	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2072	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2073	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2074	* @param[in] a_GCPtrMem The memory address.
2075	* @remarks Will return/long jump on errors.
2076	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2077	*/
2078	#ifndef IEM_WITH_SETJMP
2079	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2080	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2081	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2082	#else
2083	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2084	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2085	#endif
2086
2087	/**
2088	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2089	*
2090	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2091	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2092	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2093	* @param[in] a_GCPtrMem The memory address.
2094	* @remarks Will return/long jump on errors.
2095	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2096	*/
2097	#ifndef IEM_WITH_SETJMP
2098	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2099	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2100	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2101	#else
2102	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2103	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2104	#endif
2105
2106	/**
2107	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2108	*
2109	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2110	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2111	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2112	* @param[in] a_GCPtrMem The memory address.
2113	* @remarks Will return/long jump on errors.
2114	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2115	*/
2116	#ifndef IEM_WITH_SETJMP
2117	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2118	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2119	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2120	#else
2121	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2122	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2123	#endif
2124
2125	/**
2126	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2127	* acccess, flat address variant.
2128	*
2129	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2130	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2131	* @param[in] a_GCPtrMem The memory address.
2132	* @remarks Will return/long jump on errors.
2133	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2134	*/
2135	#ifndef IEM_WITH_SETJMP
2136	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2137	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2138	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2139	#else
2140	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2141	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2142	#endif
2143
2144	/**
2145	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2146	* flat address variant.
2147	*
2148	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2149	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2150	* @param[in] a_GCPtrMem The memory address.
2151	* @remarks Will return/long jump on errors.
2152	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2153	*/
2154	#ifndef IEM_WITH_SETJMP
2155	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2156	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2157	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2158	#else
2159	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2160	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2161	#endif
2162
2163	/**
2164	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2165	* address variant.
2166	*
2167	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2168	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2169	* @param[in] a_GCPtrMem The memory address.
2170	* @remarks Will return/long jump on errors.
2171	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2172	*/
2173	#ifndef IEM_WITH_SETJMP
2174	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2175	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2176	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2177	#else
2178	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2179	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2180	#endif
2181
2182	/**
2183	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2184	* address variant.
2185	*
2186	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2187	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2188	* @param[in] a_GCPtrMem The memory address.
2189	* @remarks Will return/long jump on errors.
2190	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2191	*/
2192	#ifndef IEM_WITH_SETJMP
2193	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2194	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2195	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2196	#else
2197	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2198	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2199	#endif
2200
2201	/** int64_t alias. */
2202	#ifndef IEM_WITH_SETJMP
2203	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2204	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2205	#else
2206	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2207	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2208	#endif
2209
2210	/** Flat int64_t alias. */
2211	#ifndef IEM_WITH_SETJMP
2212	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2213	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2214	#else
2215	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2216	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2217	#endif
2218
2219	/** RTFLOAT64U alias. */
2220	#ifndef IEM_WITH_SETJMP
2221	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2222	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2223	#else
2224	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2225	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2226	#endif
2227
2228	/** Flat RTFLOAT64U alias. */
2229	#ifndef IEM_WITH_SETJMP
2230	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2231	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2232	#else
2233	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2234	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2235	#endif
2236
2237
2238	/* 128-bit */
2239
2240	/**
2241	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2242	*
2243	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2244	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2245	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2246	* @param[in] a_GCPtrMem The memory address.
2247	* @remarks Will return/long jump on errors.
2248	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2249	*/
2250	#ifndef IEM_WITH_SETJMP
2251	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2252	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2253	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2254	#else
2255	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2256	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2257	#endif
2258
2259	/**
2260	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2261	*
2262	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2263	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2264	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2265	* @param[in] a_GCPtrMem The memory address.
2266	* @remarks Will return/long jump on errors.
2267	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2268	*/
2269	#ifndef IEM_WITH_SETJMP
2270	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2271	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2272	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2273	#else
2274	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2275	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2276	#endif
2277
2278	/**
2279	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2280	*
2281	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2282	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2283	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2284	* @param[in] a_GCPtrMem The memory address.
2285	* @remarks Will return/long jump on errors.
2286	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2287	*/
2288	#ifndef IEM_WITH_SETJMP
2289	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2290	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2291	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2292	#else
2293	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2294	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2295	#endif
2296
2297	/**
2298	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2299	*
2300	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2301	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2302	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2303	* @param[in] a_GCPtrMem The memory address.
2304	* @remarks Will return/long jump on errors.
2305	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2306	*/
2307	#ifndef IEM_WITH_SETJMP
2308	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2309	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2310	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2311	#else
2312	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2313	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2314	#endif
2315
2316	/**
2317	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2318	* access, flat address variant.
2319	*
2320	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2321	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2322	* @param[in] a_GCPtrMem The memory address.
2323	* @remarks Will return/long jump on errors.
2324	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2325	*/
2326	#ifndef IEM_WITH_SETJMP
2327	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2328	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2329	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2330	#else
2331	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2332	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2333	#endif
2334
2335	/**
2336	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2337	* flat address variant.
2338	*
2339	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2340	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2341	* @param[in] a_GCPtrMem The memory address.
2342	* @remarks Will return/long jump on errors.
2343	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2344	*/
2345	#ifndef IEM_WITH_SETJMP
2346	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2347	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2348	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2349	#else
2350	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2351	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2352	#endif
2353
2354	/**
2355	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2356	* flat address variant.
2357	*
2358	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2359	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2360	* @param[in] a_GCPtrMem The memory address.
2361	* @remarks Will return/long jump on errors.
2362	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2363	*/
2364	#ifndef IEM_WITH_SETJMP
2365	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2366	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2367	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2368	#else
2369	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2370	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2371	#endif
2372
2373	/**
2374	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2375	* address variant.
2376	*
2377	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2378	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2379	* @param[in] a_GCPtrMem The memory address.
2380	* @remarks Will return/long jump on errors.
2381	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2382	*/
2383	#ifndef IEM_WITH_SETJMP
2384	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2385	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2386	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2387	#else
2388	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2389	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2390	#endif
2391
2392
2393	/* misc */
2394
2395	/**
2396	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2397	*
2398	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2399	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2400	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2401	* @param[in] a_GCPtrMem The memory address.
2402	* @remarks Will return/long jump on errors.
2403	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2404	*/
2405	#ifndef IEM_WITH_SETJMP
2406	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2407	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2408	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2409	#else
2410	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2411	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2412	#endif
2413
2414	/**
2415	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2416	*
2417	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2418	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2419	* @param[in] a_GCPtrMem The memory address.
2420	* @remarks Will return/long jump on errors.
2421	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2422	*/
2423	#ifndef IEM_WITH_SETJMP
2424	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2425	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2426	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2427	#else
2428	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2429	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2430	#endif
2431
2432
2433	/**
2434	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2435	*
2436	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2437	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2438	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2439	* @param[in] a_GCPtrMem The memory address.
2440	* @remarks Will return/long jump on errors.
2441	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2442	*/
2443	#ifndef IEM_WITH_SETJMP
2444	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2445	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2446	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2447	#else
2448	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2449	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2450	#endif
2451
2452	/**
2453	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2454	*
2455	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2456	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2457	* @param[in] a_GCPtrMem The memory address.
2458	* @remarks Will return/long jump on errors.
2459	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2460	*/
2461	#ifndef IEM_WITH_SETJMP
2462	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2463	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2464	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2465	#else
2466	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2467	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2468	#endif
2469
2470
2471
2472	/* commit + unmap */
2473
2474	/** Commits the memory and unmaps guest memory previously mapped RW.
2475	* @remarks May return.
2476	* @note Implictly frees the a_bMapInfo variable.
2477	*/
2478	#ifndef IEM_WITH_SETJMP
2479	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2480	#else
2481	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2482	#endif
2483
2484	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2485	* @remarks May return.
2486	* @note Implictly frees the a_bMapInfo variable.
2487	*/
2488	#ifndef IEM_WITH_SETJMP
2489	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2490	#else
2491	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2492	#endif
2493
2494	/** Commits the memory and unmaps guest memory previously mapped W.
2495	* @remarks May return.
2496	* @note Implictly frees the a_bMapInfo variable.
2497	*/
2498	#ifndef IEM_WITH_SETJMP
2499	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2500	#else
2501	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2502	#endif
2503
2504	/** Commits the memory and unmaps guest memory previously mapped R.
2505	* @remarks May return.
2506	* @note Implictly frees the a_bMapInfo variable.
2507	*/
2508	#ifndef IEM_WITH_SETJMP
2509	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2510	#else
2511	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2512	#endif
2513
2514
2515	/** Commits the memory and unmaps the guest memory unless the FPU status word
2516	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2517	* that would cause FLD not to store.
2518	*
2519	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2520	* store, while \#P will not.
2521	*
2522	* @remarks May in theory return - for now.
2523	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2524	*/
2525	#ifndef IEM_WITH_SETJMP
2526	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2527	if ( !(a_u16FSW & X86_FSW_ES) \
2528	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2529	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2530	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2531	else \
2532	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2533	} while (0)
2534	#else
2535	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2536	if ( !(a_u16FSW & X86_FSW_ES) \
2537	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2538	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2539	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2540	else \
2541	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2542	} while (0)
2543	#endif
2544
2545	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2546	* @note Implictly frees the a_bMapInfo variable. */
2547	#ifndef IEM_WITH_SETJMP
2548	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2549	#else
2550	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2551	#endif
2552
2553
2554
2555	/** Calculate efficient address from R/M. */
2556	#ifndef IEM_WITH_SETJMP
2557	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2558	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2559	#else
2560	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2561	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2562	#endif
2563
2564
2565	/** The @a a_fSpportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2566	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2567	#define IEM_MC_NATIVE_ELSE() } else {
2568	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2569
2570	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2571	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2572	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2573	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2574	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2575	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2576	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2577	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2578	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2579
2580
2581	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2582	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2583	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2584	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2585	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2586	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
2587	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
2588
2589
2590	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2591	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2592	*/
2593	#ifdef VBOX_STRICT
2594	#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2595	do { \
2596	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2597	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2598	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2599	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2600	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2601	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2602	if (rcStrictHlp == VINF_SUCCESS) \
2603	{ \
2604	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2605	\|\| ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
2606	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2607	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2608	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2609	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2610	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2611	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
2612	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2613	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2614	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2615	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2616	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2617	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2618	else \
2619	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2620	== (fEflBefore & ~(X86_EFL_RF)), \
2621	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2622	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2623	{ \
2624	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2625	AssertMsg( fExecBefore == fExecRecalc \
2626	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2627	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2628	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2629	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2630	} \
2631	} \
2632	return rcStrictHlp; \
2633	} while (0)
2634	#else
2635	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2636	#endif
2637
2638	/**
2639	* Defers the rest of the instruction emulation to a C implementation routine
2640	* and returns, only taking the standard parameters.
2641	*
2642	* @param a_fFlags IEM_CIMPL_F_XXX.
2643	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2644	* in the native recompiler.
2645	* @param a_pfnCImpl The pointer to the C routine.
2646	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2647	*/
2648	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2649	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2650
2651	/**
2652	* Defers the rest of instruction emulation to a C implementation routine and
2653	* returns, taking one argument in addition to the standard ones.
2654	*
2655	* @param a_fFlags IEM_CIMPL_F_XXX.
2656	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2657	* in the native recompiler.
2658	* @param a_pfnCImpl The pointer to the C routine.
2659	* @param a0 The argument.
2660	*/
2661	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2662	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2663
2664	/**
2665	* Defers the rest of the instruction emulation to a C implementation routine
2666	* and returns, taking two arguments in addition to the standard ones.
2667	*
2668	* @param a_fFlags IEM_CIMPL_F_XXX.
2669	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2670	* in the native recompiler.
2671	* @param a_pfnCImpl The pointer to the C routine.
2672	* @param a0 The first extra argument.
2673	* @param a1 The second extra argument.
2674	*/
2675	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2676	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2677
2678	/**
2679	* Defers the rest of the instruction emulation to a C implementation routine
2680	* and returns, taking three arguments in addition to the standard ones.
2681	*
2682	* @param a_fFlags IEM_CIMPL_F_XXX.
2683	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2684	* in the native recompiler.
2685	* @param a_pfnCImpl The pointer to the C routine.
2686	* @param a0 The first extra argument.
2687	* @param a1 The second extra argument.
2688	* @param a2 The third extra argument.
2689	*/
2690	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2691	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2692
2693	/**
2694	* Defers the rest of the instruction emulation to a C implementation routine
2695	* and returns, taking four arguments in addition to the standard ones.
2696	*
2697	* @param a_fFlags IEM_CIMPL_F_XXX.
2698	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2699	* in the native recompiler.
2700	* @param a_pfnCImpl The pointer to the C routine.
2701	* @param a0 The first extra argument.
2702	* @param a1 The second extra argument.
2703	* @param a2 The third extra argument.
2704	* @param a3 The fourth extra argument.
2705	*/
2706	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2707	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2708
2709	/**
2710	* Defers the rest of the instruction emulation to a C implementation routine
2711	* and returns, taking two arguments in addition to the standard ones.
2712	*
2713	* @param a_fFlags IEM_CIMPL_F_XXX.
2714	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2715	* in the native recompiler.
2716	* @param a_pfnCImpl The pointer to the C routine.
2717	* @param a0 The first extra argument.
2718	* @param a1 The second extra argument.
2719	* @param a2 The third extra argument.
2720	* @param a3 The fourth extra argument.
2721	* @param a4 The fifth extra argument.
2722	*/
2723	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2724	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2725
2726	/**
2727	* Defers the entire instruction emulation to a C implementation routine and
2728	* returns, only taking the standard parameters.
2729	*
2730	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2731	*
2732	* @param a_fFlags IEM_CIMPL_F_XXX.
2733	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2734	* in the native recompiler.
2735	* @param a_pfnCImpl The pointer to the C routine.
2736	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2737	*/
2738	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2739	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2740
2741	/**
2742	* Defers the entire instruction emulation to a C implementation routine and
2743	* returns, taking one argument in addition to the standard ones.
2744	*
2745	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2746	*
2747	* @param a_fFlags IEM_CIMPL_F_XXX.
2748	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2749	* in the native recompiler.
2750	* @param a_pfnCImpl The pointer to the C routine.
2751	* @param a0 The argument.
2752	*/
2753	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2754	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2755
2756	/**
2757	* Defers the entire instruction emulation to a C implementation routine and
2758	* returns, taking two arguments in addition to the standard ones.
2759	*
2760	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2761	*
2762	* @param a_fFlags IEM_CIMPL_F_XXX.
2763	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2764	* in the native recompiler.
2765	* @param a_pfnCImpl The pointer to the C routine.
2766	* @param a0 The first extra argument.
2767	* @param a1 The second extra argument.
2768	*/
2769	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2770	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2771
2772	/**
2773	* Defers the entire instruction emulation to a C implementation routine and
2774	* returns, taking three arguments in addition to the standard ones.
2775	*
2776	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2777	*
2778	* @param a_fFlags IEM_CIMPL_F_XXX.
2779	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2780	* in the native recompiler.
2781	* @param a_pfnCImpl The pointer to the C routine.
2782	* @param a0 The first extra argument.
2783	* @param a1 The second extra argument.
2784	* @param a2 The third extra argument.
2785	*/
2786	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2787	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2788
2789
2790	/**
2791	* Calls a FPU assembly implementation taking one visible argument.
2792	*
2793	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2794	* @param a0 The first extra argument.
2795	*/
2796	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2797	do { \
2798	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2799	} while (0)
2800
2801	/**
2802	* Calls a FPU assembly implementation taking two visible arguments.
2803	*
2804	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2805	* @param a0 The first extra argument.
2806	* @param a1 The second extra argument.
2807	*/
2808	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2809	do { \
2810	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2811	} while (0)
2812
2813	/**
2814	* Calls a FPU assembly implementation taking three visible arguments.
2815	*
2816	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2817	* @param a0 The first extra argument.
2818	* @param a1 The second extra argument.
2819	* @param a2 The third extra argument.
2820	*/
2821	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2822	do { \
2823	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2824	} while (0)
2825
2826	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2827	do { \
2828	(a_FpuData).FSW = (a_FSW); \
2829	(a_FpuData).r80Result = *(a_pr80Value); \
2830	} while (0)
2831
2832	/** Pushes FPU result onto the stack. */
2833	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2834	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2835	/** Pushes FPU result onto the stack and sets the FPUDP. */
2836	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2837	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2838
2839	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2840	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2841	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2842
2843	/** Stores FPU result in a stack register. */
2844	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2845	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2846	/** Stores FPU result in a stack register and pops the stack. */
2847	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2848	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2849	/** Stores FPU result in a stack register and sets the FPUDP. */
2850	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2851	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2852	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2853	* stack. */
2854	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2855	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2856
2857	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2858	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2859	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2860	/** Free a stack register (for FFREE and FFREEP). */
2861	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2862	iemFpuStackFree(pVCpu, a_iStReg)
2863	/** Increment the FPU stack pointer. */
2864	#define IEM_MC_FPU_STACK_INC_TOP() \
2865	iemFpuStackIncTop(pVCpu)
2866	/** Decrement the FPU stack pointer. */
2867	#define IEM_MC_FPU_STACK_DEC_TOP() \
2868	iemFpuStackDecTop(pVCpu)
2869
2870	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2871	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2872	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2873	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2874	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2875	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2876	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2877	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2878	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2879	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2880	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2881	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2882	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2883	* stack. */
2884	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2885	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2886	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2887	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2888	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2889
2890	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2891	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2892	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2893	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2894	* stack. */
2895	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2896	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2897	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2898	* FPUDS. */
2899	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2900	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2901	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2902	* FPUDS. Pops stack. */
2903	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2904	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2905	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2906	* stack twice. */
2907	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2908	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2909	/** Raises a FPU stack underflow exception for an instruction pushing a result
2910	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
2911	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2912	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2913	/** Raises a FPU stack underflow exception for an instruction pushing a result
2914	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2915	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2916	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2917
2918	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2919	* FPUIP, FPUCS and FOP. */
2920	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2921	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2922	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2923	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2924	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2925	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2926	/** Prepares for using the FPU state.
2927	* Ensures that we can use the host FPU in the current context (RC+R0.
2928	* Ensures the guest FPU state in the CPUMCTX is up to date. */
2929	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2930	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2931	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2932	/** Actualizes the guest FPU state so it can be accessed and modified. */
2933	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2934
2935	/** Stores SSE SIMD result updating MXCSR. */
2936	#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
2937	iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
2938	/** Updates MXCSR. */
2939	#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
2940	iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
2941
2942	/** Prepares for using the SSE state.
2943	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2944	* Ensures the guest SSE state in the CPUMCTX is up to date. */
2945	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2946	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2947	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
2948	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
2949	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
2950
2951	/** Prepares for using the AVX state.
2952	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
2953	* Ensures the guest AVX state in the CPUMCTX is up to date.
2954	* @note This will include the AVX512 state too when support for it is added
2955	* due to the zero extending feature of VEX instruction. */
2956	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
2957	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2958	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
2959	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
2960	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
2961
2962	/**
2963	* Calls a MMX assembly implementation taking two visible arguments.
2964	*
2965	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2966	* @param a0 The first extra argument.
2967	* @param a1 The second extra argument.
2968	*/
2969	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
2970	do { \
2971	IEM_MC_PREPARE_FPU_USAGE(); \
2972	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2973	} while (0)
2974
2975	/**
2976	* Calls a MMX assembly implementation taking three visible arguments.
2977	*
2978	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2979	* @param a0 The first extra argument.
2980	* @param a1 The second extra argument.
2981	* @param a2 The third extra argument.
2982	*/
2983	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2984	do { \
2985	IEM_MC_PREPARE_FPU_USAGE(); \
2986	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2987	} while (0)
2988
2989
2990	/**
2991	* Calls a SSE assembly implementation taking two visible arguments.
2992	*
2993	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2994	* @param a0 The first extra argument.
2995	* @param a1 The second extra argument.
2996	*/
2997	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
2998	do { \
2999	IEM_MC_PREPARE_SSE_USAGE(); \
3000	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3001	} while (0)
3002
3003	/**
3004	* Calls a SSE assembly implementation taking three visible arguments.
3005	*
3006	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3007	* @param a0 The first extra argument.
3008	* @param a1 The second extra argument.
3009	* @param a2 The third extra argument.
3010	*/
3011	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3012	do { \
3013	IEM_MC_PREPARE_SSE_USAGE(); \
3014	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3015	} while (0)
3016
3017
3018	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
3019	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ...
3020	* @note IEMAllInstPython.py duplicates the expansion. */
3021	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
3022	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
3023
3024	/**
3025	* Calls a AVX assembly implementation taking two visible arguments.
3026	*
3027	* There is one implicit zero'th argument, a pointer to the extended state.
3028	*
3029	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3030	* @param a1 The first extra argument.
3031	* @param a2 The second extra argument.
3032	*/
3033	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
3034	do { \
3035	IEM_MC_PREPARE_AVX_USAGE(); \
3036	a_pfnAImpl(pXState, (a1), (a2)); \
3037	} while (0)
3038
3039	/**
3040	* Calls a AVX assembly implementation taking three visible arguments.
3041	*
3042	* There is one implicit zero'th argument, a pointer to the extended state.
3043	*
3044	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3045	* @param a1 The first extra argument.
3046	* @param a2 The second extra argument.
3047	* @param a3 The third extra argument.
3048	*/
3049	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
3050	do { \
3051	IEM_MC_PREPARE_AVX_USAGE(); \
3052	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
3053	} while (0)
3054
3055	/** @note Not for IOPL or IF testing. */
3056	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3057	/** @note Not for IOPL or IF testing. */
3058	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3059	/** @note Not for IOPL or IF testing. */
3060	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3061	/** @note Not for IOPL or IF testing. */
3062	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3063	/** @note Not for IOPL or IF testing. */
3064	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3065	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3066	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3067	/** @note Not for IOPL or IF testing. */
3068	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3069	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3070	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3071	/** @note Not for IOPL or IF testing. */
3072	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3073	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3074	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3075	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3076	/** @note Not for IOPL or IF testing. */
3077	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3078	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3079	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3080	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3081	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3082	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3083	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3084	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3085	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3086	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3087	/** @note Not for IOPL or IF testing. */
3088	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3089	if ( pVCpu->cpum.GstCtx.cx != 1 \
3090	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3091	/** @note Not for IOPL or IF testing. */
3092	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3093	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3094	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3095	/** @note Not for IOPL or IF testing. */
3096	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3097	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3098	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3099	/** @note Not for IOPL or IF testing. */
3100	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3101	if ( pVCpu->cpum.GstCtx.cx != 1 \
3102	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3103	/** @note Not for IOPL or IF testing. */
3104	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3105	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3106	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3107	/** @note Not for IOPL or IF testing. */
3108	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3109	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3110	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3111	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3112	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3113
3114	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3115	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3116	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3117	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3118	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3119	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3120	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3121	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3122	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3123	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3124	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3125	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3126	#define IEM_MC_IF_FCW_IM() \
3127	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3128	#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
3129	if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3130	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
3131
3132	#define IEM_MC_ELSE() } else {
3133	#define IEM_MC_ENDIF() } do {} while (0)
3134
3135
3136	/** Recompiler debugging: Flush guest register shadow copies. */
3137	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3138
3139	/** @} */
3140
3141	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3142

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 104016

Download in other formats: