IEMMc.h@ 107218

Last change on this file since 107218 was 107218, checked in by vboxsync, 2 months ago

ValidationKit/bootsectors: fix IEM implementation of vcmpp[sd]; bugref: 10658; jiraref:VBP-1208

part 3 of previous fix (r166170)
add microcode macros to fetch media register + memory without alignment checks:
add IEM_MC_FETCH_MEM_YMM_NO_AC_AND_YREG_YMM
add IEM_MC_FETCH_MEM_XMM_NO_AC_AND_XREG_XMM
add IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC_AND_XREG_XMM
remove unused IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM
remove unused IEM_MC_FETCH_MEM_XMM_ALIGN_AVX_AND_XREG_XMM

(this commit breaks the build until immediately following vdpp[sd] commit)

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 183.6 KB

Line
1	/* $Id: IEMMc.h 107218 2024-12-03 09:33:07Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
85	* @note only usable in 16-bit op size mode. */
86	#define IEM_MC_REL_CALL_S16_AND_FINISH(a_i16) \
87	return iemRegRipRelativeCallS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
88	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
89	#define IEM_MC_REL_CALL_S32_AND_FINISH(a_i32) \
90	return iemRegEip32RelativeCallS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32))
91	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
92	#define IEM_MC_REL_CALL_S64_AND_FINISH(a_i64) \
93	return iemRegRip64RelativeCallS64AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i64))
94	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
95	#define IEM_MC_IND_CALL_U16_AND_FINISH(a_u16NewIP) \
96	return iemRegIp16IndirectCallU16AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16NewIP))
97	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
98	#define IEM_MC_IND_CALL_U32_AND_FINISH(a_u32NewIP) \
99	return iemRegEip32IndirectCallU32AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u32NewIP))
100	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
101	#define IEM_MC_IND_CALL_U64_AND_FINISH(a_u64NewIP) \
102	return iemRegRip64IndirectCallU64AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u64NewIP))
103
104
105	/** Fetches the near return address from the stack, sets RIP and RSP (may trigger
106	* \#GP or \#SS), finishes the instruction and returns. */
107	#define IEM_MC_RETN_AND_FINISH(a_cbPopArgs) \
108	return iemRegRipNearReturnAndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_cbPopArgs), pVCpu->iem.s.enmEffOpSize)
109
110
111	#define IEM_MC_RAISE_DIVIDE_ERROR_IF_LOCAL_IS_ZERO(a_uVar) \
112	do { \
113	if (RT_LIKELY((a_uVar) != 0)) \
114	{ /* probable */ } \
115	else return iemRaiseDivideError(pVCpu); \
116	} while (0)
117	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
118	do { \
119	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
120	{ /* probable */ } \
121	else return iemRaiseDeviceNotAvailable(pVCpu); \
122	} while (0)
123	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
124	do { \
125	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
126	{ /* probable */ } \
127	else return iemRaiseDeviceNotAvailable(pVCpu); \
128	} while (0)
129	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
130	do { \
131	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
132	{ /* probable */ } \
133	else return iemRaiseMathFault(pVCpu); \
134	} while (0)
135	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
136	do { \
137	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
138	be reduced to a single compare branch in the more probably code path. */ \
139	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
140	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
141	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
142	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
143	{ /* probable */ } \
144	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
145	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
146	return iemRaiseUndefinedOpcode(pVCpu); \
147	else \
148	return iemRaiseDeviceNotAvailable(pVCpu); \
149	} while (0)
150	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
151	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
152	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
153	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
154	do { \
155	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
156	single compare branch in the more probable code path. */ \
157	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
158	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
159	== X86_CR4_OSFXSR)) \
160	{ /* likely */ } \
161	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
162	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
163	return iemRaiseUndefinedOpcode(pVCpu); \
164	else \
165	return iemRaiseDeviceNotAvailable(pVCpu); \
166	} while (0)
167	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
168	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
169	do { \
170	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
171	single compare branch in the more probable code path. */ \
172	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
173	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
174	{ /* probable */ } \
175	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
176	return iemRaiseUndefinedOpcode(pVCpu); \
177	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
178	return iemRaiseDeviceNotAvailable(pVCpu); \
179	else \
180	return iemRaiseMathFault(pVCpu); \
181	} while (0)
182	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
183	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
184	* count the CPL into the TB key. However it is safe enough for now, as
185	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
186	* emitted for it. */
187	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
188	do { \
189	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
190	else return iemRaiseGeneralProtectionFault0(pVCpu); \
191	} while (0)
192	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
193	do { \
194	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
195	else return iemRaiseGeneralProtectionFault0(pVCpu); \
196	} while (0)
197	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
198	do { \
199	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
200	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
201	{ /* probable */ } \
202	else return iemRaiseUndefinedOpcode(pVCpu); \
203	} while (0)
204	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
205	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
206	do { \
207	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
208	else return iemRaiseGeneralProtectionFault0(pVCpu); \
209	} while (0)
210
211
212	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
213	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
214	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
215	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
216	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
217	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
218	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
219	/** @note IEMAllInstPython.py duplicates the expansion. */
220	#define IEM_MC_ARG_EFLAGS(a_Name, a_iArg) uint32_t const a_Name = pVCpu->cpum.GstCtx.eflags.u
221	/** @note IEMAllInstPython.py duplicates the expansion. */
222	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
223	uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
224	uint32_t *a_pName = &a_Name
225	/** @note IEMAllInstPython.py duplicates the expansion. */
226	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
227	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
228	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
229	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
230	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
231	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
232	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
233	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
234	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
235	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
236	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
237	} while (0)
238	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
239	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
240
241	/** ASSUMES the source variable not used after this statement. */
242	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
243
244	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
245	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
246	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
247	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
248	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
249	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
250	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
251	#define IEM_MC_FETCH_GREG_I16(a_i16Dst, a_iGReg) (a_i16Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
252	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
253	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
254	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
255	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
256	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
257	#define IEM_MC_FETCH_GREG_I32(a_i32Dst, a_iGReg) (a_i32Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
258	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
259	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
260	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
261	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
262	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
263	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
264	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
265	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
266	} while(0)
267	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
268	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
269	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
270	} while(0)
271	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
272	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
273	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
274	} while (0)
275	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
276	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
277	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
278	} while (0)
279	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
280	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
281	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
282	} while (0)
283	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
284	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
285	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
286	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
287	} while (0)
288	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
289	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
290	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
291	} while (0)
292	/** @note Not for IOPL or IF testing or modification. */
293	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
294	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
295	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
296	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
297	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
298
299	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
300	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
301	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
302	#define IEM_MC_STORE_GREG_I32(a_iGReg, a_i32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_i32Value) / clear high bits. */
303	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
304	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
305	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
306	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
307	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
308	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
309	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
310	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
311	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
312	} while(0)
313	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
314	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
315	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
316	} while(0)
317	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
318
319	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
320	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
321	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
322	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
323	} while (0)
324	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
325	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
326	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
327	} while (0)
328	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
329	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
330
331
332	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
333	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
334	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
335	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
336	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
337	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
338	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
339	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
340	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
341	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
342	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
343	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
344	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
345	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
346	/** @note Not for IOPL or IF testing or modification.
347	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
348	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
349	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
350
351	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
352	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
353	do { \
354	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
355	*pu32Reg += (a_u32Value); \
356	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
357	} while (0)
358	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
359
360	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
361	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
362	do { \
363	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
364	*pu32Reg -= (a_u8Const); \
365	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
366	} while (0)
367	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
368	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
369
370	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
371	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
372	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
373	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
374	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
375	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
376	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
377
378	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
379	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
380	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
381	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
382
383	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
384	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
385	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
386
387	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
388	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
389	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
390
391	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
392	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
393	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
394
395	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
396
397	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
398	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
399	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
400
401	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
402
403	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
404
405	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
406	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
407	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
408	do { \
409	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
410	*pu32Reg &= (a_u32Value); \
411	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
412	} while (0)
413	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
414
415	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
416	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
417	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
418	do { \
419	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
420	*pu32Reg \|= (a_u32Value); \
421	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
422	} while (0)
423	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
424
425	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
426	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
427	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
428
429	/** @note Not for IOPL or IF modification. */
430	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
431	/** @note Not for IOPL or IF modification. */
432	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
433	/** @note Not for IOPL or IF modification. */
434	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
435
436	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
437
438	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
439	#define IEM_MC_FPU_TO_MMX_MODE() do { \
440	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
441	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
442	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
443	} while (0)
444
445	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
446	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
447	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
448	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
449	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
450	} while (0)
451
452	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
453	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
454	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
455	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
456	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
457	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
458	#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
459	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
460	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
461	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
462	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
463	} while (0)
464	#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
465	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
466	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
467	} while (0)
468	#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
469	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
470	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
471	} while (0)
472	#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
473	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
474	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
475	} while (0)
476	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
477	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
478	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
479	} while (0)
480	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
481	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
482	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
483	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
484	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
485	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
486	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
487	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
488	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
489	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
490
491	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
492	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
493	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
494	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
495	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
496	} while (0)
497	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
498	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
499	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
500	} while (0)
501	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
502	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
503	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
504	} while (0)
505	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
506	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
507	#define IEM_MC_FETCH_XREG_R64(a_r64Value, a_iXReg, a_iQWord) \
508	do { (a_r64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[(a_iQWord)]; } while (0)
509	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
510	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
511	#define IEM_MC_FETCH_XREG_R32(a_r32Value, a_iXReg, a_iDWord) \
512	do { (a_r32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[(a_iDWord)]; } while (0)
513	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
514	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
515	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
516	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
517	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
518	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
519	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
520	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
521	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
522	} while (0)
523	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
524	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
525	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
526	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
527	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
528	} while (0)
529	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
530	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
531	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
532	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
533	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
534	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
535	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
536	} while (0)
537	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
538	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
539	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
540	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
541	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
542	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
543	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
544	} while (0)
545	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
546	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
547	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
548	} while (0)
549	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
550	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
551	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
552	} while (0)
553	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
554	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
555	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
556	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
557	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
558	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
559	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
560	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
561	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
562	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
563	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
564	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
565
566	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
567	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
568	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
569	} while (0)
570
571	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
572	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
573	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
574	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
575	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
576	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
577	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
578	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
579	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
580	} while (0)
581
582	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
583	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
584	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
585	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
586	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
587	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
588	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
590	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
591	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
598	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
599	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
600	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
601	} while (0)
602	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
603	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
605	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
606	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
607	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
608	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
612	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
613	} while (0)
614	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
615	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
616	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
617	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
618	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
619	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
620	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
621	} while (0)
622	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
623	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
624	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
625	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
626	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
627	} while (0)
628
629	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
630	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
631	#define IEM_MC_REF_XREG_XMM(a_pXmmDst, a_iXReg) \
632	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
633	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
634	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
635	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
636	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
637	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
638	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
639	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
640	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
641	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
642	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
643	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
644	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
645	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
646	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
647	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
648	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
649	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
650	} while (0)
651
652	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
653	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
654	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
655	} while (0)
656	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
657	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
658	if ((a_iQWord) < 2) \
659	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
660	else \
661	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[(a_iQWord) - 2]; \
662	} while (0)
663	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
664	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
665	if ((a_iDQword) == 0) \
666	{ \
667	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
668	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
669	} \
670	else \
671	{ \
672	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
673	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
674	} \
675	} while (0)
676	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
677	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
678	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
679	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
680	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
681	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
682	} while (0)
683	#define IEM_MC_FETCH_YREG_YMM(a_uYmmDst, a_iYRegSrc) \
684	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
685	(a_uYmmDst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
686	(a_uYmmDst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
687	(a_uYmmDst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
688	(a_uYmmDst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
689	} while (0)
690	#define IEM_MC_FETCH_YREG_PAIR_YMM(a_uYmmDst, a_iYRegSrc1, a_iYRegSrc2) \
691	do { uintptr_t const iYRegSrc1Tmp = (a_iYRegSrc1); \
692	uintptr_t const iYRegSrc2Tmp = (a_iYRegSrc2); \
693	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[0]; \
694	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[1]; \
695	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[0]; \
696	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[1]; \
697	(a_uYmmDst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[0]; \
698	(a_uYmmDst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[1]; \
699	(a_uYmmDst).uSrc2.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[0]; \
700	(a_uYmmDst).uSrc2.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[1]; \
701	} while (0)
702
703	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
704	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
705	if ((a_iDQword) == 0) \
706	{ \
707	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
708	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
709	} \
710	else \
711	{ \
712	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
713	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
714	} \
715	} while (0)
716
717	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
718	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
719	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
720	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
721	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
722	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
723	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
724	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
725	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
726	} while (0)
727	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
728	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
729	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
730	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
731	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
732	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
733	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
734	} while (0)
735	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
736	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
737	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
738	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
739	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
740	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
741	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
742	} while (0)
743	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
744	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
745	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
747	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
749	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
750	} while (0)
751	#define IEM_MC_STORE_YREG_YMM_ZX_VLMAX(a_iYRegDst, a_uYmmSrc) \
752	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
753	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[0]; \
754	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[1]; \
755	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[2]; \
756	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[3]; \
757	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
758	} while (0)
759	#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
760	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
761	if ((a_iDwDst) < 4) \
762	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
763	else \
764	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
765	} while (0)
766	#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
767	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
768	if ((a_iQwDst) < 2) \
769	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
770	else \
771	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
772	} while (0)
773	#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
774	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
775	if ((a_iQword) < 2) \
776	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
777	else \
778	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
779	} while (0)
780
781	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
782	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
783	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
784	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
785	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
790	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
791	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
792	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
793	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
798	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
799	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
800	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
801	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
802	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
803	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
804	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
809	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
810	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
811	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
812	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
813	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
814	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
815	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
816	} while (0)
817	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
818	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
819	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
820	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
821	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
822	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
823	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
824	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
825	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
826	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
827	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
828	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
829	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
830	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
831	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
832	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
833	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
834	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
835	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
836	} while (0)
837	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
838	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
839	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
840	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
841	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
842	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
843	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
844	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
845	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
846	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
847	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
848	} while (0)
849	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
850	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
851	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
852	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
853	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
854	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
855	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
856	} while (0)
857	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
858	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
859	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
860	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
861	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
862	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
863	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
864	} while (0)
865
866	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
867	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
868	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
869	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
870	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
871	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
872	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
873	do { uintptr_t const iYRegTmp = (a_iYReg); \
874	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
875	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
876	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
877	} while (0)
878
879	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
880	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
881	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
882	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
883	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
884	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
885	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
886	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
887	} while (0)
888	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
889	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
890	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
891	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
892	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
893	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
894	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
895	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
896	} while (0)
897	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
898	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
899	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
900	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
901	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
902	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
903	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
904	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
905	} while (0)
906
907	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
908	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
909	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
910	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
911	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
912	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
913	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
914	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
915	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
916	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
917	} while (0)
918	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
919	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
920	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
921	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
922	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
923	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
924	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
925	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
926	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
927	} while (0)
928	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovlhps */ \
929	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
930	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
931	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
932	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
933	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
934	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
935	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
936	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
937	} while (0)
938	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
939	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
940	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
941	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
942	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
943	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
944	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
945	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
946	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
947	} while (0)
948	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
949	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
950	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
951	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
952	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
953	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
954	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
955	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
956	} while (0)
957	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
958	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
959	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
960	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
961	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
962	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
963	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
964	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
965	} while (0)
966
967	#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
968	do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
969
970	#ifndef IEM_WITH_SETJMP
971	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
972	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
973	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
974	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
975	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
976	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
977	#else
978	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
979	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
980	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
981	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
982	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
983	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
984
985	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
986	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
987	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
988	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
989	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
990	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
991	#endif
992
993	#ifndef IEM_WITH_SETJMP
994	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
995	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
996	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
997	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
998	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
999	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
1000	# define IEM_MC_FETCH_MEM_I16_DISP(a_i16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1001	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1002	#else
1003	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1004	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1005	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1006	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1007	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1008	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1009	# define IEM_MC_FETCH_MEM_I16_DISP(a_i16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1010	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1011
1012	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
1013	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1014	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
1015	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1016	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
1017	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1018	# define IEM_MC_FETCH_MEM_FLAT_I16_DISP(a_i16Dst, a_GCPtrMem, a_offDisp) \
1019	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1020	#endif
1021
1022	#ifndef IEM_WITH_SETJMP
1023	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1024	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
1025	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1026	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1027	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1028	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
1029	# define IEM_MC_FETCH_MEM_I32_DISP(a_i32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1030	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1031	#else
1032	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1033	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1034	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1035	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1036	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1037	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1038	# define IEM_MC_FETCH_MEM_I32_DISP(a_i32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1039	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1040
1041	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
1042	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1043	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
1044	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1045	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
1046	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1047	# define IEM_MC_FETCH_MEM_FLAT_I32_DISP(a_i32Dst, a_GCPtrMem, a_offDisp) \
1048	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1049	#endif
1050
1051	#ifndef IEM_WITH_SETJMP
1052	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1053	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1054	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1055	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1056	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1057	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1058	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1059	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
1060	#else
1061	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1062	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1063	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1064	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1065	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1066	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1067	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1068	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1069
1070	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1071	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1072	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1073	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1074	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1075	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1076	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1077	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1078	#endif
1079
1080	#ifndef IEM_WITH_SETJMP
1081	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1082	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1083	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1084	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1085	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1086	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1087	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1088	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1089	#else
1090	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1091	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1092	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1093	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1094	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1095	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1096	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1097	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1098
1099	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1100	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1101	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1102	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1103	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1104	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1105	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1106	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1107	#endif
1108
1109	#ifndef IEM_WITH_SETJMP
1110	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1111	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1112	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1113	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1114	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1115	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1116
1117	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1118	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1119	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1120	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1121
1122	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1123	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1124	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1125	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1126	} while (0)
1127
1128	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1129	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1130	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1131	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1132	} while (0)
1133
1134	# define IEM_MC_FETCH_MEM_XMM_NO_AC_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1135	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1136	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1137	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1138	} while (0)
1139
1140	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1141	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1142	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1143	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1144	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1145	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1146	} while (0)
1147
1148	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1149	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1150	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1151	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1152	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1153	} while (0)
1154
1155	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1156	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1157	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1158	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1159	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1160	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1161	} while (0)
1162	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1163	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1164	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1165	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1166	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1167	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1168	} while (0)
1169
1170	#else
1171	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1172	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1173	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1174	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1175	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1176	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1177
1178	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1179	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1180	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1181	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1182	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1183	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1184
1185	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1186	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1187	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1188	iemMemFlatFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1189	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1190	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1191
1192	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1193	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1194	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1195	iemMemFlatFetchDataU128NoAcJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1196	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1197	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1198
1199	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1200	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1201	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1202	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1203	} while (0)
1204	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1205	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1206	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1207	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1208	} while (0)
1209
1210	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1211	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1212	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1213	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1214	} while (0)
1215
1216	# define IEM_MC_FETCH_MEM_XMM_NO_AC_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1217	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1218	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1219	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1220	} while (0)
1221
1222	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1223	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1224	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1225	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1226	} while (0)
1227
1228	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1229	iemMemFlatFetchDataU128NoAcJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1230	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1231	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1232	} while (0)
1233
1234	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1235	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1236	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1237	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1238	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1239	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1240	} while (0)
1241	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1242	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1243	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1244	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1245	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1246	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1247	} while (0)
1248
1249	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1250	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1251	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1252	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1253	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1254	} while (0)
1255	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1256	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1257	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1258	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1259	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1260	} while (0)
1261
1262
1263	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1264	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1265	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1266	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1267	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1268	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1269	} while (0)
1270	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1271	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1272	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1273	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1274	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1275	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1276	} while (0)
1277
1278	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1279	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1280	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1281	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1282	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1283	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1284	} while (0)
1285	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1286	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1287	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1288	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1289	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1290	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1291	} while (0)
1292
1293	#endif
1294
1295	#ifndef IEM_WITH_SETJMP
1296	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1297	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1298	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1299	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1300	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1301	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1302
1303	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1304	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1305	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1306	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1307	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1308	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1309
1310	# define IEM_MC_FETCH_MEM_YMM_NO_AC_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1311	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1312	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2))); \
1313	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1314	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1315	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1316	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1317	} while (0)
1318
1319	#else
1320	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1321	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1322	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1323	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1324	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1325	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1326
1327	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1328	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1329	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1330	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1331	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1332	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1333
1334	# define IEM_MC_FETCH_MEM_YMM_NO_AC_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1335	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1336	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2)); \
1337	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1338	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1339	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1340	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1341	} while (0)
1342
1343	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1344	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1345	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1346	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1347	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1348	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1349
1350	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1351	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1352	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1353	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1354	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1355	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1356
1357	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_GCPtrMem2) do { \
1358	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1359	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_GCPtrMem2)); \
1360	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1361	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1362	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1363	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1364	} while (0)
1365
1366	#endif
1367
1368
1369
1370	#ifndef IEM_WITH_SETJMP
1371	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1372	do { \
1373	uint8_t u8Tmp; \
1374	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1375	(a_u16Dst) = u8Tmp; \
1376	} while (0)
1377	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1378	do { \
1379	uint8_t u8Tmp; \
1380	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1381	(a_u32Dst) = u8Tmp; \
1382	} while (0)
1383	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1384	do { \
1385	uint8_t u8Tmp; \
1386	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1387	(a_u64Dst) = u8Tmp; \
1388	} while (0)
1389	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1390	do { \
1391	uint16_t u16Tmp; \
1392	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1393	(a_u32Dst) = u16Tmp; \
1394	} while (0)
1395	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1396	do { \
1397	uint16_t u16Tmp; \
1398	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1399	(a_u64Dst) = u16Tmp; \
1400	} while (0)
1401	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1402	do { \
1403	uint32_t u32Tmp; \
1404	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1405	(a_u64Dst) = u32Tmp; \
1406	} while (0)
1407	#else /* IEM_WITH_SETJMP */
1408	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1409	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1410	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1411	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1412	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1413	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1414	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1415	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1416	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1417	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1418	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1419	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1420
1421	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1422	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1423	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1424	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1425	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1426	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1427	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1428	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1429	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1430	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1431	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1432	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1433	#endif /* IEM_WITH_SETJMP */
1434
1435	#ifndef IEM_WITH_SETJMP
1436	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1437	do { \
1438	uint8_t u8Tmp; \
1439	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1440	(a_u16Dst) = (int8_t)u8Tmp; \
1441	} while (0)
1442	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1443	do { \
1444	uint8_t u8Tmp; \
1445	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1446	(a_u32Dst) = (int8_t)u8Tmp; \
1447	} while (0)
1448	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1449	do { \
1450	uint8_t u8Tmp; \
1451	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1452	(a_u64Dst) = (int8_t)u8Tmp; \
1453	} while (0)
1454	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1455	do { \
1456	uint16_t u16Tmp; \
1457	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1458	(a_u32Dst) = (int16_t)u16Tmp; \
1459	} while (0)
1460	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1461	do { \
1462	uint16_t u16Tmp; \
1463	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1464	(a_u64Dst) = (int16_t)u16Tmp; \
1465	} while (0)
1466	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1467	do { \
1468	uint32_t u32Tmp; \
1469	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1470	(a_u64Dst) = (int32_t)u32Tmp; \
1471	} while (0)
1472	#else /* IEM_WITH_SETJMP */
1473	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1474	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1475	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1476	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1477	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1478	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1479	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1480	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1481	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1482	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1483	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1484	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1485
1486	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1487	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1488	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1489	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1490	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1491	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1492	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1493	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1494	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1495	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1496	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1497	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1498	#endif /* IEM_WITH_SETJMP */
1499
1500	#ifndef IEM_WITH_SETJMP
1501	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1502	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1503	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1504	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1505	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1506	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1507	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1508	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1509	#else
1510	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1511	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1512	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1513	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1514	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1515	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1516	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1517	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1518
1519	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1520	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1521	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1522	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1523	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1524	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1525	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1526	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1527	#endif
1528
1529	#ifndef IEM_WITH_SETJMP
1530	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1531	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1532	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1533	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1534	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1535	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1536	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1537	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1538	#else
1539	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1540	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1541	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1542	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1543	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1544	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1545	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1546	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1547
1548	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1549	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1550	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1551	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1552	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1553	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1554	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1555	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1556	#endif
1557
1558	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1559	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1560	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1561	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1562	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1563	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1564	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1565	do { \
1566	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1567	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1568	} while (0)
1569	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1570	do { \
1571	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1572	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1573	} while (0)
1574
1575	#ifndef IEM_WITH_SETJMP
1576	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1577	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1578	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1579	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1580	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1581	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1582	#else
1583	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1584	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1585	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1586	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1587	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1588	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1589
1590	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1591	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1592	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1593	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1594	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1595	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1596	#endif
1597
1598	#ifndef IEM_WITH_SETJMP
1599	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1600	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1601	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1602	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1603	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1604	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1605	#else
1606	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1607	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1608	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1609	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1610	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1611	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1612
1613	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1614	iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1615	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1616	iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1617	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1618	iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1619	#endif
1620
1621	/* Regular stack push and pop: */
1622	#ifndef IEM_WITH_SETJMP
1623	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1624	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1625	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1626	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1627
1628	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1629	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1630	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1631	#else
1632	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1633	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1634	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1635	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1636
1637	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1638	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1639	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1640	#endif
1641
1642	/* 32-bit flat stack push and pop: */
1643	#ifndef IEM_WITH_SETJMP
1644	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1645	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1646	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1647
1648	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1649	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1650	#else
1651	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1652	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1653	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1654
1655	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1656	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1657	#endif
1658
1659	/* 64-bit flat stack push and pop: */
1660	#ifndef IEM_WITH_SETJMP
1661	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1662	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1663
1664	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1665	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1666	#else
1667	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1668	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1669
1670	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1671	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1672	#endif
1673
1674
1675	/* 8-bit */
1676
1677	/**
1678	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1679	* acccess, for atomic operations.
1680	*
1681	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1682	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1683	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1684	* @param[in] a_GCPtrMem The memory address.
1685	* @remarks Will return/long jump on errors.
1686	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1687	*/
1688	#ifndef IEM_WITH_SETJMP
1689	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1690	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1691	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1692	#else
1693	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1694	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1695	#endif
1696
1697	/**
1698	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1699	*
1700	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1701	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1702	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1703	* @param[in] a_GCPtrMem The memory address.
1704	* @remarks Will return/long jump on errors.
1705	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1706	*/
1707	#ifndef IEM_WITH_SETJMP
1708	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1709	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1710	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1711	#else
1712	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1713	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1714	#endif
1715
1716	/**
1717	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1718	*
1719	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1720	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1721	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1722	* @param[in] a_GCPtrMem The memory address.
1723	* @remarks Will return/long jump on errors.
1724	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1725	*/
1726	#ifndef IEM_WITH_SETJMP
1727	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1728	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1729	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1730	#else
1731	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1732	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1733	#endif
1734
1735	/**
1736	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1737	*
1738	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1739	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1740	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1741	* @param[in] a_GCPtrMem The memory address.
1742	* @remarks Will return/long jump on errors.
1743	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1744	*/
1745	#ifndef IEM_WITH_SETJMP
1746	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1747	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1748	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1749	#else
1750	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1751	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1752	#endif
1753
1754	/**
1755	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1756	* acccess, flat address variant.
1757	*
1758	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1759	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1760	* @param[in] a_GCPtrMem The memory address.
1761	* @remarks Will return/long jump on errors.
1762	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1763	*/
1764	#ifndef IEM_WITH_SETJMP
1765	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1766	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1767	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1768	#else
1769	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1770	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1771	#endif
1772
1773	/**
1774	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1775	* address variant.
1776	*
1777	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1778	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1779	* @param[in] a_GCPtrMem The memory address.
1780	* @remarks Will return/long jump on errors.
1781	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1782	*/
1783	#ifndef IEM_WITH_SETJMP
1784	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1785	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1786	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1787	#else
1788	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1789	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1790	#endif
1791
1792	/**
1793	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1794	* address variant.
1795	*
1796	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1797	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1798	* @param[in] a_GCPtrMem The memory address.
1799	* @remarks Will return/long jump on errors.
1800	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1801	*/
1802	#ifndef IEM_WITH_SETJMP
1803	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1804	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1805	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1806	#else
1807	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1808	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1809	#endif
1810
1811	/**
1812	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1813	* address variant.
1814	*
1815	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1816	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1817	* @param[in] a_GCPtrMem The memory address.
1818	* @remarks Will return/long jump on errors.
1819	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1820	*/
1821	#ifndef IEM_WITH_SETJMP
1822	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1823	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1824	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1825	#else
1826	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1827	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1828	#endif
1829
1830
1831	/* 16-bit */
1832
1833	/**
1834	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1835	*
1836	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1837	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1838	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1839	* @param[in] a_GCPtrMem The memory address.
1840	* @remarks Will return/long jump on errors.
1841	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1842	*/
1843	#ifndef IEM_WITH_SETJMP
1844	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1845	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1846	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1847	#else
1848	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1849	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1850	#endif
1851
1852	/**
1853	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1854	*
1855	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1856	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1857	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1858	* @param[in] a_GCPtrMem The memory address.
1859	* @remarks Will return/long jump on errors.
1860	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1861	*/
1862	#ifndef IEM_WITH_SETJMP
1863	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1864	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1865	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1866	#else
1867	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1868	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1869	#endif
1870
1871	/**
1872	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1873	*
1874	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1875	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1876	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1877	* @param[in] a_GCPtrMem The memory address.
1878	* @remarks Will return/long jump on errors.
1879	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1880	*/
1881	#ifndef IEM_WITH_SETJMP
1882	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1883	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1884	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1885	#else
1886	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1887	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1888	#endif
1889
1890	/**
1891	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1892	*
1893	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1894	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1895	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1896	* @param[in] a_GCPtrMem The memory address.
1897	* @remarks Will return/long jump on errors.
1898	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1899	*/
1900	#ifndef IEM_WITH_SETJMP
1901	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1902	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1903	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1904	#else
1905	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1906	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1907	#endif
1908
1909	/**
1910	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1911	* acccess, flat address variant.
1912	*
1913	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1914	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1915	* @param[in] a_GCPtrMem The memory address.
1916	* @remarks Will return/long jump on errors.
1917	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1918	*/
1919	#ifndef IEM_WITH_SETJMP
1920	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1921	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1922	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1923	#else
1924	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1925	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1926	#endif
1927
1928	/**
1929	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1930	* address variant.
1931	*
1932	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1933	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1934	* @param[in] a_GCPtrMem The memory address.
1935	* @remarks Will return/long jump on errors.
1936	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1937	*/
1938	#ifndef IEM_WITH_SETJMP
1939	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1940	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1941	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1942	#else
1943	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1944	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1945	#endif
1946
1947	/**
1948	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1949	* address variant.
1950	*
1951	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1952	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1953	* @param[in] a_GCPtrMem The memory address.
1954	* @remarks Will return/long jump on errors.
1955	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1956	*/
1957	#ifndef IEM_WITH_SETJMP
1958	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1959	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1960	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1961	#else
1962	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1963	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1964	#endif
1965
1966	/**
1967	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1968	* address variant.
1969	*
1970	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1971	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1972	* @param[in] a_GCPtrMem The memory address.
1973	* @remarks Will return/long jump on errors.
1974	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1975	*/
1976	#ifndef IEM_WITH_SETJMP
1977	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1978	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1979	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1980	#else
1981	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1982	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1983	#endif
1984
1985	/** int16_t alias. */
1986	#ifndef IEM_WITH_SETJMP
1987	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1988	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1989	#else
1990	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1991	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1992	#endif
1993
1994	/** Flat int16_t alias. */
1995	#ifndef IEM_WITH_SETJMP
1996	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1997	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1998	#else
1999	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
2000	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2001	#endif
2002
2003
2004	/* 32-bit */
2005
2006	/**
2007	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
2008	*
2009	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2010	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2011	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2012	* @param[in] a_GCPtrMem The memory address.
2013	* @remarks Will return/long jump on errors.
2014	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2015	*/
2016	#ifndef IEM_WITH_SETJMP
2017	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2018	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2019	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2020	#else
2021	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2022	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2023	#endif
2024
2025	/**
2026	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
2027	*
2028	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2029	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2030	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2031	* @param[in] a_GCPtrMem The memory address.
2032	* @remarks Will return/long jump on errors.
2033	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2034	*/
2035	#ifndef IEM_WITH_SETJMP
2036	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2037	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2038	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2039	#else
2040	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2041	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2042	#endif
2043
2044	/**
2045	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
2046	*
2047	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2048	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2049	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2050	* @param[in] a_GCPtrMem The memory address.
2051	* @remarks Will return/long jump on errors.
2052	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2053	*/
2054	#ifndef IEM_WITH_SETJMP
2055	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2056	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2057	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2058	#else
2059	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2060	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2061	#endif
2062
2063	/**
2064	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
2065	*
2066	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2067	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2068	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2069	* @param[in] a_GCPtrMem The memory address.
2070	* @remarks Will return/long jump on errors.
2071	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2072	*/
2073	#ifndef IEM_WITH_SETJMP
2074	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2075	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2076	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2077	#else
2078	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2079	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2080	#endif
2081
2082	/**
2083	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
2084	* acccess, flat address variant.
2085	*
2086	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2087	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2088	* @param[in] a_GCPtrMem The memory address.
2089	* @remarks Will return/long jump on errors.
2090	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2091	*/
2092	#ifndef IEM_WITH_SETJMP
2093	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2094	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2095	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2096	#else
2097	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2098	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2099	#endif
2100
2101	/**
2102	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2103	* flat address variant.
2104	*
2105	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2106	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2107	* @param[in] a_GCPtrMem The memory address.
2108	* @remarks Will return/long jump on errors.
2109	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2110	*/
2111	#ifndef IEM_WITH_SETJMP
2112	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2113	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2114	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2115	#else
2116	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2117	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2118	#endif
2119
2120	/**
2121	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2122	* address variant.
2123	*
2124	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2125	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2126	* @param[in] a_GCPtrMem The memory address.
2127	* @remarks Will return/long jump on errors.
2128	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2129	*/
2130	#ifndef IEM_WITH_SETJMP
2131	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2132	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2133	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2134	#else
2135	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2136	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2137	#endif
2138
2139	/**
2140	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2141	* address variant.
2142	*
2143	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2144	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2145	* @param[in] a_GCPtrMem The memory address.
2146	* @remarks Will return/long jump on errors.
2147	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2148	*/
2149	#ifndef IEM_WITH_SETJMP
2150	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2151	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2152	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2153	#else
2154	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2155	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2156	#endif
2157
2158	/** int32_t alias. */
2159	#ifndef IEM_WITH_SETJMP
2160	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2161	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2162	#else
2163	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2164	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2165	#endif
2166
2167	/** Flat int32_t alias. */
2168	#ifndef IEM_WITH_SETJMP
2169	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2170	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2171	#else
2172	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2173	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2174	#endif
2175
2176	/** RTFLOAT32U alias. */
2177	#ifndef IEM_WITH_SETJMP
2178	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2179	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2180	#else
2181	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2182	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2183	#endif
2184
2185	/** Flat RTFLOAT32U alias. */
2186	#ifndef IEM_WITH_SETJMP
2187	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2188	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2189	#else
2190	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2191	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2192	#endif
2193
2194
2195	/* 64-bit */
2196
2197	/**
2198	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2199	*
2200	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2201	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2202	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2203	* @param[in] a_GCPtrMem The memory address.
2204	* @remarks Will return/long jump on errors.
2205	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2206	*/
2207	#ifndef IEM_WITH_SETJMP
2208	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2209	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2210	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2211	#else
2212	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2213	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2214	#endif
2215
2216	/**
2217	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2218	*
2219	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2220	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2221	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2222	* @param[in] a_GCPtrMem The memory address.
2223	* @remarks Will return/long jump on errors.
2224	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2225	*/
2226	#ifndef IEM_WITH_SETJMP
2227	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2228	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2229	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2230	#else
2231	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2232	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2233	#endif
2234
2235	/**
2236	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2237	*
2238	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2239	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2240	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2241	* @param[in] a_GCPtrMem The memory address.
2242	* @remarks Will return/long jump on errors.
2243	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2244	*/
2245	#ifndef IEM_WITH_SETJMP
2246	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2247	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2248	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2249	#else
2250	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2251	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2252	#endif
2253
2254	/**
2255	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2256	*
2257	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2258	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2259	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2260	* @param[in] a_GCPtrMem The memory address.
2261	* @remarks Will return/long jump on errors.
2262	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2263	*/
2264	#ifndef IEM_WITH_SETJMP
2265	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2266	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2267	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2268	#else
2269	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2270	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2271	#endif
2272
2273	/**
2274	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2275	* acccess, flat address variant.
2276	*
2277	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2278	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2279	* @param[in] a_GCPtrMem The memory address.
2280	* @remarks Will return/long jump on errors.
2281	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2282	*/
2283	#ifndef IEM_WITH_SETJMP
2284	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2285	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2286	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2287	#else
2288	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2289	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2290	#endif
2291
2292	/**
2293	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2294	* flat address variant.
2295	*
2296	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2297	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2298	* @param[in] a_GCPtrMem The memory address.
2299	* @remarks Will return/long jump on errors.
2300	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2301	*/
2302	#ifndef IEM_WITH_SETJMP
2303	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2304	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2305	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2306	#else
2307	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2308	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2309	#endif
2310
2311	/**
2312	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2313	* address variant.
2314	*
2315	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2316	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2317	* @param[in] a_GCPtrMem The memory address.
2318	* @remarks Will return/long jump on errors.
2319	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2320	*/
2321	#ifndef IEM_WITH_SETJMP
2322	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2323	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2324	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2325	#else
2326	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2327	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2328	#endif
2329
2330	/**
2331	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2332	* address variant.
2333	*
2334	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2335	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2336	* @param[in] a_GCPtrMem The memory address.
2337	* @remarks Will return/long jump on errors.
2338	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2339	*/
2340	#ifndef IEM_WITH_SETJMP
2341	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2342	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2343	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2344	#else
2345	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2346	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2347	#endif
2348
2349	/** int64_t alias. */
2350	#ifndef IEM_WITH_SETJMP
2351	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2352	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2353	#else
2354	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2355	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2356	#endif
2357
2358	/** Flat int64_t alias. */
2359	#ifndef IEM_WITH_SETJMP
2360	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2361	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2362	#else
2363	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2364	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2365	#endif
2366
2367	/** RTFLOAT64U alias. */
2368	#ifndef IEM_WITH_SETJMP
2369	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2370	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2371	#else
2372	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2373	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2374	#endif
2375
2376	/** Flat RTFLOAT64U alias. */
2377	#ifndef IEM_WITH_SETJMP
2378	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2379	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2380	#else
2381	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2382	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2383	#endif
2384
2385
2386	/* 128-bit */
2387
2388	/**
2389	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2390	*
2391	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2392	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2393	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2394	* @param[in] a_GCPtrMem The memory address.
2395	* @remarks Will return/long jump on errors.
2396	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2397	*/
2398	#ifndef IEM_WITH_SETJMP
2399	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2400	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2401	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2402	#else
2403	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2404	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2405	#endif
2406
2407	/**
2408	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2409	*
2410	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2411	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2412	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2413	* @param[in] a_GCPtrMem The memory address.
2414	* @remarks Will return/long jump on errors.
2415	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2416	*/
2417	#ifndef IEM_WITH_SETJMP
2418	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2419	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2420	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2421	#else
2422	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2423	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2424	#endif
2425
2426	/**
2427	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2428	*
2429	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2430	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2431	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2432	* @param[in] a_GCPtrMem The memory address.
2433	* @remarks Will return/long jump on errors.
2434	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2435	*/
2436	#ifndef IEM_WITH_SETJMP
2437	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2438	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2439	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2440	#else
2441	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2442	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2443	#endif
2444
2445	/**
2446	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2447	*
2448	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2449	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2450	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2451	* @param[in] a_GCPtrMem The memory address.
2452	* @remarks Will return/long jump on errors.
2453	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2454	*/
2455	#ifndef IEM_WITH_SETJMP
2456	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2457	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2458	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2459	#else
2460	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2461	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2462	#endif
2463
2464	/**
2465	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2466	* access, flat address variant.
2467	*
2468	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2469	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2470	* @param[in] a_GCPtrMem The memory address.
2471	* @remarks Will return/long jump on errors.
2472	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2473	*/
2474	#ifndef IEM_WITH_SETJMP
2475	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2476	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2477	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2478	#else
2479	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2480	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2481	#endif
2482
2483	/**
2484	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2485	* flat address variant.
2486	*
2487	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2488	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2489	* @param[in] a_GCPtrMem The memory address.
2490	* @remarks Will return/long jump on errors.
2491	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2492	*/
2493	#ifndef IEM_WITH_SETJMP
2494	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2495	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2496	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2497	#else
2498	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2499	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2500	#endif
2501
2502	/**
2503	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2504	* flat address variant.
2505	*
2506	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2507	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2508	* @param[in] a_GCPtrMem The memory address.
2509	* @remarks Will return/long jump on errors.
2510	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2511	*/
2512	#ifndef IEM_WITH_SETJMP
2513	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2514	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2515	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2516	#else
2517	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2518	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2519	#endif
2520
2521	/**
2522	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2523	* address variant.
2524	*
2525	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2526	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2527	* @param[in] a_GCPtrMem The memory address.
2528	* @remarks Will return/long jump on errors.
2529	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2530	*/
2531	#ifndef IEM_WITH_SETJMP
2532	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2533	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2534	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2535	#else
2536	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2537	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2538	#endif
2539
2540
2541	/* misc */
2542
2543	/**
2544	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2545	*
2546	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2547	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2548	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2549	* @param[in] a_GCPtrMem The memory address.
2550	* @remarks Will return/long jump on errors.
2551	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2552	*/
2553	#ifndef IEM_WITH_SETJMP
2554	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2555	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2556	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2557	#else
2558	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2559	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2560	#endif
2561
2562	/**
2563	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2564	*
2565	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2566	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2567	* @param[in] a_GCPtrMem The memory address.
2568	* @remarks Will return/long jump on errors.
2569	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2570	*/
2571	#ifndef IEM_WITH_SETJMP
2572	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2573	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2574	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2575	#else
2576	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2577	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2578	#endif
2579
2580
2581	/**
2582	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2583	*
2584	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2585	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2586	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2587	* @param[in] a_GCPtrMem The memory address.
2588	* @remarks Will return/long jump on errors.
2589	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2590	*/
2591	#ifndef IEM_WITH_SETJMP
2592	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2593	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2594	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2595	#else
2596	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2597	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2598	#endif
2599
2600	/**
2601	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2602	*
2603	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2604	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2605	* @param[in] a_GCPtrMem The memory address.
2606	* @remarks Will return/long jump on errors.
2607	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2608	*/
2609	#ifndef IEM_WITH_SETJMP
2610	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2611	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2612	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2613	#else
2614	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2615	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2616	#endif
2617
2618
2619
2620	/* commit + unmap */
2621
2622	/** Commits the memory and unmaps guest memory previously mapped RW.
2623	* @remarks May return.
2624	* @note Implictly frees the a_bMapInfo variable.
2625	*/
2626	#ifndef IEM_WITH_SETJMP
2627	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2628	#else
2629	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2630	#endif
2631
2632	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2633	* @remarks May return.
2634	* @note Implictly frees the a_bMapInfo variable.
2635	*/
2636	#ifndef IEM_WITH_SETJMP
2637	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2638	#else
2639	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2640	#endif
2641
2642	/** Commits the memory and unmaps guest memory previously mapped W.
2643	* @remarks May return.
2644	* @note Implictly frees the a_bMapInfo variable.
2645	*/
2646	#ifndef IEM_WITH_SETJMP
2647	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2648	#else
2649	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2650	#endif
2651
2652	/** Commits the memory and unmaps guest memory previously mapped R.
2653	* @remarks May return.
2654	* @note Implictly frees the a_bMapInfo variable.
2655	*/
2656	#ifndef IEM_WITH_SETJMP
2657	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2658	#else
2659	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2660	#endif
2661
2662
2663	/** Commits the memory and unmaps the guest memory unless the FPU status word
2664	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2665	* that would cause FLD not to store.
2666	*
2667	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2668	* store, while \#P will not.
2669	*
2670	* @remarks May in theory return - for now.
2671	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2672	*/
2673	#ifndef IEM_WITH_SETJMP
2674	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2675	if ( !(a_u16FSW & X86_FSW_ES) \
2676	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2677	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2678	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2679	else \
2680	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2681	} while (0)
2682	#else
2683	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2684	if ( !(a_u16FSW & X86_FSW_ES) \
2685	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2686	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2687	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2688	else \
2689	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2690	} while (0)
2691	#endif
2692
2693	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2694	* @note Implictly frees the a_bMapInfo variable. */
2695	#ifndef IEM_WITH_SETJMP
2696	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2697	#else
2698	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2699	#endif
2700
2701
2702
2703	/** Calculate efficient address from R/M. */
2704	#ifndef IEM_WITH_SETJMP
2705	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2706	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2707	#else
2708	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2709	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2710	#endif
2711
2712
2713	/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2714	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2715	#define IEM_MC_NATIVE_ELSE() } else {
2716	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2717
2718	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2719	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2720	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2721	#define IEM_MC_NATIVE_EMIT_2_EX(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2722	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2723	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2724	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2725	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2726	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2727	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2728
2729	/** This can be used to direct the register allocator when dealing with
2730	* x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2731	#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2732
2733
2734	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2735	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2736	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2737	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2738	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2739	#define IEM_MC_CALL_AIMPL_3(a_rcType, a_rc, a_pfn, a0, a1, a2) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2))
2740	#define IEM_MC_CALL_AIMPL_4(a_rcType, a_rc, a_pfn, a0, a1, a2, a3) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2), (a3))
2741
2742
2743	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2744	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2745	*/
2746	#ifdef VBOX_STRICT
2747	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2748	do { \
2749	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2750	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2751	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2752	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2753	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2754	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2755	if (rcStrictHlp == VINF_SUCCESS) \
2756	{ \
2757	uint64_t const fRipMask = (pVCpu->iem.s.fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT ? UINT64_MAX : UINT32_MAX; \
2758	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2759	\|\| ( ((uRipBefore + cbInstr) & fRipMask) == pVCpu->cpum.GstCtx.rip \
2760	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2761	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2762	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2763	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2764	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2765	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, (uRipBefore + cbInstr) & fRipMask)); \
2766	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2767	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2768	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2769	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2770	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2771	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2772	else \
2773	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2774	== (fEflBefore & ~(X86_EFL_RF)), \
2775	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2776	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2777	{ \
2778	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2779	AssertMsg( fExecBefore == fExecRecalc \
2780	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2781	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2782	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2783	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2784	} \
2785	} \
2786	return rcStrictHlp; \
2787	} while (0)
2788	#else
2789	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2790	#endif
2791
2792	/**
2793	* Defers the rest of the instruction emulation to a C implementation routine
2794	* and returns, only taking the standard parameters.
2795	*
2796	* @param a_fFlags IEM_CIMPL_F_XXX.
2797	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2798	* in the native recompiler.
2799	* @param a_pfnCImpl The pointer to the C routine.
2800	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2801	*/
2802	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2803	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2804
2805	/**
2806	* Defers the rest of instruction emulation to a C implementation routine and
2807	* returns, taking one argument in addition to the standard ones.
2808	*
2809	* @param a_fFlags IEM_CIMPL_F_XXX.
2810	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2811	* in the native recompiler.
2812	* @param a_pfnCImpl The pointer to the C routine.
2813	* @param a0 The argument.
2814	*/
2815	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2816	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2817
2818	/**
2819	* Defers the rest of the instruction emulation to a C implementation routine
2820	* and returns, taking two arguments in addition to the standard ones.
2821	*
2822	* @param a_fFlags IEM_CIMPL_F_XXX.
2823	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2824	* in the native recompiler.
2825	* @param a_pfnCImpl The pointer to the C routine.
2826	* @param a0 The first extra argument.
2827	* @param a1 The second extra argument.
2828	*/
2829	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2830	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2831
2832	/**
2833	* Defers the rest of the instruction emulation to a C implementation routine
2834	* and returns, taking three arguments in addition to the standard ones.
2835	*
2836	* @param a_fFlags IEM_CIMPL_F_XXX.
2837	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2838	* in the native recompiler.
2839	* @param a_pfnCImpl The pointer to the C routine.
2840	* @param a0 The first extra argument.
2841	* @param a1 The second extra argument.
2842	* @param a2 The third extra argument.
2843	*/
2844	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2845	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2846
2847	/**
2848	* Defers the rest of the instruction emulation to a C implementation routine
2849	* and returns, taking four arguments in addition to the standard ones.
2850	*
2851	* @param a_fFlags IEM_CIMPL_F_XXX.
2852	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2853	* in the native recompiler.
2854	* @param a_pfnCImpl The pointer to the C routine.
2855	* @param a0 The first extra argument.
2856	* @param a1 The second extra argument.
2857	* @param a2 The third extra argument.
2858	* @param a3 The fourth extra argument.
2859	*/
2860	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2861	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2862
2863	/**
2864	* Defers the rest of the instruction emulation to a C implementation routine
2865	* and returns, taking five arguments in addition to the standard ones.
2866	*
2867	* @param a_fFlags IEM_CIMPL_F_XXX.
2868	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2869	* in the native recompiler.
2870	* @param a_pfnCImpl The pointer to the C routine.
2871	* @param a0 The first extra argument.
2872	* @param a1 The second extra argument.
2873	* @param a2 The third extra argument.
2874	* @param a3 The fourth extra argument.
2875	* @param a4 The fifth extra argument.
2876	*/
2877	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2878	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2879
2880	/**
2881	* Defers the entire instruction emulation to a C implementation routine and
2882	* returns, only taking the standard parameters.
2883	*
2884	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2885	*
2886	* @param a_fFlags IEM_CIMPL_F_XXX.
2887	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2888	* in the native recompiler.
2889	* @param a_pfnCImpl The pointer to the C routine.
2890	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2891	*/
2892	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2893	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2894
2895	/**
2896	* Defers the entire instruction emulation to a C implementation routine and
2897	* returns, taking one argument in addition to the standard ones.
2898	*
2899	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2900	*
2901	* @param a_fFlags IEM_CIMPL_F_XXX.
2902	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2903	* in the native recompiler.
2904	* @param a_pfnCImpl The pointer to the C routine.
2905	* @param a0 The argument.
2906	*/
2907	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2908	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2909
2910	/**
2911	* Defers the entire instruction emulation to a C implementation routine and
2912	* returns, taking two arguments in addition to the standard ones.
2913	*
2914	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2915	*
2916	* @param a_fFlags IEM_CIMPL_F_XXX.
2917	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2918	* in the native recompiler.
2919	* @param a_pfnCImpl The pointer to the C routine.
2920	* @param a0 The first extra argument.
2921	* @param a1 The second extra argument.
2922	*/
2923	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2924	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2925
2926	/**
2927	* Defers the entire instruction emulation to a C implementation routine and
2928	* returns, taking three arguments in addition to the standard ones.
2929	*
2930	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2931	*
2932	* @param a_fFlags IEM_CIMPL_F_XXX.
2933	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2934	* in the native recompiler.
2935	* @param a_pfnCImpl The pointer to the C routine.
2936	* @param a0 The first extra argument.
2937	* @param a1 The second extra argument.
2938	* @param a2 The third extra argument.
2939	*/
2940	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2941	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2942
2943
2944	/**
2945	* Calls a FPU assembly implementation taking one visible argument.
2946	*
2947	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2948	* @param a0 The first extra argument.
2949	*/
2950	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2951	do { \
2952	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2953	} while (0)
2954
2955	/**
2956	* Calls a FPU assembly implementation taking two visible arguments.
2957	*
2958	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2959	* @param a0 The first extra argument.
2960	* @param a1 The second extra argument.
2961	*/
2962	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2963	do { \
2964	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2965	} while (0)
2966
2967	/**
2968	* Calls a FPU assembly implementation taking three visible arguments.
2969	*
2970	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2971	* @param a0 The first extra argument.
2972	* @param a1 The second extra argument.
2973	* @param a2 The third extra argument.
2974	*/
2975	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2976	do { \
2977	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2978	} while (0)
2979
2980	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2981	do { \
2982	(a_FpuData).FSW = (a_FSW); \
2983	(a_FpuData).r80Result = *(a_pr80Value); \
2984	} while (0)
2985
2986	/** Pushes FPU result onto the stack. */
2987	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2988	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2989	/** Pushes FPU result onto the stack and sets the FPUDP. */
2990	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2991	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2992
2993	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2994	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2995	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2996
2997	/** Stores FPU result in a stack register. */
2998	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2999	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
3000	/** Stores FPU result in a stack register and pops the stack. */
3001	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
3002	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
3003	/** Stores FPU result in a stack register and sets the FPUDP. */
3004	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3005	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3006	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
3007	* stack. */
3008	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3009	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3010
3011	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
3012	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
3013	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
3014	/** Free a stack register (for FFREE and FFREEP). */
3015	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
3016	iemFpuStackFree(pVCpu, a_iStReg)
3017	/** Increment the FPU stack pointer. */
3018	#define IEM_MC_FPU_STACK_INC_TOP() \
3019	iemFpuStackIncTop(pVCpu)
3020	/** Decrement the FPU stack pointer. */
3021	#define IEM_MC_FPU_STACK_DEC_TOP() \
3022	iemFpuStackDecTop(pVCpu)
3023
3024	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
3025	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
3026	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3027	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
3028	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
3029	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3030	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
3031	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3032	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3033	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
3034	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
3035	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3036	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
3037	* stack. */
3038	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3039	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3040	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
3041	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
3042	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3043
3044	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
3045	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
3046	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
3047	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3048	* stack. */
3049	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
3050	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
3051	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3052	* FPUDS. */
3053	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3054	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3055	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3056	* FPUDS. Pops stack. */
3057	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3058	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3059	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3060	* stack twice. */
3061	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
3062	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
3063	/** Raises a FPU stack underflow exception for an instruction pushing a result
3064	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
3065	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
3066	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
3067	/** Raises a FPU stack underflow exception for an instruction pushing a result
3068	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
3069	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
3070	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
3071
3072	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3073	* FPUIP, FPUCS and FOP. */
3074	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
3075	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
3076	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3077	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
3078	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3079	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3080	/** Prepares for using the FPU state.
3081	* Ensures that we can use the host FPU in the current context (RC+R0.
3082	* Ensures the guest FPU state in the CPUMCTX is up to date. */
3083	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
3084	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
3085	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
3086	/** Actualizes the guest FPU state so it can be accessed and modified. */
3087	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
3088
3089	/** Prepares for using the SSE state.
3090	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
3091	* Ensures the guest SSE state in the CPUMCTX is up to date. */
3092	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
3093	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3094	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3095	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3096	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3097
3098	/** Prepares for using the AVX state.
3099	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3100	* Ensures the guest AVX state in the CPUMCTX is up to date.
3101	* @note This will include the AVX512 state too when support for it is added
3102	* due to the zero extending feature of VEX instruction. */
3103	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3104	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3105	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3106	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3107	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3108
3109	/**
3110	* Calls a MMX assembly implementation taking two visible arguments.
3111	*
3112	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3113	* @param a0 The first extra argument.
3114	* @param a1 The second extra argument.
3115	*/
3116	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3117	do { \
3118	IEM_MC_PREPARE_FPU_USAGE(); \
3119	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3120	} while (0)
3121
3122	/**
3123	* Calls a MMX assembly implementation taking three visible arguments.
3124	*
3125	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3126	* @param a0 The first extra argument.
3127	* @param a1 The second extra argument.
3128	* @param a2 The third extra argument.
3129	*/
3130	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3131	do { \
3132	IEM_MC_PREPARE_FPU_USAGE(); \
3133	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3134	} while (0)
3135
3136
3137	/**
3138	* Calls a SSE assembly implementation taking two visible arguments.
3139	*
3140	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3141	* @param a0 The first extra argument.
3142	* @param a1 The second extra argument.
3143	*
3144	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3145	* which is unmasked in the guest's MXCSR.
3146	*/
3147	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3148	do { \
3149	IEM_MC_PREPARE_SSE_USAGE(); \
3150	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3151	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3152	(a0), (a1)); \
3153	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3154	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3155	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3156	{ /* probable */ } \
3157	else \
3158	{ \
3159	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3160	return iemRaiseSimdFpException(pVCpu); \
3161	return iemRaiseUndefinedOpcode(pVCpu); \
3162	} \
3163	} while (0)
3164
3165	/**
3166	* Calls a SSE assembly implementation taking three visible arguments.
3167	*
3168	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3169	* @param a0 The first extra argument.
3170	* @param a1 The second extra argument.
3171	* @param a2 The third extra argument.
3172	*
3173	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3174	* which is unmasked in the guest's MXCSR.
3175	*/
3176	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3177	do { \
3178	IEM_MC_PREPARE_SSE_USAGE(); \
3179	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3180	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3181	(a0), (a1), (a2)); \
3182	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3183	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3184	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3185	{ /* probable */ } \
3186	else \
3187	{ \
3188	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3189	return iemRaiseSimdFpException(pVCpu); \
3190	return iemRaiseUndefinedOpcode(pVCpu); \
3191	} \
3192	} while (0)
3193
3194
3195	/**
3196	* Calls a AVX assembly implementation taking two visible arguments.
3197	*
3198	* There is one implicit zero'th argument, a pointer to the extended state.
3199	*
3200	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3201	* @param a0 The first extra argument.
3202	* @param a1 The second extra argument.
3203	*
3204	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3205	* which is unmasked in the guest's MXCSR.
3206	*/
3207	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3208	do { \
3209	IEM_MC_PREPARE_AVX_USAGE(); \
3210	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3211	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3212	(a0), (a1)); \
3213	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3214	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3215	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3216	{ /* probable */ } \
3217	else \
3218	{ \
3219	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3220	return iemRaiseSimdFpException(pVCpu); \
3221	return iemRaiseUndefinedOpcode(pVCpu); \
3222	} \
3223	} while (0)
3224
3225	/**
3226	* Calls a AVX assembly implementation taking three visible arguments.
3227	*
3228	* There is one implicit zero'th argument, a pointer to the extended state.
3229	*
3230	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3231	* @param a0 The first extra argument.
3232	* @param a1 The second extra argument.
3233	* @param a2 The third extra argument.
3234	*
3235	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3236	* which is unmasked in the guest's MXCSR.
3237	*/
3238	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3239	do { \
3240	IEM_MC_PREPARE_AVX_USAGE(); \
3241	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3242	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3243	(a0), (a1), (a2)); \
3244	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3245	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3246	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3247	{ /* probable */ } \
3248	else \
3249	{ \
3250	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3251	return iemRaiseSimdFpException(pVCpu); \
3252	return iemRaiseUndefinedOpcode(pVCpu); \
3253	} \
3254	} while (0)
3255
3256	/** @note Not for IOPL or IF testing. */
3257	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3258	/** @note Not for IOPL or IF testing. */
3259	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3260	/** @note Not for IOPL or IF testing. */
3261	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3262	/** @note Not for IOPL or IF testing. */
3263	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3264	/** @note Not for IOPL or IF testing. */
3265	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3266	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3267	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3268	/** @note Not for IOPL or IF testing. */
3269	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3270	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3271	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3272	/** @note Not for IOPL or IF testing. */
3273	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3274	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3275	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3276	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3277	/** @note Not for IOPL or IF testing. */
3278	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3279	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3280	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3281	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3282	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3283	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3284	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3285	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3286	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3287	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3288	/** @note Not for IOPL or IF testing. */
3289	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3290	if ( pVCpu->cpum.GstCtx.cx != 1 \
3291	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3292	/** @note Not for IOPL or IF testing. */
3293	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3294	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3295	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3296	/** @note Not for IOPL or IF testing. */
3297	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3298	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3299	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3300	/** @note Not for IOPL or IF testing. */
3301	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3302	if ( pVCpu->cpum.GstCtx.cx != 1 \
3303	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3304	/** @note Not for IOPL or IF testing. */
3305	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3306	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3307	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3308	/** @note Not for IOPL or IF testing. */
3309	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3310	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3311	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3312	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3313	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3314
3315	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3316	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3317	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3318	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3319	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3320	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3321	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3322	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3323	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3324	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3325	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3326	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3327	#define IEM_MC_IF_FCW_IM() \
3328	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3329
3330	#define IEM_MC_ELSE() } else {
3331	#define IEM_MC_ENDIF() } do {} while (0)
3332
3333
3334	/** Recompiler debugging: Flush guest register shadow copies. */
3335	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3336
3337	/** Recompiler liveness info: input GPR */
3338	#define IEM_MC_LIVENESS_GREG_INPUT(a_iGReg) ((void)0)
3339	/** Recompiler liveness info: clobbered GPR */
3340	#define IEM_MC_LIVENESS_GREG_CLOBBER(a_iGReg) ((void)0)
3341	/** Recompiler liveness info: modified GPR register (i.e. input & output) */
3342	#define IEM_MC_LIVENESS_GREG_MODIFY(a_iGReg) ((void)0)
3343
3344	/** Recompiler liveness info: input MM register */
3345	#define IEM_MC_LIVENESS_MREG_INPUT(a_iMReg) ((void)0)
3346	/** Recompiler liveness info: clobbered MM register */
3347	#define IEM_MC_LIVENESS_MREG_CLOBBER(a_iMReg) ((void)0)
3348	/** Recompiler liveness info: modified MM register (i.e. input & output) */
3349	#define IEM_MC_LIVENESS_MREG_MODIFY(a_iMReg) ((void)0)
3350
3351	/** Recompiler liveness info: input SSE register */
3352	#define IEM_MC_LIVENESS_XREG_INPUT(a_iXReg) ((void)0)
3353	/** Recompiler liveness info: clobbered SSE register */
3354	#define IEM_MC_LIVENESS_XREG_CLOBBER(a_iXReg) ((void)0)
3355	/** Recompiler liveness info: modified SSE register (i.e. input & output) */
3356	#define IEM_MC_LIVENESS_XREG_MODIFY(a_iXReg) ((void)0)
3357
3358	/** Recompiler liveness info: input MXCSR */
3359	#define IEM_MC_LIVENESS_MXCSR_INPUT() ((void)0)
3360	/** Recompiler liveness info: clobbered MXCSR */
3361	#define IEM_MC_LIVENESS_MXCSR_CLOBBER() ((void)0)
3362	/** Recompiler liveness info: modified MXCSR (i.e. input & output) */
3363	#define IEM_MC_LIVENESS_MXCSR_MODIFY() ((void)0)
3364
3365	/** @todo add more as needed. */
3366
3367	/** @} */
3368
3369	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3370

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 107218

Download in other formats: