IEMMc.h@ 105316

Last change on this file since 105316 was 105315, checked in by vboxsync, 5 months ago
VMM/IEM: Implement vcvttss2si, vcvttsd2si, vcvtss2si and vcvtsd2si instruction emulations, bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 177.5 KB

Line
1	/* $Id: IEMMc.h 105315 2024-07-12 17:38:53Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
85	* @note only usable in 16-bit op size mode. */
86	#define IEM_MC_REL_CALL_S16_AND_FINISH(a_i16) \
87	return iemRegRipRelativeCallS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
88	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
89	#define IEM_MC_REL_CALL_S32_AND_FINISH(a_i32) \
90	return iemRegEip32RelativeCallS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32))
91	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
92	#define IEM_MC_REL_CALL_S64_AND_FINISH(a_i64) \
93	return iemRegRip64RelativeCallS64AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i64))
94	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
95	#define IEM_MC_IND_CALL_U16_AND_FINISH(a_u16NewIP) \
96	return iemRegIp16IndirectCallU16AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16NewIP))
97	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
98	#define IEM_MC_IND_CALL_U32_AND_FINISH(a_u32NewIP) \
99	return iemRegEip32IndirectCallU32AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u32NewIP))
100	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
101	#define IEM_MC_IND_CALL_U64_AND_FINISH(a_u64NewIP) \
102	return iemRegRip64IndirectCallU64AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u64NewIP))
103
104
105	/** Fetches the near return address from the stack, sets RIP and RSP (may trigger
106	* \#GP or \#SS), finishes the instruction and returns. */
107	#define IEM_MC_RETN_AND_FINISH(a_u16Pop) \
108	return iemRegRipNearReturnAndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16Pop), pVCpu->iem.s.enmEffOpSize)
109
110
111	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
112	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
113	do { \
114	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
115	{ /* probable */ } \
116	else return iemRaiseDeviceNotAvailable(pVCpu); \
117	} while (0)
118	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
119	do { \
120	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
121	{ /* probable */ } \
122	else return iemRaiseDeviceNotAvailable(pVCpu); \
123	} while (0)
124	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
125	do { \
126	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
127	{ /* probable */ } \
128	else return iemRaiseMathFault(pVCpu); \
129	} while (0)
130	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
131	do { \
132	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
133	be reduced to a single compare branch in the more probably code path. */ \
134	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
135	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
136	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
137	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
138	{ /* probable */ } \
139	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
140	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
141	return iemRaiseUndefinedOpcode(pVCpu); \
142	else \
143	return iemRaiseDeviceNotAvailable(pVCpu); \
144	} while (0)
145	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
146	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
147	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
148	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
149	do { \
150	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
151	single compare branch in the more probable code path. */ \
152	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
153	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
154	== X86_CR4_OSFXSR)) \
155	{ /* likely */ } \
156	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
157	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
158	return iemRaiseUndefinedOpcode(pVCpu); \
159	else \
160	return iemRaiseDeviceNotAvailable(pVCpu); \
161	} while (0)
162	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
163	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
164	do { \
165	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
166	single compare branch in the more probable code path. */ \
167	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
168	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
169	{ /* probable */ } \
170	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
171	return iemRaiseUndefinedOpcode(pVCpu); \
172	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
173	return iemRaiseDeviceNotAvailable(pVCpu); \
174	else \
175	return iemRaiseMathFault(pVCpu); \
176	} while (0)
177	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
178	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
179	* count the CPL into the TB key. However it is safe enough for now, as
180	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
181	* emitted for it. */
182	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
183	do { \
184	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
185	else return iemRaiseGeneralProtectionFault0(pVCpu); \
186	} while (0)
187	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
188	do { \
189	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
190	else return iemRaiseGeneralProtectionFault0(pVCpu); \
191	} while (0)
192	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
193	do { \
194	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
195	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
196	{ /* probable */ } \
197	else return iemRaiseUndefinedOpcode(pVCpu); \
198	} while (0)
199	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
200	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
201	do { \
202	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
203	else return iemRaiseGeneralProtectionFault0(pVCpu); \
204	} while (0)
205	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
206	do { \
207	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
208	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
209	{ /* probable */ } \
210	else \
211	{ \
212	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
213	return iemRaiseSimdFpException(pVCpu); \
214	return iemRaiseUndefinedOpcode(pVCpu); \
215	} \
216	} while (0)
217
218
219	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
220	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
221	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
222	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
223	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
224	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
225	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
226	/** @note IEMAllInstPython.py duplicates the expansion. */
227	#define IEM_MC_ARG_EFLAGS(a_Name, a_iArg) uint32_t const a_Name = pVCpu->cpum.GstCtx.eflags.u
228	/** @note IEMAllInstPython.py duplicates the expansion. */
229	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
230	uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
231	uint32_t *a_pName = &a_Name
232	/** @note IEMAllInstPython.py duplicates the expansion. */
233	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
234	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
235	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
236	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
237	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
238	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
239	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
240	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
241	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
242	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
243	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
244	} while (0)
245	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
246	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
247
248	/** ASSUMES the source variable not used after this statement. */
249	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
250
251	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
252	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
253	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
254	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
255	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
256	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
257	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
258	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
259	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
260	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
261	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
262	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
263	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
264	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
265	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
266	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
267	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
268	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
269	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
270	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
271	} while(0)
272	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
273	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
274	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
275	} while(0)
276	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
277	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
278	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
279	} while (0)
280	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
281	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
282	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
283	} while (0)
284	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
285	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
286	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
287	} while (0)
288	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
289	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
290	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
291	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
292	} while (0)
293	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
294	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
295	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
296	} while (0)
297	/** @note Not for IOPL or IF testing or modification. */
298	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
299	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
300	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
301	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
302	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
303
304	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
305	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
306	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
307	#define IEM_MC_STORE_GREG_I32(a_iGReg, a_i32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_i32Value) / clear high bits. */
308	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
309	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
310	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
311	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
312	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
313	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
314	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
315	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
316	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
317	} while(0)
318	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
319	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
320	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
321	} while(0)
322	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
323
324	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
325	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
326	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
327	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
328	} while (0)
329	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
330	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
331	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
332	} while (0)
333	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
334	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
335
336
337	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
338	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
339	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
340	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
341	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
342	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
343	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
344	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
345	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
346	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
347	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
348	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
349	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
350	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
351	/** @note Not for IOPL or IF testing or modification.
352	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
353	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
354	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
355
356	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
357	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
358	do { \
359	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
360	*pu32Reg += (a_u32Value); \
361	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
362	} while (0)
363	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
364
365	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
366	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
367	do { \
368	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
369	*pu32Reg -= (a_u8Const); \
370	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
371	} while (0)
372	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
373	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
374
375	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
376	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
377	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
378	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
379	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
380	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
381	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
382
383	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
384	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
385	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
386	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
387
388	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
389	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
390	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
391
392	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
393	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
394	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
395
396	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
397	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
398	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
399
400	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
401
402	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
403	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
404	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
405
406	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
407
408	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
409
410	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
411	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
412	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
413	do { \
414	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
415	*pu32Reg &= (a_u32Value); \
416	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
417	} while (0)
418	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
419
420	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
421	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
422	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
423	do { \
424	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
425	*pu32Reg \|= (a_u32Value); \
426	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
427	} while (0)
428	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
429
430	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
431	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
432	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
433
434	/** @note Not for IOPL or IF modification. */
435	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
436	/** @note Not for IOPL or IF modification. */
437	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
438	/** @note Not for IOPL or IF modification. */
439	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
440
441	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
442
443	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
444	#define IEM_MC_FPU_TO_MMX_MODE() do { \
445	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
446	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
447	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
448	} while (0)
449
450	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
451	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
452	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
453	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
454	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
455	} while (0)
456
457	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
458	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
459	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
460	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
461	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
462	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
463	#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
464	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
465	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
466	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
467	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
468	} while (0)
469	#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
470	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
471	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
472	} while (0)
473	#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
474	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
475	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
476	} while (0)
477	#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
478	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
479	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
480	} while (0)
481	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
482	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
483	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
484	} while (0)
485	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
486	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
487	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
488	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
489	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
490	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
491	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
492	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
493	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
494	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
495
496	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
497	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
498	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
499	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
500	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
501	} while (0)
502	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
503	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
504	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
505	} while (0)
506	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
507	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
508	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
509	} while (0)
510	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
511	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
512	#define IEM_MC_FETCH_XREG_R64(a_r64Value, a_iXReg, a_iQWord) \
513	do { (a_r64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[(a_iQWord)]; } while (0)
514	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
515	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
516	#define IEM_MC_FETCH_XREG_R32(a_r32Value, a_iXReg, a_iDWord) \
517	do { (a_r32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[(a_iDWord)]; } while (0)
518	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
519	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
520	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
521	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
522	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
523	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
524	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
525	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
526	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
527	} while (0)
528	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
529	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
530	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
531	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
532	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
533	} while (0)
534	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
535	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
536	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
537	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
538	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
539	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
540	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
541	} while (0)
542	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
543	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
544	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
545	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
546	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
547	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
548	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
549	} while (0)
550	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
551	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
552	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
553	} while (0)
554	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
555	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
556	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
557	} while (0)
558	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
559	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
560	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
561	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
562	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
563	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
564	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
565	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
566	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
567	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
568	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
569	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
570
571	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
572	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
573	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
574	} while (0)
575
576	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
577	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
578	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
579	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
580	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
581	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
582	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
583	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
584	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
585	} while (0)
586
587	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
588	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
590	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
591	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
598	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
599	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
600	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
601	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
602	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
603	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
605	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
606	} while (0)
607	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
608	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
612	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
613	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
614	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
615	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
616	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
617	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
618	} while (0)
619	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
620	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
621	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
622	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
623	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
624	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
625	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
626	} while (0)
627	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
628	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
630	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
631	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
632	} while (0)
633
634	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
635	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
636	#define IEM_MC_REF_XREG_XMM(a_pXmmDst, a_iXReg) \
637	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
638	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
639	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
640	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
641	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
642	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
643	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
644	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
645	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
646	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
647	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
648	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
649	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
650	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
651	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
652	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
653	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
654	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
655	} while (0)
656
657	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
658	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
659	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
660	} while (0)
661	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
662	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
663	if ((a_iQWord) < 2) \
664	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
665	else \
666	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[(a_iQWord) - 2]; \
667	} while (0)
668	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
669	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
670	if ((a_iDQword) == 0) \
671	{ \
672	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
673	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
674	} \
675	else \
676	{ \
677	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
678	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
679	} \
680	} while (0)
681	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
682	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
683	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
684	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
685	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
686	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
687	} while (0)
688	#define IEM_MC_FETCH_YREG_YMM(a_uYmmDst, a_iYRegSrc) \
689	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
690	(a_uYmmDst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
691	(a_uYmmDst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
692	(a_uYmmDst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
693	(a_uYmmDst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
694	} while (0)
695	#define IEM_MC_FETCH_YREG_PAIR_YMM(a_uYmmDst, a_iYRegSrc1, a_iYRegSrc2) \
696	do { uintptr_t const iYRegSrc1Tmp = (a_iYRegSrc1); \
697	uintptr_t const iYRegSrc2Tmp = (a_iYRegSrc2); \
698	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[0]; \
699	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[1]; \
700	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[0]; \
701	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[1]; \
702	(a_uYmmDst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[0]; \
703	(a_uYmmDst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[1]; \
704	(a_uYmmDst).uSrc2.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[0]; \
705	(a_uYmmDst).uSrc2.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[1]; \
706	} while (0)
707
708	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
709	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
710	if ((a_iDQword) == 0) \
711	{ \
712	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
713	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
714	} \
715	else \
716	{ \
717	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
718	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
719	} \
720	} while (0)
721
722	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
723	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
724	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
725	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
726	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
727	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
728	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
729	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
730	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
731	} while (0)
732	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
733	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
734	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
735	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
736	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
737	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
738	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
739	} while (0)
740	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
741	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
742	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
743	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
744	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
745	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
746	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
747	} while (0)
748	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
749	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
750	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
751	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
752	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
753	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
754	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
755	} while (0)
756	#define IEM_MC_STORE_YREG_YMM_ZX_VLMAX(a_iYRegDst, a_uYmmSrc) \
757	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
758	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[0]; \
759	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[1]; \
760	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[2]; \
761	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[3]; \
762	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
763	} while (0)
764	#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
765	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
766	if ((a_iDwDst) < 4) \
767	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
768	else \
769	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
770	} while (0)
771	#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
772	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
773	if ((a_iQwDst) < 2) \
774	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
775	else \
776	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
777	} while (0)
778	#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
779	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
780	if ((a_iQword) < 2) \
781	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
782	else \
783	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
784	} while (0)
785
786	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
787	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
790	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
791	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
792	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
793	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
798	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
799	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
800	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
801	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
802	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
803	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
804	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
809	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
810	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
811	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
812	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
813	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
814	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
815	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
816	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
817	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
818	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
819	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
820	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
821	} while (0)
822	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
823	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
824	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
825	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
826	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
827	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
828	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
829	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
830	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
831	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
832	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
833	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
834	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
835	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
836	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
837	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
838	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
839	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
840	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
841	} while (0)
842	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
843	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
844	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
845	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
846	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
847	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
848	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
849	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
850	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
851	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
852	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
853	} while (0)
854	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
855	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
856	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
857	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
858	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
859	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
860	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
861	} while (0)
862	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
863	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
864	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
865	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
866	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
867	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
868	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
869	} while (0)
870
871	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
872	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
873	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
874	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
875	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
876	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
877	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
878	do { uintptr_t const iYRegTmp = (a_iYReg); \
879	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
880	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
881	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
882	} while (0)
883
884	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
885	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
886	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
887	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
888	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
889	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
890	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
891	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
892	} while (0)
893	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
894	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
895	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
896	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
897	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
898	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
899	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
900	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
901	} while (0)
902	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
903	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
904	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
905	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
906	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
907	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
908	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
909	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
910	} while (0)
911
912	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
913	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
914	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
915	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
916	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
917	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
918	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
919	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
920	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
921	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
922	} while (0)
923	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
924	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
925	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
926	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
927	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
928	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
929	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
930	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
931	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
932	} while (0)
933	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
934	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
935	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
936	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
937	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
938	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
939	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
940	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
941	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
942	} while (0)
943	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
944	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
945	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
946	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
947	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
948	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
949	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
950	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
951	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
952	} while (0)
953	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
954	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
955	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
956	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
957	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
958	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
959	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
960	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
961	} while (0)
962	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
963	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
964	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
965	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
966	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
967	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
968	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
969	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
970	} while (0)
971
972	#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
973	do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
974
975	#ifndef IEM_WITH_SETJMP
976	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
977	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
978	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
979	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
980	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
981	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
982	#else
983	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
984	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
985	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
986	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
987	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
988	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
989
990	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
991	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
992	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
993	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
994	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
995	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
996	#endif
997
998	#ifndef IEM_WITH_SETJMP
999	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1000	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
1001	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1002	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1003	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1004	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
1005	#else
1006	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1007	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1008	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1009	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1010	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1011	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1012
1013	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
1014	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1015	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
1016	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1017	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
1018	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1019	#endif
1020
1021	#ifndef IEM_WITH_SETJMP
1022	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1023	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
1024	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1025	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1026	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1027	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
1028	#else
1029	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1030	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1031	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1032	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1033	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1034	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1035
1036	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
1037	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1038	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
1039	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1040	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
1041	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1042	#endif
1043
1044	#ifndef IEM_WITH_SETJMP
1045	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1046	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1047	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1048	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1049	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1050	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1051	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1052	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
1053	#else
1054	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1055	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1056	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1057	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1058	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1059	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1060	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1061	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1062
1063	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1064	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1065	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1066	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1067	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1068	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1069	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1070	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1071	#endif
1072
1073	#ifndef IEM_WITH_SETJMP
1074	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1075	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1076	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1077	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1078	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1079	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1080	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1081	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1082	#else
1083	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1084	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1085	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1086	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1087	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1088	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1089	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1090	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1091
1092	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1093	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1094	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1095	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1096	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1097	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1098	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1099	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1100	#endif
1101
1102	#ifndef IEM_WITH_SETJMP
1103	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1104	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1105	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1106	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1107	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1108	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1109
1110	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1111	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1112	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1113	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1114
1115	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1116	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1117	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1118	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1119	} while (0)
1120
1121	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1122	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1123	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1124	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1125	} while (0)
1126
1127	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1128	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1129	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1130	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1131	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1132	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1133	} while (0)
1134
1135	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1136	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1137	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1138	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1139	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1140	} while (0)
1141
1142	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1143	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1144	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1145	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1146	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1147	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1148	} while (0)
1149	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1150	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1151	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1152	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1153	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1154	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1155	} while (0)
1156
1157	#else
1158	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1159	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1160	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1161	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1162	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1163	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1164
1165	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1166	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1167	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1168	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1169	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1170	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1171
1172	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1173	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1174	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1175	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1176	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1177	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1178
1179	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1180	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1181	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1182	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1183	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1184	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1185
1186	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1187	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1188	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1189	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1190	} while (0)
1191	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1192	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1193	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1194	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1195	} while (0)
1196
1197	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1198	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1199	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1200	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1201	} while (0)
1202	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1203	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1204	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1205	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1206	} while (0)
1207
1208	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1209	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1210	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1211	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1212	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1213	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1214	} while (0)
1215	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1216	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1217	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1218	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1219	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1220	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1221	} while (0)
1222
1223	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1224	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1225	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1226	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1227	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1228	} while (0)
1229	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1230	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1231	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1232	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1233	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1234	} while (0)
1235
1236
1237	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1238	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1239	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1240	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1241	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1242	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1243	} while (0)
1244	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1245	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1246	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1247	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1248	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1249	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1250	} while (0)
1251
1252	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1253	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1254	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1255	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1256	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1257	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1258	} while (0)
1259	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1260	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1261	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1262	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1263	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1264	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1265	} while (0)
1266
1267	#endif
1268
1269	#ifndef IEM_WITH_SETJMP
1270	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1271	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1272	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1273	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1274	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1275	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1276
1277	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1278	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1279	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1280	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1281	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1282	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1283
1284	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1285	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1286	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2))); \
1287	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1288	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1289	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1290	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1291	} while (0)
1292
1293	#else
1294	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1295	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1296	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1297	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1298	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1299	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1300
1301	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1302	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1303	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1304	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1305	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1306	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1307
1308	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1309	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1310	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2)); \
1311	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1312	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1313	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1314	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1315	} while (0)
1316
1317	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1318	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1319	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1320	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1321	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1322	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1323
1324	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1325	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1326	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1327	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1328	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1329	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1330
1331	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_GCPtrMem2) do { \
1332	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1333	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_GCPtrMem2)); \
1334	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1335	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1336	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1337	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1338	} while (0)
1339
1340	#endif
1341
1342
1343
1344	#ifndef IEM_WITH_SETJMP
1345	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1346	do { \
1347	uint8_t u8Tmp; \
1348	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1349	(a_u16Dst) = u8Tmp; \
1350	} while (0)
1351	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1352	do { \
1353	uint8_t u8Tmp; \
1354	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1355	(a_u32Dst) = u8Tmp; \
1356	} while (0)
1357	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1358	do { \
1359	uint8_t u8Tmp; \
1360	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1361	(a_u64Dst) = u8Tmp; \
1362	} while (0)
1363	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1364	do { \
1365	uint16_t u16Tmp; \
1366	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1367	(a_u32Dst) = u16Tmp; \
1368	} while (0)
1369	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1370	do { \
1371	uint16_t u16Tmp; \
1372	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1373	(a_u64Dst) = u16Tmp; \
1374	} while (0)
1375	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1376	do { \
1377	uint32_t u32Tmp; \
1378	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1379	(a_u64Dst) = u32Tmp; \
1380	} while (0)
1381	#else /* IEM_WITH_SETJMP */
1382	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1383	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1384	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1385	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1386	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1387	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1388	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1389	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1390	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1391	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1392	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1393	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1394
1395	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1396	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1397	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1398	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1399	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1400	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1401	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1402	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1403	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1404	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1405	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1406	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1407	#endif /* IEM_WITH_SETJMP */
1408
1409	#ifndef IEM_WITH_SETJMP
1410	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1411	do { \
1412	uint8_t u8Tmp; \
1413	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1414	(a_u16Dst) = (int8_t)u8Tmp; \
1415	} while (0)
1416	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1417	do { \
1418	uint8_t u8Tmp; \
1419	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1420	(a_u32Dst) = (int8_t)u8Tmp; \
1421	} while (0)
1422	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1423	do { \
1424	uint8_t u8Tmp; \
1425	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1426	(a_u64Dst) = (int8_t)u8Tmp; \
1427	} while (0)
1428	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1429	do { \
1430	uint16_t u16Tmp; \
1431	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1432	(a_u32Dst) = (int16_t)u16Tmp; \
1433	} while (0)
1434	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1435	do { \
1436	uint16_t u16Tmp; \
1437	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1438	(a_u64Dst) = (int16_t)u16Tmp; \
1439	} while (0)
1440	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1441	do { \
1442	uint32_t u32Tmp; \
1443	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1444	(a_u64Dst) = (int32_t)u32Tmp; \
1445	} while (0)
1446	#else /* IEM_WITH_SETJMP */
1447	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1448	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1449	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1450	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1451	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1452	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1453	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1454	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1455	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1456	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1457	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1458	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1459
1460	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1461	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1462	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1463	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1464	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1465	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1466	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1467	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1468	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1469	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1470	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1471	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1472	#endif /* IEM_WITH_SETJMP */
1473
1474	#ifndef IEM_WITH_SETJMP
1475	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1476	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1477	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1478	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1479	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1480	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1481	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1482	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1483	#else
1484	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1485	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1486	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1487	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1488	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1489	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1490	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1491	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1492
1493	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1494	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1495	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1496	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1497	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1498	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1499	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1500	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1501	#endif
1502
1503	#ifndef IEM_WITH_SETJMP
1504	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1505	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1506	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1507	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1508	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1509	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1510	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1511	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1512	#else
1513	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1514	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1515	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1516	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1517	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1518	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1519	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1520	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1521
1522	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1523	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1524	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1525	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1526	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1527	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1528	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1529	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1530	#endif
1531
1532	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1533	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1534	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1535	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1536	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1537	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1538	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1539	do { \
1540	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1541	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1542	} while (0)
1543	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1544	do { \
1545	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1546	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1547	} while (0)
1548
1549	#ifndef IEM_WITH_SETJMP
1550	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1551	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1552	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1553	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1554	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1555	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1556	#else
1557	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1558	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1559	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1560	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1561	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1562	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1563
1564	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1565	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1566	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1567	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1568	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1569	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1570	#endif
1571
1572	#ifndef IEM_WITH_SETJMP
1573	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1574	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1575	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1576	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1577	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1578	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1579	#else
1580	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1581	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1582	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1583	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1584	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1585	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1586
1587	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1588	iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1589	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1590	iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1591	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1592	iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1593	#endif
1594
1595	/* Regular stack push and pop: */
1596	#ifndef IEM_WITH_SETJMP
1597	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1598	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1599	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1600	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1601
1602	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1603	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1604	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1605	#else
1606	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1607	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1608	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1609	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1610
1611	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1612	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1613	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1614	#endif
1615
1616	/* 32-bit flat stack push and pop: */
1617	#ifndef IEM_WITH_SETJMP
1618	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1619	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1620	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1621
1622	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1623	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1624	#else
1625	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1626	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1627	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1628
1629	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1630	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1631	#endif
1632
1633	/* 64-bit flat stack push and pop: */
1634	#ifndef IEM_WITH_SETJMP
1635	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1636	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1637
1638	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1639	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1640	#else
1641	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1642	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1643
1644	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1645	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1646	#endif
1647
1648
1649	/* 8-bit */
1650
1651	/**
1652	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1653	* acccess, for atomic operations.
1654	*
1655	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1656	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1657	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1658	* @param[in] a_GCPtrMem The memory address.
1659	* @remarks Will return/long jump on errors.
1660	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1661	*/
1662	#ifndef IEM_WITH_SETJMP
1663	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1664	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1665	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1666	#else
1667	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1668	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1669	#endif
1670
1671	/**
1672	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1673	*
1674	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1675	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1676	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1677	* @param[in] a_GCPtrMem The memory address.
1678	* @remarks Will return/long jump on errors.
1679	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1680	*/
1681	#ifndef IEM_WITH_SETJMP
1682	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1683	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1684	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1685	#else
1686	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1687	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1688	#endif
1689
1690	/**
1691	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1692	*
1693	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1694	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1695	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1696	* @param[in] a_GCPtrMem The memory address.
1697	* @remarks Will return/long jump on errors.
1698	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1699	*/
1700	#ifndef IEM_WITH_SETJMP
1701	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1702	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1703	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1704	#else
1705	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1706	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1707	#endif
1708
1709	/**
1710	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1711	*
1712	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1713	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1714	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1715	* @param[in] a_GCPtrMem The memory address.
1716	* @remarks Will return/long jump on errors.
1717	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1718	*/
1719	#ifndef IEM_WITH_SETJMP
1720	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1721	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1722	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1723	#else
1724	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1725	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1726	#endif
1727
1728	/**
1729	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1730	* acccess, flat address variant.
1731	*
1732	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1733	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1734	* @param[in] a_GCPtrMem The memory address.
1735	* @remarks Will return/long jump on errors.
1736	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1737	*/
1738	#ifndef IEM_WITH_SETJMP
1739	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1740	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1741	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1742	#else
1743	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1744	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1745	#endif
1746
1747	/**
1748	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1749	* address variant.
1750	*
1751	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1752	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1753	* @param[in] a_GCPtrMem The memory address.
1754	* @remarks Will return/long jump on errors.
1755	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1756	*/
1757	#ifndef IEM_WITH_SETJMP
1758	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1759	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1760	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1761	#else
1762	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1763	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1764	#endif
1765
1766	/**
1767	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1768	* address variant.
1769	*
1770	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1771	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1772	* @param[in] a_GCPtrMem The memory address.
1773	* @remarks Will return/long jump on errors.
1774	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1775	*/
1776	#ifndef IEM_WITH_SETJMP
1777	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1778	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1779	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1780	#else
1781	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1782	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1783	#endif
1784
1785	/**
1786	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1787	* address variant.
1788	*
1789	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1790	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1791	* @param[in] a_GCPtrMem The memory address.
1792	* @remarks Will return/long jump on errors.
1793	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1794	*/
1795	#ifndef IEM_WITH_SETJMP
1796	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1797	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1798	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1799	#else
1800	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1801	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1802	#endif
1803
1804
1805	/* 16-bit */
1806
1807	/**
1808	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1809	*
1810	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1811	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1812	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1813	* @param[in] a_GCPtrMem The memory address.
1814	* @remarks Will return/long jump on errors.
1815	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1816	*/
1817	#ifndef IEM_WITH_SETJMP
1818	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1819	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1820	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1821	#else
1822	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1823	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1824	#endif
1825
1826	/**
1827	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1828	*
1829	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1830	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1831	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1832	* @param[in] a_GCPtrMem The memory address.
1833	* @remarks Will return/long jump on errors.
1834	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1835	*/
1836	#ifndef IEM_WITH_SETJMP
1837	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1838	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1839	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1840	#else
1841	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1842	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1843	#endif
1844
1845	/**
1846	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1847	*
1848	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1849	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1850	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1851	* @param[in] a_GCPtrMem The memory address.
1852	* @remarks Will return/long jump on errors.
1853	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1854	*/
1855	#ifndef IEM_WITH_SETJMP
1856	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1857	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1858	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1859	#else
1860	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1861	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1862	#endif
1863
1864	/**
1865	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1866	*
1867	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1868	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1869	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1870	* @param[in] a_GCPtrMem The memory address.
1871	* @remarks Will return/long jump on errors.
1872	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1873	*/
1874	#ifndef IEM_WITH_SETJMP
1875	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1876	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1877	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1878	#else
1879	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1880	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1881	#endif
1882
1883	/**
1884	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1885	* acccess, flat address variant.
1886	*
1887	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1888	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1889	* @param[in] a_GCPtrMem The memory address.
1890	* @remarks Will return/long jump on errors.
1891	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1892	*/
1893	#ifndef IEM_WITH_SETJMP
1894	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1895	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1896	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1897	#else
1898	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1899	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1900	#endif
1901
1902	/**
1903	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1904	* address variant.
1905	*
1906	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1907	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1908	* @param[in] a_GCPtrMem The memory address.
1909	* @remarks Will return/long jump on errors.
1910	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1911	*/
1912	#ifndef IEM_WITH_SETJMP
1913	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1914	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1915	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1916	#else
1917	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1918	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1919	#endif
1920
1921	/**
1922	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1923	* address variant.
1924	*
1925	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1926	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1927	* @param[in] a_GCPtrMem The memory address.
1928	* @remarks Will return/long jump on errors.
1929	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1930	*/
1931	#ifndef IEM_WITH_SETJMP
1932	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1933	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1934	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1935	#else
1936	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1937	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1938	#endif
1939
1940	/**
1941	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1942	* address variant.
1943	*
1944	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1945	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1946	* @param[in] a_GCPtrMem The memory address.
1947	* @remarks Will return/long jump on errors.
1948	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1949	*/
1950	#ifndef IEM_WITH_SETJMP
1951	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1952	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1953	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1954	#else
1955	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1956	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1957	#endif
1958
1959	/** int16_t alias. */
1960	#ifndef IEM_WITH_SETJMP
1961	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1962	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1963	#else
1964	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1965	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1966	#endif
1967
1968	/** Flat int16_t alias. */
1969	#ifndef IEM_WITH_SETJMP
1970	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1971	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1972	#else
1973	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1974	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1975	#endif
1976
1977
1978	/* 32-bit */
1979
1980	/**
1981	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1982	*
1983	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1984	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1985	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1986	* @param[in] a_GCPtrMem The memory address.
1987	* @remarks Will return/long jump on errors.
1988	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1989	*/
1990	#ifndef IEM_WITH_SETJMP
1991	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1992	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1993	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1994	#else
1995	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1996	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1997	#endif
1998
1999	/**
2000	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
2001	*
2002	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2003	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2004	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2005	* @param[in] a_GCPtrMem The memory address.
2006	* @remarks Will return/long jump on errors.
2007	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2008	*/
2009	#ifndef IEM_WITH_SETJMP
2010	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2011	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2012	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2013	#else
2014	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2015	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2016	#endif
2017
2018	/**
2019	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
2020	*
2021	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2022	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2023	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2024	* @param[in] a_GCPtrMem The memory address.
2025	* @remarks Will return/long jump on errors.
2026	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2027	*/
2028	#ifndef IEM_WITH_SETJMP
2029	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2030	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2031	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2032	#else
2033	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2034	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2035	#endif
2036
2037	/**
2038	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
2039	*
2040	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2041	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2042	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2043	* @param[in] a_GCPtrMem The memory address.
2044	* @remarks Will return/long jump on errors.
2045	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2046	*/
2047	#ifndef IEM_WITH_SETJMP
2048	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2049	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2050	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2051	#else
2052	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2053	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2054	#endif
2055
2056	/**
2057	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
2058	* acccess, flat address variant.
2059	*
2060	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2061	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2062	* @param[in] a_GCPtrMem The memory address.
2063	* @remarks Will return/long jump on errors.
2064	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2065	*/
2066	#ifndef IEM_WITH_SETJMP
2067	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2068	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2069	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2070	#else
2071	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2072	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2073	#endif
2074
2075	/**
2076	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2077	* flat address variant.
2078	*
2079	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2080	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2081	* @param[in] a_GCPtrMem The memory address.
2082	* @remarks Will return/long jump on errors.
2083	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2084	*/
2085	#ifndef IEM_WITH_SETJMP
2086	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2087	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2088	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2089	#else
2090	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2091	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2092	#endif
2093
2094	/**
2095	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2096	* address variant.
2097	*
2098	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2099	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2100	* @param[in] a_GCPtrMem The memory address.
2101	* @remarks Will return/long jump on errors.
2102	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2103	*/
2104	#ifndef IEM_WITH_SETJMP
2105	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2106	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2107	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2108	#else
2109	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2110	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2111	#endif
2112
2113	/**
2114	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2115	* address variant.
2116	*
2117	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2118	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2119	* @param[in] a_GCPtrMem The memory address.
2120	* @remarks Will return/long jump on errors.
2121	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2122	*/
2123	#ifndef IEM_WITH_SETJMP
2124	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2125	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2126	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2127	#else
2128	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2129	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2130	#endif
2131
2132	/** int32_t alias. */
2133	#ifndef IEM_WITH_SETJMP
2134	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2135	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2136	#else
2137	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2138	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2139	#endif
2140
2141	/** Flat int32_t alias. */
2142	#ifndef IEM_WITH_SETJMP
2143	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2144	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2145	#else
2146	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2147	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2148	#endif
2149
2150	/** RTFLOAT32U alias. */
2151	#ifndef IEM_WITH_SETJMP
2152	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2153	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2154	#else
2155	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2156	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2157	#endif
2158
2159	/** Flat RTFLOAT32U alias. */
2160	#ifndef IEM_WITH_SETJMP
2161	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2162	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2163	#else
2164	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2165	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2166	#endif
2167
2168
2169	/* 64-bit */
2170
2171	/**
2172	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2173	*
2174	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2175	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2176	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2177	* @param[in] a_GCPtrMem The memory address.
2178	* @remarks Will return/long jump on errors.
2179	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2180	*/
2181	#ifndef IEM_WITH_SETJMP
2182	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2183	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2184	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2185	#else
2186	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2187	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2188	#endif
2189
2190	/**
2191	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2192	*
2193	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2194	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2195	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2196	* @param[in] a_GCPtrMem The memory address.
2197	* @remarks Will return/long jump on errors.
2198	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2199	*/
2200	#ifndef IEM_WITH_SETJMP
2201	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2202	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2203	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2204	#else
2205	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2206	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2207	#endif
2208
2209	/**
2210	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2211	*
2212	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2213	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2214	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2215	* @param[in] a_GCPtrMem The memory address.
2216	* @remarks Will return/long jump on errors.
2217	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2218	*/
2219	#ifndef IEM_WITH_SETJMP
2220	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2221	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2222	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2223	#else
2224	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2225	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2226	#endif
2227
2228	/**
2229	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2230	*
2231	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2232	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2233	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2234	* @param[in] a_GCPtrMem The memory address.
2235	* @remarks Will return/long jump on errors.
2236	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2237	*/
2238	#ifndef IEM_WITH_SETJMP
2239	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2240	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2241	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2242	#else
2243	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2244	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2245	#endif
2246
2247	/**
2248	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2249	* acccess, flat address variant.
2250	*
2251	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2252	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2253	* @param[in] a_GCPtrMem The memory address.
2254	* @remarks Will return/long jump on errors.
2255	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2256	*/
2257	#ifndef IEM_WITH_SETJMP
2258	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2259	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2260	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2261	#else
2262	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2263	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2264	#endif
2265
2266	/**
2267	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2268	* flat address variant.
2269	*
2270	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2271	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2272	* @param[in] a_GCPtrMem The memory address.
2273	* @remarks Will return/long jump on errors.
2274	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2275	*/
2276	#ifndef IEM_WITH_SETJMP
2277	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2278	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2279	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2280	#else
2281	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2282	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2283	#endif
2284
2285	/**
2286	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2287	* address variant.
2288	*
2289	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2290	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2291	* @param[in] a_GCPtrMem The memory address.
2292	* @remarks Will return/long jump on errors.
2293	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2294	*/
2295	#ifndef IEM_WITH_SETJMP
2296	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2297	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2298	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2299	#else
2300	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2301	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2302	#endif
2303
2304	/**
2305	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2306	* address variant.
2307	*
2308	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2309	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2310	* @param[in] a_GCPtrMem The memory address.
2311	* @remarks Will return/long jump on errors.
2312	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2313	*/
2314	#ifndef IEM_WITH_SETJMP
2315	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2316	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2317	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2318	#else
2319	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2320	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2321	#endif
2322
2323	/** int64_t alias. */
2324	#ifndef IEM_WITH_SETJMP
2325	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2326	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2327	#else
2328	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2329	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2330	#endif
2331
2332	/** Flat int64_t alias. */
2333	#ifndef IEM_WITH_SETJMP
2334	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2335	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2336	#else
2337	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2338	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2339	#endif
2340
2341	/** RTFLOAT64U alias. */
2342	#ifndef IEM_WITH_SETJMP
2343	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2344	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2345	#else
2346	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2347	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2348	#endif
2349
2350	/** Flat RTFLOAT64U alias. */
2351	#ifndef IEM_WITH_SETJMP
2352	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2353	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2354	#else
2355	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2356	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2357	#endif
2358
2359
2360	/* 128-bit */
2361
2362	/**
2363	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2364	*
2365	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2366	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2367	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2368	* @param[in] a_GCPtrMem The memory address.
2369	* @remarks Will return/long jump on errors.
2370	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2371	*/
2372	#ifndef IEM_WITH_SETJMP
2373	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2374	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2375	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2376	#else
2377	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2378	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2379	#endif
2380
2381	/**
2382	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2383	*
2384	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2385	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2386	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2387	* @param[in] a_GCPtrMem The memory address.
2388	* @remarks Will return/long jump on errors.
2389	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2390	*/
2391	#ifndef IEM_WITH_SETJMP
2392	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2393	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2394	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2395	#else
2396	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2397	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2398	#endif
2399
2400	/**
2401	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2402	*
2403	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2404	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2405	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2406	* @param[in] a_GCPtrMem The memory address.
2407	* @remarks Will return/long jump on errors.
2408	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2409	*/
2410	#ifndef IEM_WITH_SETJMP
2411	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2412	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2413	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2414	#else
2415	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2416	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2417	#endif
2418
2419	/**
2420	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2421	*
2422	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2423	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2424	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2425	* @param[in] a_GCPtrMem The memory address.
2426	* @remarks Will return/long jump on errors.
2427	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2428	*/
2429	#ifndef IEM_WITH_SETJMP
2430	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2431	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2432	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2433	#else
2434	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2435	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2436	#endif
2437
2438	/**
2439	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2440	* access, flat address variant.
2441	*
2442	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2443	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2444	* @param[in] a_GCPtrMem The memory address.
2445	* @remarks Will return/long jump on errors.
2446	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2447	*/
2448	#ifndef IEM_WITH_SETJMP
2449	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2450	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2451	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2452	#else
2453	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2454	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2455	#endif
2456
2457	/**
2458	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2459	* flat address variant.
2460	*
2461	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2462	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2463	* @param[in] a_GCPtrMem The memory address.
2464	* @remarks Will return/long jump on errors.
2465	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2466	*/
2467	#ifndef IEM_WITH_SETJMP
2468	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2469	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2470	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2471	#else
2472	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2473	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2474	#endif
2475
2476	/**
2477	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2478	* flat address variant.
2479	*
2480	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2481	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2482	* @param[in] a_GCPtrMem The memory address.
2483	* @remarks Will return/long jump on errors.
2484	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2485	*/
2486	#ifndef IEM_WITH_SETJMP
2487	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2488	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2489	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2490	#else
2491	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2492	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2493	#endif
2494
2495	/**
2496	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2497	* address variant.
2498	*
2499	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2500	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2501	* @param[in] a_GCPtrMem The memory address.
2502	* @remarks Will return/long jump on errors.
2503	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2504	*/
2505	#ifndef IEM_WITH_SETJMP
2506	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2507	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2508	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2509	#else
2510	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2511	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2512	#endif
2513
2514
2515	/* misc */
2516
2517	/**
2518	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2519	*
2520	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2521	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2522	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2523	* @param[in] a_GCPtrMem The memory address.
2524	* @remarks Will return/long jump on errors.
2525	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2526	*/
2527	#ifndef IEM_WITH_SETJMP
2528	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2529	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2530	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2531	#else
2532	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2533	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2534	#endif
2535
2536	/**
2537	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2538	*
2539	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2540	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2541	* @param[in] a_GCPtrMem The memory address.
2542	* @remarks Will return/long jump on errors.
2543	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2544	*/
2545	#ifndef IEM_WITH_SETJMP
2546	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2547	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2548	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2549	#else
2550	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2551	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2552	#endif
2553
2554
2555	/**
2556	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2557	*
2558	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2559	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2560	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2561	* @param[in] a_GCPtrMem The memory address.
2562	* @remarks Will return/long jump on errors.
2563	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2564	*/
2565	#ifndef IEM_WITH_SETJMP
2566	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2567	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2568	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2569	#else
2570	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2571	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2572	#endif
2573
2574	/**
2575	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2576	*
2577	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2578	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2579	* @param[in] a_GCPtrMem The memory address.
2580	* @remarks Will return/long jump on errors.
2581	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2582	*/
2583	#ifndef IEM_WITH_SETJMP
2584	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2585	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2586	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2587	#else
2588	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2589	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2590	#endif
2591
2592
2593
2594	/* commit + unmap */
2595
2596	/** Commits the memory and unmaps guest memory previously mapped RW.
2597	* @remarks May return.
2598	* @note Implictly frees the a_bMapInfo variable.
2599	*/
2600	#ifndef IEM_WITH_SETJMP
2601	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2602	#else
2603	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2604	#endif
2605
2606	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2607	* @remarks May return.
2608	* @note Implictly frees the a_bMapInfo variable.
2609	*/
2610	#ifndef IEM_WITH_SETJMP
2611	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2612	#else
2613	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2614	#endif
2615
2616	/** Commits the memory and unmaps guest memory previously mapped W.
2617	* @remarks May return.
2618	* @note Implictly frees the a_bMapInfo variable.
2619	*/
2620	#ifndef IEM_WITH_SETJMP
2621	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2622	#else
2623	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2624	#endif
2625
2626	/** Commits the memory and unmaps guest memory previously mapped R.
2627	* @remarks May return.
2628	* @note Implictly frees the a_bMapInfo variable.
2629	*/
2630	#ifndef IEM_WITH_SETJMP
2631	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2632	#else
2633	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2634	#endif
2635
2636
2637	/** Commits the memory and unmaps the guest memory unless the FPU status word
2638	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2639	* that would cause FLD not to store.
2640	*
2641	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2642	* store, while \#P will not.
2643	*
2644	* @remarks May in theory return - for now.
2645	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2646	*/
2647	#ifndef IEM_WITH_SETJMP
2648	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2649	if ( !(a_u16FSW & X86_FSW_ES) \
2650	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2651	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2652	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2653	else \
2654	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2655	} while (0)
2656	#else
2657	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2658	if ( !(a_u16FSW & X86_FSW_ES) \
2659	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2660	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2661	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2662	else \
2663	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2664	} while (0)
2665	#endif
2666
2667	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2668	* @note Implictly frees the a_bMapInfo variable. */
2669	#ifndef IEM_WITH_SETJMP
2670	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2671	#else
2672	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2673	#endif
2674
2675
2676
2677	/** Calculate efficient address from R/M. */
2678	#ifndef IEM_WITH_SETJMP
2679	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2680	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2681	#else
2682	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2683	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2684	#endif
2685
2686
2687	/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2688	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2689	#define IEM_MC_NATIVE_ELSE() } else {
2690	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2691
2692	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2693	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2694	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2695	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2696	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2697	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2698	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2699	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2700	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2701
2702	/** This can be used to direct the register allocator when dealing with
2703	* x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2704	#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2705
2706
2707	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2708	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2709	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2710	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2711	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2712	#define IEM_MC_CALL_AIMPL_3(a_rcType, a_rc, a_pfn, a0, a1, a2) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2))
2713	#define IEM_MC_CALL_AIMPL_4(a_rcType, a_rc, a_pfn, a0, a1, a2, a3) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2), (a3))
2714
2715
2716	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2717	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2718	*/
2719	#ifdef VBOX_STRICT
2720	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2721	do { \
2722	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2723	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2724	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2725	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2726	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2727	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2728	if (rcStrictHlp == VINF_SUCCESS) \
2729	{ \
2730	uint64_t const fRipMask = (pVCpu->iem.s.fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT ? UINT64_MAX : UINT32_MAX; \
2731	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2732	\|\| ( ((uRipBefore + cbInstr) & fRipMask) == pVCpu->cpum.GstCtx.rip \
2733	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2734	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2735	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2736	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2737	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2738	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, (uRipBefore + cbInstr) & fRipMask)); \
2739	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2740	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2741	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2742	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2743	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2744	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2745	else \
2746	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2747	== (fEflBefore & ~(X86_EFL_RF)), \
2748	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2749	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2750	{ \
2751	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2752	AssertMsg( fExecBefore == fExecRecalc \
2753	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2754	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2755	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2756	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2757	} \
2758	} \
2759	return rcStrictHlp; \
2760	} while (0)
2761	#else
2762	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2763	#endif
2764
2765	/**
2766	* Defers the rest of the instruction emulation to a C implementation routine
2767	* and returns, only taking the standard parameters.
2768	*
2769	* @param a_fFlags IEM_CIMPL_F_XXX.
2770	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2771	* in the native recompiler.
2772	* @param a_pfnCImpl The pointer to the C routine.
2773	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2774	*/
2775	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2776	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2777
2778	/**
2779	* Defers the rest of instruction emulation to a C implementation routine and
2780	* returns, taking one argument in addition to the standard ones.
2781	*
2782	* @param a_fFlags IEM_CIMPL_F_XXX.
2783	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2784	* in the native recompiler.
2785	* @param a_pfnCImpl The pointer to the C routine.
2786	* @param a0 The argument.
2787	*/
2788	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2789	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2790
2791	/**
2792	* Defers the rest of the instruction emulation to a C implementation routine
2793	* and returns, taking two arguments in addition to the standard ones.
2794	*
2795	* @param a_fFlags IEM_CIMPL_F_XXX.
2796	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2797	* in the native recompiler.
2798	* @param a_pfnCImpl The pointer to the C routine.
2799	* @param a0 The first extra argument.
2800	* @param a1 The second extra argument.
2801	*/
2802	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2803	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2804
2805	/**
2806	* Defers the rest of the instruction emulation to a C implementation routine
2807	* and returns, taking three arguments in addition to the standard ones.
2808	*
2809	* @param a_fFlags IEM_CIMPL_F_XXX.
2810	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2811	* in the native recompiler.
2812	* @param a_pfnCImpl The pointer to the C routine.
2813	* @param a0 The first extra argument.
2814	* @param a1 The second extra argument.
2815	* @param a2 The third extra argument.
2816	*/
2817	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2818	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2819
2820	/**
2821	* Defers the rest of the instruction emulation to a C implementation routine
2822	* and returns, taking four arguments in addition to the standard ones.
2823	*
2824	* @param a_fFlags IEM_CIMPL_F_XXX.
2825	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2826	* in the native recompiler.
2827	* @param a_pfnCImpl The pointer to the C routine.
2828	* @param a0 The first extra argument.
2829	* @param a1 The second extra argument.
2830	* @param a2 The third extra argument.
2831	* @param a3 The fourth extra argument.
2832	*/
2833	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2834	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2835
2836	/**
2837	* Defers the rest of the instruction emulation to a C implementation routine
2838	* and returns, taking two arguments in addition to the standard ones.
2839	*
2840	* @param a_fFlags IEM_CIMPL_F_XXX.
2841	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2842	* in the native recompiler.
2843	* @param a_pfnCImpl The pointer to the C routine.
2844	* @param a0 The first extra argument.
2845	* @param a1 The second extra argument.
2846	* @param a2 The third extra argument.
2847	* @param a3 The fourth extra argument.
2848	* @param a4 The fifth extra argument.
2849	*/
2850	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2851	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2852
2853	/**
2854	* Defers the entire instruction emulation to a C implementation routine and
2855	* returns, only taking the standard parameters.
2856	*
2857	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2858	*
2859	* @param a_fFlags IEM_CIMPL_F_XXX.
2860	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2861	* in the native recompiler.
2862	* @param a_pfnCImpl The pointer to the C routine.
2863	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2864	*/
2865	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2866	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2867
2868	/**
2869	* Defers the entire instruction emulation to a C implementation routine and
2870	* returns, taking one argument in addition to the standard ones.
2871	*
2872	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2873	*
2874	* @param a_fFlags IEM_CIMPL_F_XXX.
2875	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2876	* in the native recompiler.
2877	* @param a_pfnCImpl The pointer to the C routine.
2878	* @param a0 The argument.
2879	*/
2880	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2881	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2882
2883	/**
2884	* Defers the entire instruction emulation to a C implementation routine and
2885	* returns, taking two arguments in addition to the standard ones.
2886	*
2887	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2888	*
2889	* @param a_fFlags IEM_CIMPL_F_XXX.
2890	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2891	* in the native recompiler.
2892	* @param a_pfnCImpl The pointer to the C routine.
2893	* @param a0 The first extra argument.
2894	* @param a1 The second extra argument.
2895	*/
2896	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2897	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2898
2899	/**
2900	* Defers the entire instruction emulation to a C implementation routine and
2901	* returns, taking three arguments in addition to the standard ones.
2902	*
2903	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2904	*
2905	* @param a_fFlags IEM_CIMPL_F_XXX.
2906	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2907	* in the native recompiler.
2908	* @param a_pfnCImpl The pointer to the C routine.
2909	* @param a0 The first extra argument.
2910	* @param a1 The second extra argument.
2911	* @param a2 The third extra argument.
2912	*/
2913	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2914	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2915
2916
2917	/**
2918	* Calls a FPU assembly implementation taking one visible argument.
2919	*
2920	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2921	* @param a0 The first extra argument.
2922	*/
2923	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2924	do { \
2925	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2926	} while (0)
2927
2928	/**
2929	* Calls a FPU assembly implementation taking two visible arguments.
2930	*
2931	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2932	* @param a0 The first extra argument.
2933	* @param a1 The second extra argument.
2934	*/
2935	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2936	do { \
2937	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2938	} while (0)
2939
2940	/**
2941	* Calls a FPU assembly implementation taking three visible arguments.
2942	*
2943	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2944	* @param a0 The first extra argument.
2945	* @param a1 The second extra argument.
2946	* @param a2 The third extra argument.
2947	*/
2948	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2949	do { \
2950	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2951	} while (0)
2952
2953	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2954	do { \
2955	(a_FpuData).FSW = (a_FSW); \
2956	(a_FpuData).r80Result = *(a_pr80Value); \
2957	} while (0)
2958
2959	/** Pushes FPU result onto the stack. */
2960	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2961	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2962	/** Pushes FPU result onto the stack and sets the FPUDP. */
2963	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2964	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2965
2966	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2967	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2968	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2969
2970	/** Stores FPU result in a stack register. */
2971	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2972	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2973	/** Stores FPU result in a stack register and pops the stack. */
2974	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2975	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2976	/** Stores FPU result in a stack register and sets the FPUDP. */
2977	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2978	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2979	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2980	* stack. */
2981	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2982	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2983
2984	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2985	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2986	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2987	/** Free a stack register (for FFREE and FFREEP). */
2988	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2989	iemFpuStackFree(pVCpu, a_iStReg)
2990	/** Increment the FPU stack pointer. */
2991	#define IEM_MC_FPU_STACK_INC_TOP() \
2992	iemFpuStackIncTop(pVCpu)
2993	/** Decrement the FPU stack pointer. */
2994	#define IEM_MC_FPU_STACK_DEC_TOP() \
2995	iemFpuStackDecTop(pVCpu)
2996
2997	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2998	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2999	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3000	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
3001	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
3002	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3003	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
3004	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3005	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3006	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
3007	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
3008	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3009	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
3010	* stack. */
3011	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3012	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3013	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
3014	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
3015	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3016
3017	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
3018	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
3019	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
3020	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3021	* stack. */
3022	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
3023	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
3024	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3025	* FPUDS. */
3026	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3027	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3028	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3029	* FPUDS. Pops stack. */
3030	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3031	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3032	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3033	* stack twice. */
3034	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
3035	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
3036	/** Raises a FPU stack underflow exception for an instruction pushing a result
3037	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
3038	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
3039	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
3040	/** Raises a FPU stack underflow exception for an instruction pushing a result
3041	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
3042	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
3043	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
3044
3045	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3046	* FPUIP, FPUCS and FOP. */
3047	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
3048	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
3049	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3050	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
3051	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3052	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3053	/** Prepares for using the FPU state.
3054	* Ensures that we can use the host FPU in the current context (RC+R0.
3055	* Ensures the guest FPU state in the CPUMCTX is up to date. */
3056	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
3057	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
3058	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
3059	/** Actualizes the guest FPU state so it can be accessed and modified. */
3060	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
3061
3062	/** Prepares for using the SSE state.
3063	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
3064	* Ensures the guest SSE state in the CPUMCTX is up to date. */
3065	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
3066	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3067	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3068	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3069	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3070
3071	/** Prepares for using the AVX state.
3072	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3073	* Ensures the guest AVX state in the CPUMCTX is up to date.
3074	* @note This will include the AVX512 state too when support for it is added
3075	* due to the zero extending feature of VEX instruction. */
3076	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3077	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3078	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3079	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3080	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3081
3082	/**
3083	* Calls a MMX assembly implementation taking two visible arguments.
3084	*
3085	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3086	* @param a0 The first extra argument.
3087	* @param a1 The second extra argument.
3088	*/
3089	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3090	do { \
3091	IEM_MC_PREPARE_FPU_USAGE(); \
3092	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3093	} while (0)
3094
3095	/**
3096	* Calls a MMX assembly implementation taking three visible arguments.
3097	*
3098	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3099	* @param a0 The first extra argument.
3100	* @param a1 The second extra argument.
3101	* @param a2 The third extra argument.
3102	*/
3103	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3104	do { \
3105	IEM_MC_PREPARE_FPU_USAGE(); \
3106	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3107	} while (0)
3108
3109
3110	/**
3111	* Calls a SSE assembly implementation taking two visible arguments.
3112	*
3113	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3114	* @param a0 The first extra argument.
3115	* @param a1 The second extra argument.
3116	*/
3117	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3118	do { \
3119	IEM_MC_PREPARE_SSE_USAGE(); \
3120	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3121	(a0), (a1)); \
3122	} while (0)
3123
3124	/**
3125	* Calls a SSE assembly implementation taking three visible arguments.
3126	*
3127	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3128	* @param a0 The first extra argument.
3129	* @param a1 The second extra argument.
3130	* @param a2 The third extra argument.
3131	*/
3132	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3133	do { \
3134	IEM_MC_PREPARE_SSE_USAGE(); \
3135	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3136	(a0), (a1), (a2)); \
3137	} while (0)
3138
3139
3140	/**
3141	* Calls a AVX assembly implementation taking two visible arguments.
3142	*
3143	* There is one implicit zero'th argument, a pointer to the extended state.
3144	*
3145	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3146	* @param a0 The first extra argument.
3147	* @param a1 The second extra argument.
3148	*/
3149	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3150	do { \
3151	IEM_MC_PREPARE_AVX_USAGE(); \
3152	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3153	(a0), (a1)); \
3154	} while (0)
3155
3156	/**
3157	* Calls a AVX assembly implementation taking three visible arguments.
3158	*
3159	* There is one implicit zero'th argument, a pointer to the extended state.
3160	*
3161	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3162	* @param a0 The first extra argument.
3163	* @param a1 The second extra argument.
3164	* @param a2 The third extra argument.
3165	*/
3166	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3167	do { \
3168	IEM_MC_PREPARE_AVX_USAGE(); \
3169	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3170	(a0), (a1), (a2)); \
3171	} while (0)
3172
3173	/** @note Not for IOPL or IF testing. */
3174	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3175	/** @note Not for IOPL or IF testing. */
3176	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3177	/** @note Not for IOPL or IF testing. */
3178	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3179	/** @note Not for IOPL or IF testing. */
3180	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3181	/** @note Not for IOPL or IF testing. */
3182	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3183	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3184	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3185	/** @note Not for IOPL or IF testing. */
3186	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3187	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3188	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3189	/** @note Not for IOPL or IF testing. */
3190	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3191	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3192	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3193	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3194	/** @note Not for IOPL or IF testing. */
3195	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3196	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3197	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3198	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3199	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3200	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3201	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3202	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3203	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3204	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3205	/** @note Not for IOPL or IF testing. */
3206	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3207	if ( pVCpu->cpum.GstCtx.cx != 1 \
3208	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3209	/** @note Not for IOPL or IF testing. */
3210	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3211	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3212	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3213	/** @note Not for IOPL or IF testing. */
3214	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3215	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3216	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3217	/** @note Not for IOPL or IF testing. */
3218	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3219	if ( pVCpu->cpum.GstCtx.cx != 1 \
3220	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3221	/** @note Not for IOPL or IF testing. */
3222	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3223	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3224	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3225	/** @note Not for IOPL or IF testing. */
3226	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3227	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3228	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3229	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3230	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3231
3232	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3233	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3234	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3235	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3236	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3237	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3238	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3239	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3240	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3241	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3242	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3243	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3244	#define IEM_MC_IF_FCW_IM() \
3245	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3246
3247	#define IEM_MC_ELSE() } else {
3248	#define IEM_MC_ENDIF() } do {} while (0)
3249
3250
3251	/** Recompiler debugging: Flush guest register shadow copies. */
3252	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3253
3254	/** @} */
3255
3256	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3257

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 105316

Download in other formats: