IEMMc.h@ 105447

Last change on this file since 105447 was 105446, checked in by vboxsync, 4 months ago
VMM/IEM: Doxygen build fix after r164103
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 179.4 KB

Line
1	/* $Id: IEMMc.h 105446 2024-07-23 13:02:22Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
85	* @note only usable in 16-bit op size mode. */
86	#define IEM_MC_REL_CALL_S16_AND_FINISH(a_i16) \
87	return iemRegRipRelativeCallS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
88	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
89	#define IEM_MC_REL_CALL_S32_AND_FINISH(a_i32) \
90	return iemRegEip32RelativeCallS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32))
91	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
92	#define IEM_MC_REL_CALL_S64_AND_FINISH(a_i64) \
93	return iemRegRip64RelativeCallS64AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i64))
94	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
95	#define IEM_MC_IND_CALL_U16_AND_FINISH(a_u16NewIP) \
96	return iemRegIp16IndirectCallU16AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16NewIP))
97	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
98	#define IEM_MC_IND_CALL_U32_AND_FINISH(a_u32NewIP) \
99	return iemRegEip32IndirectCallU32AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u32NewIP))
100	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
101	#define IEM_MC_IND_CALL_U64_AND_FINISH(a_u64NewIP) \
102	return iemRegRip64IndirectCallU64AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u64NewIP))
103
104
105	/** Fetches the near return address from the stack, sets RIP and RSP (may trigger
106	* \#GP or \#SS), finishes the instruction and returns. */
107	#define IEM_MC_RETN_AND_FINISH(a_u16Pop) \
108	return iemRegRipNearReturnAndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16Pop), pVCpu->iem.s.enmEffOpSize)
109
110
111	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
112	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
113	do { \
114	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
115	{ /* probable */ } \
116	else return iemRaiseDeviceNotAvailable(pVCpu); \
117	} while (0)
118	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
119	do { \
120	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
121	{ /* probable */ } \
122	else return iemRaiseDeviceNotAvailable(pVCpu); \
123	} while (0)
124	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
125	do { \
126	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
127	{ /* probable */ } \
128	else return iemRaiseMathFault(pVCpu); \
129	} while (0)
130	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
131	do { \
132	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
133	be reduced to a single compare branch in the more probably code path. */ \
134	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
135	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
136	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
137	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
138	{ /* probable */ } \
139	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
140	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
141	return iemRaiseUndefinedOpcode(pVCpu); \
142	else \
143	return iemRaiseDeviceNotAvailable(pVCpu); \
144	} while (0)
145	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
146	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
147	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
148	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
149	do { \
150	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
151	single compare branch in the more probable code path. */ \
152	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
153	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
154	== X86_CR4_OSFXSR)) \
155	{ /* likely */ } \
156	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
157	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
158	return iemRaiseUndefinedOpcode(pVCpu); \
159	else \
160	return iemRaiseDeviceNotAvailable(pVCpu); \
161	} while (0)
162	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
163	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
164	do { \
165	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
166	single compare branch in the more probable code path. */ \
167	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
168	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
169	{ /* probable */ } \
170	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
171	return iemRaiseUndefinedOpcode(pVCpu); \
172	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
173	return iemRaiseDeviceNotAvailable(pVCpu); \
174	else \
175	return iemRaiseMathFault(pVCpu); \
176	} while (0)
177	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
178	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
179	* count the CPL into the TB key. However it is safe enough for now, as
180	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
181	* emitted for it. */
182	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
183	do { \
184	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
185	else return iemRaiseGeneralProtectionFault0(pVCpu); \
186	} while (0)
187	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
188	do { \
189	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
190	else return iemRaiseGeneralProtectionFault0(pVCpu); \
191	} while (0)
192	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
193	do { \
194	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
195	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
196	{ /* probable */ } \
197	else return iemRaiseUndefinedOpcode(pVCpu); \
198	} while (0)
199	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
200	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
201	do { \
202	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
203	else return iemRaiseGeneralProtectionFault0(pVCpu); \
204	} while (0)
205
206
207	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
208	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
209	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
210	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
211	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
212	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
213	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
214	/** @note IEMAllInstPython.py duplicates the expansion. */
215	#define IEM_MC_ARG_EFLAGS(a_Name, a_iArg) uint32_t const a_Name = pVCpu->cpum.GstCtx.eflags.u
216	/** @note IEMAllInstPython.py duplicates the expansion. */
217	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
218	uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
219	uint32_t *a_pName = &a_Name
220	/** @note IEMAllInstPython.py duplicates the expansion. */
221	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
222	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
223	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
224	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
225	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
226	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
227	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
228	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
229	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
230	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
231	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
232	} while (0)
233	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
234	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
235
236	/** ASSUMES the source variable not used after this statement. */
237	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
238
239	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
240	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
241	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
242	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
243	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
244	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
245	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
246	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
247	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
248	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
249	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
250	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
251	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
252	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
253	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
254	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
255	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
256	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
257	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
258	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
259	} while(0)
260	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
261	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
262	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
263	} while(0)
264	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
265	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
266	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
267	} while (0)
268	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
269	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
270	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
271	} while (0)
272	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
273	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
274	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
275	} while (0)
276	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
277	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
278	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
279	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
280	} while (0)
281	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
282	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
283	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
284	} while (0)
285	/** @note Not for IOPL or IF testing or modification. */
286	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
287	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
288	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
289	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
290	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
291
292	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
293	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
294	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
295	#define IEM_MC_STORE_GREG_I32(a_iGReg, a_i32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_i32Value) / clear high bits. */
296	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
297	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
298	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
299	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
300	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
301	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
302	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
303	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
304	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
305	} while(0)
306	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
307	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
308	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
309	} while(0)
310	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
311
312	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
313	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
314	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
315	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
316	} while (0)
317	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
318	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
319	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
320	} while (0)
321	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
322	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
323
324
325	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
326	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
327	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
328	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
329	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
330	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
331	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
332	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
333	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
334	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
335	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
336	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
337	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
338	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
339	/** @note Not for IOPL or IF testing or modification.
340	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
341	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
342	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
343
344	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
345	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
346	do { \
347	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
348	*pu32Reg += (a_u32Value); \
349	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
350	} while (0)
351	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
352
353	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
354	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
355	do { \
356	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
357	*pu32Reg -= (a_u8Const); \
358	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
359	} while (0)
360	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
361	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
362
363	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
364	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
365	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
366	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
367	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
368	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
369	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
370
371	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
372	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
373	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
374	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
375
376	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
377	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
378	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
379
380	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
381	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
382	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
383
384	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
385	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
386	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
387
388	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
389
390	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
391	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
392	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
393
394	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
395
396	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
397
398	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
399	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
400	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
401	do { \
402	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
403	*pu32Reg &= (a_u32Value); \
404	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
405	} while (0)
406	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
407
408	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
409	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
410	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
411	do { \
412	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
413	*pu32Reg \|= (a_u32Value); \
414	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
415	} while (0)
416	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
417
418	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
419	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
420	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
421
422	/** @note Not for IOPL or IF modification. */
423	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
424	/** @note Not for IOPL or IF modification. */
425	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
426	/** @note Not for IOPL or IF modification. */
427	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
428
429	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
430
431	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
432	#define IEM_MC_FPU_TO_MMX_MODE() do { \
433	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
434	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
435	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
436	} while (0)
437
438	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
439	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
440	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
441	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
442	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
443	} while (0)
444
445	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
446	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
447	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
448	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
449	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
450	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
451	#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
452	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
453	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
454	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
455	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
456	} while (0)
457	#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
458	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
459	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
460	} while (0)
461	#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
462	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
463	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
464	} while (0)
465	#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
466	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
467	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
468	} while (0)
469	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
470	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
471	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
472	} while (0)
473	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
474	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
475	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
476	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
477	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
478	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
479	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
480	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
481	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
482	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
483
484	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
485	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
486	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
487	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
488	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
489	} while (0)
490	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
491	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
492	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
493	} while (0)
494	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
495	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
496	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
497	} while (0)
498	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
499	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
500	#define IEM_MC_FETCH_XREG_R64(a_r64Value, a_iXReg, a_iQWord) \
501	do { (a_r64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[(a_iQWord)]; } while (0)
502	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
503	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
504	#define IEM_MC_FETCH_XREG_R32(a_r32Value, a_iXReg, a_iDWord) \
505	do { (a_r32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[(a_iDWord)]; } while (0)
506	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
507	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
508	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
509	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
510	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
511	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
512	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
513	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
514	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
515	} while (0)
516	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
517	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
518	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
519	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
520	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
521	} while (0)
522	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
523	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
524	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
525	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
526	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
527	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
528	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
529	} while (0)
530	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
531	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
532	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
533	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
534	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
535	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
536	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
537	} while (0)
538	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
539	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
540	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
541	} while (0)
542	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
543	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
544	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
545	} while (0)
546	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
547	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
548	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
549	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
550	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
551	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
552	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
553	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
554	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
555	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
556	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
557	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
558
559	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
560	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
561	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
562	} while (0)
563
564	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
565	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
566	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
567	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
568	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
569	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
570	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
571	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
572	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
573	} while (0)
574
575	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
576	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
577	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
578	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
579	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
580	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
581	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
582	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
583	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
584	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
585	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
586	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
587	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
588	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
590	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
591	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
593	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
594	} while (0)
595	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
596	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
598	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
599	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
600	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
601	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
602	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
603	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
605	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
606	} while (0)
607	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
608	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
612	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
613	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
614	} while (0)
615	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
616	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
617	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
618	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
619	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
620	} while (0)
621
622	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
623	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
624	#define IEM_MC_REF_XREG_XMM(a_pXmmDst, a_iXReg) \
625	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
626	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
627	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
628	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
629	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
630	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
631	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
632	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
633	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
634	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
635	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
636	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
637	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
638	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
639	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
640	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
641	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
642	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
643	} while (0)
644
645	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
646	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
647	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
648	} while (0)
649	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
650	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
651	if ((a_iQWord) < 2) \
652	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
653	else \
654	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[(a_iQWord) - 2]; \
655	} while (0)
656	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
657	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
658	if ((a_iDQword) == 0) \
659	{ \
660	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
661	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
662	} \
663	else \
664	{ \
665	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
666	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
667	} \
668	} while (0)
669	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
670	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
671	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
672	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
673	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
674	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
675	} while (0)
676	#define IEM_MC_FETCH_YREG_YMM(a_uYmmDst, a_iYRegSrc) \
677	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
678	(a_uYmmDst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
679	(a_uYmmDst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
680	(a_uYmmDst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
681	(a_uYmmDst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
682	} while (0)
683	#define IEM_MC_FETCH_YREG_PAIR_YMM(a_uYmmDst, a_iYRegSrc1, a_iYRegSrc2) \
684	do { uintptr_t const iYRegSrc1Tmp = (a_iYRegSrc1); \
685	uintptr_t const iYRegSrc2Tmp = (a_iYRegSrc2); \
686	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[0]; \
687	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[1]; \
688	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[0]; \
689	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[1]; \
690	(a_uYmmDst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[0]; \
691	(a_uYmmDst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[1]; \
692	(a_uYmmDst).uSrc2.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[0]; \
693	(a_uYmmDst).uSrc2.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[1]; \
694	} while (0)
695
696	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
697	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
698	if ((a_iDQword) == 0) \
699	{ \
700	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
701	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
702	} \
703	else \
704	{ \
705	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
706	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
707	} \
708	} while (0)
709
710	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
711	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
712	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
713	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
714	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
715	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
716	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
717	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
718	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
719	} while (0)
720	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
721	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
722	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
723	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
724	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
725	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
726	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
727	} while (0)
728	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
729	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
730	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
731	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
732	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
733	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
734	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
735	} while (0)
736	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
737	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
738	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
739	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
740	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
741	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
742	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
743	} while (0)
744	#define IEM_MC_STORE_YREG_YMM_ZX_VLMAX(a_iYRegDst, a_uYmmSrc) \
745	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[0]; \
747	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[1]; \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[2]; \
749	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[3]; \
750	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
751	} while (0)
752	#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
753	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
754	if ((a_iDwDst) < 4) \
755	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
756	else \
757	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
758	} while (0)
759	#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
760	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
761	if ((a_iQwDst) < 2) \
762	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
763	else \
764	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
765	} while (0)
766	#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
767	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
768	if ((a_iQword) < 2) \
769	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
770	else \
771	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
772	} while (0)
773
774	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
775	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
776	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
777	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
778	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
779	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
780	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
781	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
782	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
783	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
784	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
785	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
790	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
791	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
792	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
793	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
794	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
795	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
796	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
797	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
798	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
799	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
800	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
801	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
802	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
803	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
804	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
808	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
809	} while (0)
810	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
811	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
812	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
813	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
814	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
815	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
816	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
817	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
818	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
819	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
820	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
821	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
822	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
823	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
824	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
825	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
826	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
827	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
828	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
829	} while (0)
830	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
831	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
832	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
833	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
834	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
835	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
836	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
837	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
838	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
839	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
840	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
841	} while (0)
842	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
843	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
844	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
845	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
846	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
847	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
848	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
849	} while (0)
850	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
851	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
852	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
853	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
854	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
855	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
856	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
857	} while (0)
858
859	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
860	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
861	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
862	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
863	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
864	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
865	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
866	do { uintptr_t const iYRegTmp = (a_iYReg); \
867	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
868	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
869	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
870	} while (0)
871
872	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
873	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
874	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
875	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
876	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
877	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
878	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
879	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
880	} while (0)
881	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
882	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
883	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
884	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
885	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
886	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
887	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
888	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
889	} while (0)
890	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
891	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
892	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
893	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
894	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
895	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
896	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
897	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
898	} while (0)
899
900	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
901	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
902	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
903	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
904	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
905	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
906	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
907	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
908	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
909	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
910	} while (0)
911	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
912	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
913	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
914	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
915	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
916	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
917	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
918	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
919	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
920	} while (0)
921	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
922	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
923	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
924	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
925	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
926	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
927	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
928	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
929	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
930	} while (0)
931	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
932	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
933	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
934	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
935	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
936	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
937	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
938	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
939	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
940	} while (0)
941	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
942	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
943	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
944	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
945	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
946	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
947	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
948	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
949	} while (0)
950	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
951	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
952	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
953	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
954	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
955	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
956	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
957	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
958	} while (0)
959
960	#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
961	do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
962
963	#ifndef IEM_WITH_SETJMP
964	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
965	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
966	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
967	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
968	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
969	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
970	#else
971	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
972	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
973	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
974	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
975	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
976	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
977
978	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
979	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
980	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
981	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
982	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
983	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
984	#endif
985
986	#ifndef IEM_WITH_SETJMP
987	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
988	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
989	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
990	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
991	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
992	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
993	#else
994	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
995	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
996	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
997	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
998	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
999	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1000
1001	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
1002	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1003	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
1004	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1005	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
1006	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1007	#endif
1008
1009	#ifndef IEM_WITH_SETJMP
1010	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1011	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
1012	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1013	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1014	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1015	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
1016	#else
1017	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1018	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1019	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1020	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1021	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1022	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1023
1024	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
1025	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1026	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
1027	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1028	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
1029	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1030	#endif
1031
1032	#ifndef IEM_WITH_SETJMP
1033	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1034	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1035	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1036	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1037	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1038	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1039	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1040	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
1041	#else
1042	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1043	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1044	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1045	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1046	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1047	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1048	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1049	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1050
1051	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1052	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1053	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1054	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1055	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1056	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1057	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1058	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1059	#endif
1060
1061	#ifndef IEM_WITH_SETJMP
1062	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1063	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1064	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1065	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1066	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1067	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1068	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1069	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1070	#else
1071	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1072	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1073	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1074	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1075	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1076	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1077	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1078	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1079
1080	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1081	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1082	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1083	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1084	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1085	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1086	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1087	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1088	#endif
1089
1090	#ifndef IEM_WITH_SETJMP
1091	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1092	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1093	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1094	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1095	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1096	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1097
1098	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1099	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1100	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1101	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1102
1103	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1104	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1105	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1106	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1107	} while (0)
1108
1109	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1110	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1111	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1112	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1113	} while (0)
1114
1115	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1116	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1117	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1118	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1119	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1120	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1121	} while (0)
1122
1123	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1124	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1125	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1126	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1127	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1128	} while (0)
1129
1130	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1131	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1132	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1133	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1134	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1135	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1136	} while (0)
1137	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1138	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1139	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1140	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1141	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1142	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1143	} while (0)
1144
1145	#else
1146	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1147	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1148	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1149	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1150	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1151	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1152
1153	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1154	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1155	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1156	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1157	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1158	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1159
1160	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1161	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1162	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1163	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1164	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1165	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1166
1167	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1168	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1169	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1170	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1171	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1172	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1173
1174	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1175	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1176	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1177	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1178	} while (0)
1179	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1180	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1181	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1182	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1183	} while (0)
1184
1185	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1186	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1187	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1188	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1189	} while (0)
1190	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1191	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1192	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1193	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1194	} while (0)
1195
1196	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1197	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1198	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1199	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1200	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1201	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1202	} while (0)
1203	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1204	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1205	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1206	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1207	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1208	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1209	} while (0)
1210
1211	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1212	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1213	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1214	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1215	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1216	} while (0)
1217	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1218	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1219	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1220	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1221	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1222	} while (0)
1223
1224
1225	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1226	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1227	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1228	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1229	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1230	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1231	} while (0)
1232	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1233	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1234	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1235	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1236	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1237	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1238	} while (0)
1239
1240	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1241	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1242	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1243	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1244	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1245	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1246	} while (0)
1247	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1248	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1249	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1250	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1251	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1252	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1253	} while (0)
1254
1255	#endif
1256
1257	#ifndef IEM_WITH_SETJMP
1258	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1259	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1260	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1261	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1262	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1263	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1264
1265	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1266	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1267	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1268	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1269	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1270	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1271
1272	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1273	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1274	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2))); \
1275	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1276	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1277	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1278	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1279	} while (0)
1280
1281	#else
1282	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1283	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1284	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1285	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1286	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1287	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1288
1289	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1290	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1291	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1292	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1293	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1294	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1295
1296	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1297	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1298	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2)); \
1299	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1300	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1301	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1302	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1303	} while (0)
1304
1305	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1306	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1307	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1308	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1309	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1310	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1311
1312	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1313	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1314	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1315	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1316	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1317	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1318
1319	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_GCPtrMem2) do { \
1320	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1321	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_GCPtrMem2)); \
1322	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1323	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1324	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1325	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1326	} while (0)
1327
1328	#endif
1329
1330
1331
1332	#ifndef IEM_WITH_SETJMP
1333	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1334	do { \
1335	uint8_t u8Tmp; \
1336	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1337	(a_u16Dst) = u8Tmp; \
1338	} while (0)
1339	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1340	do { \
1341	uint8_t u8Tmp; \
1342	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1343	(a_u32Dst) = u8Tmp; \
1344	} while (0)
1345	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1346	do { \
1347	uint8_t u8Tmp; \
1348	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1349	(a_u64Dst) = u8Tmp; \
1350	} while (0)
1351	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1352	do { \
1353	uint16_t u16Tmp; \
1354	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1355	(a_u32Dst) = u16Tmp; \
1356	} while (0)
1357	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1358	do { \
1359	uint16_t u16Tmp; \
1360	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1361	(a_u64Dst) = u16Tmp; \
1362	} while (0)
1363	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1364	do { \
1365	uint32_t u32Tmp; \
1366	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1367	(a_u64Dst) = u32Tmp; \
1368	} while (0)
1369	#else /* IEM_WITH_SETJMP */
1370	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1371	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1372	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1373	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1374	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1375	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1376	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1377	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1378	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1379	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1380	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1381	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1382
1383	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1384	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1385	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1386	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1387	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1388	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1389	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1390	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1391	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1392	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1393	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1394	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1395	#endif /* IEM_WITH_SETJMP */
1396
1397	#ifndef IEM_WITH_SETJMP
1398	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1399	do { \
1400	uint8_t u8Tmp; \
1401	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1402	(a_u16Dst) = (int8_t)u8Tmp; \
1403	} while (0)
1404	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1405	do { \
1406	uint8_t u8Tmp; \
1407	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1408	(a_u32Dst) = (int8_t)u8Tmp; \
1409	} while (0)
1410	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1411	do { \
1412	uint8_t u8Tmp; \
1413	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1414	(a_u64Dst) = (int8_t)u8Tmp; \
1415	} while (0)
1416	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1417	do { \
1418	uint16_t u16Tmp; \
1419	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1420	(a_u32Dst) = (int16_t)u16Tmp; \
1421	} while (0)
1422	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1423	do { \
1424	uint16_t u16Tmp; \
1425	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1426	(a_u64Dst) = (int16_t)u16Tmp; \
1427	} while (0)
1428	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1429	do { \
1430	uint32_t u32Tmp; \
1431	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1432	(a_u64Dst) = (int32_t)u32Tmp; \
1433	} while (0)
1434	#else /* IEM_WITH_SETJMP */
1435	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1436	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1437	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1438	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1439	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1440	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1441	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1442	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1443	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1444	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1445	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1446	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1447
1448	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1449	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1450	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1451	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1452	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1453	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1454	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1455	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1456	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1457	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1458	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1459	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1460	#endif /* IEM_WITH_SETJMP */
1461
1462	#ifndef IEM_WITH_SETJMP
1463	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1464	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1465	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1466	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1467	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1468	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1469	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1470	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1471	#else
1472	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1473	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1474	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1475	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1476	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1477	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1478	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1479	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1480
1481	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1482	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1483	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1484	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1485	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1486	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1487	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1488	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1489	#endif
1490
1491	#ifndef IEM_WITH_SETJMP
1492	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1493	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1494	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1495	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1496	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1497	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1498	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1499	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1500	#else
1501	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1502	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1503	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1504	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1505	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1506	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1507	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1508	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1509
1510	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1511	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1512	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1513	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1514	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1515	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1516	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1517	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1518	#endif
1519
1520	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1521	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1522	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1523	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1524	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1525	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1526	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1527	do { \
1528	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1529	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1530	} while (0)
1531	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1532	do { \
1533	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1534	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1535	} while (0)
1536
1537	#ifndef IEM_WITH_SETJMP
1538	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1539	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1540	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1541	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1542	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1543	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1544	#else
1545	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1546	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1547	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1548	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1549	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1550	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1551
1552	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1553	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1554	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1555	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1556	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1557	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1558	#endif
1559
1560	#ifndef IEM_WITH_SETJMP
1561	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1562	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1563	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1564	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1565	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1566	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1567	#else
1568	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1569	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1570	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1571	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1572	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1573	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1574
1575	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1576	iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1577	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1578	iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1579	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1580	iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1581	#endif
1582
1583	/* Regular stack push and pop: */
1584	#ifndef IEM_WITH_SETJMP
1585	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1586	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1587	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1588	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1589
1590	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1591	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1592	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1593	#else
1594	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1595	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1596	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1597	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1598
1599	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1600	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1601	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1602	#endif
1603
1604	/* 32-bit flat stack push and pop: */
1605	#ifndef IEM_WITH_SETJMP
1606	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1607	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1608	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1609
1610	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1611	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1612	#else
1613	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1614	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1615	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1616
1617	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1618	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1619	#endif
1620
1621	/* 64-bit flat stack push and pop: */
1622	#ifndef IEM_WITH_SETJMP
1623	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1624	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1625
1626	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1627	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1628	#else
1629	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1630	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1631
1632	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1633	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1634	#endif
1635
1636
1637	/* 8-bit */
1638
1639	/**
1640	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1641	* acccess, for atomic operations.
1642	*
1643	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1644	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1645	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1646	* @param[in] a_GCPtrMem The memory address.
1647	* @remarks Will return/long jump on errors.
1648	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1649	*/
1650	#ifndef IEM_WITH_SETJMP
1651	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1652	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1653	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1654	#else
1655	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1656	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1657	#endif
1658
1659	/**
1660	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1661	*
1662	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1663	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1664	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1665	* @param[in] a_GCPtrMem The memory address.
1666	* @remarks Will return/long jump on errors.
1667	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1668	*/
1669	#ifndef IEM_WITH_SETJMP
1670	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1671	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1672	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1673	#else
1674	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1675	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1676	#endif
1677
1678	/**
1679	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1680	*
1681	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1682	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1683	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1684	* @param[in] a_GCPtrMem The memory address.
1685	* @remarks Will return/long jump on errors.
1686	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1687	*/
1688	#ifndef IEM_WITH_SETJMP
1689	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1690	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1691	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1692	#else
1693	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1694	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1695	#endif
1696
1697	/**
1698	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1699	*
1700	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1701	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1702	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1703	* @param[in] a_GCPtrMem The memory address.
1704	* @remarks Will return/long jump on errors.
1705	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1706	*/
1707	#ifndef IEM_WITH_SETJMP
1708	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1709	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1710	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1711	#else
1712	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1713	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1714	#endif
1715
1716	/**
1717	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1718	* acccess, flat address variant.
1719	*
1720	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1721	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1722	* @param[in] a_GCPtrMem The memory address.
1723	* @remarks Will return/long jump on errors.
1724	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1725	*/
1726	#ifndef IEM_WITH_SETJMP
1727	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1728	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1729	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1730	#else
1731	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1732	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1733	#endif
1734
1735	/**
1736	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1737	* address variant.
1738	*
1739	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1740	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1741	* @param[in] a_GCPtrMem The memory address.
1742	* @remarks Will return/long jump on errors.
1743	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1744	*/
1745	#ifndef IEM_WITH_SETJMP
1746	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1747	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1748	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1749	#else
1750	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1751	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1752	#endif
1753
1754	/**
1755	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1756	* address variant.
1757	*
1758	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1759	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1760	* @param[in] a_GCPtrMem The memory address.
1761	* @remarks Will return/long jump on errors.
1762	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1763	*/
1764	#ifndef IEM_WITH_SETJMP
1765	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1766	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1767	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1768	#else
1769	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1770	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1771	#endif
1772
1773	/**
1774	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1775	* address variant.
1776	*
1777	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1778	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1779	* @param[in] a_GCPtrMem The memory address.
1780	* @remarks Will return/long jump on errors.
1781	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1782	*/
1783	#ifndef IEM_WITH_SETJMP
1784	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1785	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1786	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1787	#else
1788	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1789	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1790	#endif
1791
1792
1793	/* 16-bit */
1794
1795	/**
1796	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1797	*
1798	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1799	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1800	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1801	* @param[in] a_GCPtrMem The memory address.
1802	* @remarks Will return/long jump on errors.
1803	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1804	*/
1805	#ifndef IEM_WITH_SETJMP
1806	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1807	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1808	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1809	#else
1810	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1811	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1812	#endif
1813
1814	/**
1815	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1816	*
1817	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1818	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1819	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1820	* @param[in] a_GCPtrMem The memory address.
1821	* @remarks Will return/long jump on errors.
1822	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1823	*/
1824	#ifndef IEM_WITH_SETJMP
1825	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1826	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1827	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1828	#else
1829	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1830	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1831	#endif
1832
1833	/**
1834	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1835	*
1836	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1837	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1838	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1839	* @param[in] a_GCPtrMem The memory address.
1840	* @remarks Will return/long jump on errors.
1841	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1842	*/
1843	#ifndef IEM_WITH_SETJMP
1844	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1845	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1846	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1847	#else
1848	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1849	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1850	#endif
1851
1852	/**
1853	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1854	*
1855	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1856	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1857	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1858	* @param[in] a_GCPtrMem The memory address.
1859	* @remarks Will return/long jump on errors.
1860	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1861	*/
1862	#ifndef IEM_WITH_SETJMP
1863	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1864	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1865	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1866	#else
1867	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1868	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1869	#endif
1870
1871	/**
1872	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1873	* acccess, flat address variant.
1874	*
1875	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1876	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1877	* @param[in] a_GCPtrMem The memory address.
1878	* @remarks Will return/long jump on errors.
1879	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1880	*/
1881	#ifndef IEM_WITH_SETJMP
1882	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1883	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1884	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1885	#else
1886	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1887	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1888	#endif
1889
1890	/**
1891	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1892	* address variant.
1893	*
1894	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1895	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1896	* @param[in] a_GCPtrMem The memory address.
1897	* @remarks Will return/long jump on errors.
1898	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1899	*/
1900	#ifndef IEM_WITH_SETJMP
1901	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1902	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1903	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1904	#else
1905	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1906	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1907	#endif
1908
1909	/**
1910	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1911	* address variant.
1912	*
1913	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1914	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1915	* @param[in] a_GCPtrMem The memory address.
1916	* @remarks Will return/long jump on errors.
1917	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1918	*/
1919	#ifndef IEM_WITH_SETJMP
1920	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1921	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1922	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1923	#else
1924	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1925	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1926	#endif
1927
1928	/**
1929	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1930	* address variant.
1931	*
1932	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1933	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1934	* @param[in] a_GCPtrMem The memory address.
1935	* @remarks Will return/long jump on errors.
1936	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1937	*/
1938	#ifndef IEM_WITH_SETJMP
1939	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1940	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1941	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1942	#else
1943	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1944	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1945	#endif
1946
1947	/** int16_t alias. */
1948	#ifndef IEM_WITH_SETJMP
1949	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1950	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1951	#else
1952	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1953	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1954	#endif
1955
1956	/** Flat int16_t alias. */
1957	#ifndef IEM_WITH_SETJMP
1958	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1959	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1960	#else
1961	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1962	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1963	#endif
1964
1965
1966	/* 32-bit */
1967
1968	/**
1969	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1970	*
1971	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1972	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1973	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1974	* @param[in] a_GCPtrMem The memory address.
1975	* @remarks Will return/long jump on errors.
1976	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1977	*/
1978	#ifndef IEM_WITH_SETJMP
1979	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1980	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1981	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1982	#else
1983	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1984	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1985	#endif
1986
1987	/**
1988	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1989	*
1990	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1991	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1992	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1993	* @param[in] a_GCPtrMem The memory address.
1994	* @remarks Will return/long jump on errors.
1995	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1996	*/
1997	#ifndef IEM_WITH_SETJMP
1998	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1999	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2000	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2001	#else
2002	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2003	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2004	#endif
2005
2006	/**
2007	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
2008	*
2009	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2010	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2011	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2012	* @param[in] a_GCPtrMem The memory address.
2013	* @remarks Will return/long jump on errors.
2014	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2015	*/
2016	#ifndef IEM_WITH_SETJMP
2017	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2018	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2019	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2020	#else
2021	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2022	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2023	#endif
2024
2025	/**
2026	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
2027	*
2028	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2029	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2030	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2031	* @param[in] a_GCPtrMem The memory address.
2032	* @remarks Will return/long jump on errors.
2033	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2034	*/
2035	#ifndef IEM_WITH_SETJMP
2036	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2037	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2038	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2039	#else
2040	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2041	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2042	#endif
2043
2044	/**
2045	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
2046	* acccess, flat address variant.
2047	*
2048	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2049	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2050	* @param[in] a_GCPtrMem The memory address.
2051	* @remarks Will return/long jump on errors.
2052	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2053	*/
2054	#ifndef IEM_WITH_SETJMP
2055	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2056	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2057	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2058	#else
2059	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2060	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2061	#endif
2062
2063	/**
2064	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2065	* flat address variant.
2066	*
2067	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2068	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2069	* @param[in] a_GCPtrMem The memory address.
2070	* @remarks Will return/long jump on errors.
2071	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2072	*/
2073	#ifndef IEM_WITH_SETJMP
2074	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2075	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2076	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2077	#else
2078	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2079	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2080	#endif
2081
2082	/**
2083	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2084	* address variant.
2085	*
2086	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2087	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2088	* @param[in] a_GCPtrMem The memory address.
2089	* @remarks Will return/long jump on errors.
2090	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2091	*/
2092	#ifndef IEM_WITH_SETJMP
2093	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2094	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2095	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2096	#else
2097	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2098	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2099	#endif
2100
2101	/**
2102	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2103	* address variant.
2104	*
2105	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2106	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2107	* @param[in] a_GCPtrMem The memory address.
2108	* @remarks Will return/long jump on errors.
2109	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2110	*/
2111	#ifndef IEM_WITH_SETJMP
2112	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2113	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2114	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2115	#else
2116	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2117	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2118	#endif
2119
2120	/** int32_t alias. */
2121	#ifndef IEM_WITH_SETJMP
2122	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2123	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2124	#else
2125	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2126	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2127	#endif
2128
2129	/** Flat int32_t alias. */
2130	#ifndef IEM_WITH_SETJMP
2131	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2132	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2133	#else
2134	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2135	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2136	#endif
2137
2138	/** RTFLOAT32U alias. */
2139	#ifndef IEM_WITH_SETJMP
2140	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2141	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2142	#else
2143	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2144	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2145	#endif
2146
2147	/** Flat RTFLOAT32U alias. */
2148	#ifndef IEM_WITH_SETJMP
2149	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2150	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2151	#else
2152	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2153	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2154	#endif
2155
2156
2157	/* 64-bit */
2158
2159	/**
2160	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2161	*
2162	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2163	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2164	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2165	* @param[in] a_GCPtrMem The memory address.
2166	* @remarks Will return/long jump on errors.
2167	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2168	*/
2169	#ifndef IEM_WITH_SETJMP
2170	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2171	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2172	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2173	#else
2174	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2175	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2176	#endif
2177
2178	/**
2179	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2180	*
2181	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2182	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2183	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2184	* @param[in] a_GCPtrMem The memory address.
2185	* @remarks Will return/long jump on errors.
2186	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2187	*/
2188	#ifndef IEM_WITH_SETJMP
2189	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2190	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2191	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2192	#else
2193	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2194	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2195	#endif
2196
2197	/**
2198	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2199	*
2200	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2201	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2202	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2203	* @param[in] a_GCPtrMem The memory address.
2204	* @remarks Will return/long jump on errors.
2205	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2206	*/
2207	#ifndef IEM_WITH_SETJMP
2208	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2209	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2210	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2211	#else
2212	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2213	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2214	#endif
2215
2216	/**
2217	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2218	*
2219	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2220	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2221	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2222	* @param[in] a_GCPtrMem The memory address.
2223	* @remarks Will return/long jump on errors.
2224	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2225	*/
2226	#ifndef IEM_WITH_SETJMP
2227	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2228	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2229	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2230	#else
2231	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2232	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2233	#endif
2234
2235	/**
2236	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2237	* acccess, flat address variant.
2238	*
2239	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2240	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2241	* @param[in] a_GCPtrMem The memory address.
2242	* @remarks Will return/long jump on errors.
2243	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2244	*/
2245	#ifndef IEM_WITH_SETJMP
2246	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2247	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2248	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2249	#else
2250	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2251	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2252	#endif
2253
2254	/**
2255	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2256	* flat address variant.
2257	*
2258	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2259	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2260	* @param[in] a_GCPtrMem The memory address.
2261	* @remarks Will return/long jump on errors.
2262	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2263	*/
2264	#ifndef IEM_WITH_SETJMP
2265	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2266	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2267	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2268	#else
2269	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2270	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2271	#endif
2272
2273	/**
2274	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2275	* address variant.
2276	*
2277	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2278	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2279	* @param[in] a_GCPtrMem The memory address.
2280	* @remarks Will return/long jump on errors.
2281	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2282	*/
2283	#ifndef IEM_WITH_SETJMP
2284	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2285	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2286	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2287	#else
2288	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2289	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2290	#endif
2291
2292	/**
2293	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2294	* address variant.
2295	*
2296	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2297	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2298	* @param[in] a_GCPtrMem The memory address.
2299	* @remarks Will return/long jump on errors.
2300	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2301	*/
2302	#ifndef IEM_WITH_SETJMP
2303	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2304	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2305	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2306	#else
2307	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2308	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2309	#endif
2310
2311	/** int64_t alias. */
2312	#ifndef IEM_WITH_SETJMP
2313	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2314	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2315	#else
2316	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2317	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2318	#endif
2319
2320	/** Flat int64_t alias. */
2321	#ifndef IEM_WITH_SETJMP
2322	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2323	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2324	#else
2325	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2326	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2327	#endif
2328
2329	/** RTFLOAT64U alias. */
2330	#ifndef IEM_WITH_SETJMP
2331	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2332	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2333	#else
2334	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2335	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2336	#endif
2337
2338	/** Flat RTFLOAT64U alias. */
2339	#ifndef IEM_WITH_SETJMP
2340	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2341	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2342	#else
2343	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2344	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2345	#endif
2346
2347
2348	/* 128-bit */
2349
2350	/**
2351	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2352	*
2353	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2354	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2355	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2356	* @param[in] a_GCPtrMem The memory address.
2357	* @remarks Will return/long jump on errors.
2358	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2359	*/
2360	#ifndef IEM_WITH_SETJMP
2361	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2362	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2363	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2364	#else
2365	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2366	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2367	#endif
2368
2369	/**
2370	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2371	*
2372	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2373	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2374	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2375	* @param[in] a_GCPtrMem The memory address.
2376	* @remarks Will return/long jump on errors.
2377	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2378	*/
2379	#ifndef IEM_WITH_SETJMP
2380	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2381	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2382	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2383	#else
2384	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2385	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2386	#endif
2387
2388	/**
2389	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2390	*
2391	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2392	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2393	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2394	* @param[in] a_GCPtrMem The memory address.
2395	* @remarks Will return/long jump on errors.
2396	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2397	*/
2398	#ifndef IEM_WITH_SETJMP
2399	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2400	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2401	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2402	#else
2403	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2404	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2405	#endif
2406
2407	/**
2408	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2409	*
2410	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2411	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2412	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2413	* @param[in] a_GCPtrMem The memory address.
2414	* @remarks Will return/long jump on errors.
2415	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2416	*/
2417	#ifndef IEM_WITH_SETJMP
2418	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2419	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2420	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2421	#else
2422	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2423	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2424	#endif
2425
2426	/**
2427	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2428	* access, flat address variant.
2429	*
2430	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2431	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2432	* @param[in] a_GCPtrMem The memory address.
2433	* @remarks Will return/long jump on errors.
2434	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2435	*/
2436	#ifndef IEM_WITH_SETJMP
2437	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2438	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2439	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2440	#else
2441	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2442	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2443	#endif
2444
2445	/**
2446	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2447	* flat address variant.
2448	*
2449	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2450	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2451	* @param[in] a_GCPtrMem The memory address.
2452	* @remarks Will return/long jump on errors.
2453	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2454	*/
2455	#ifndef IEM_WITH_SETJMP
2456	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2457	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2458	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2459	#else
2460	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2461	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2462	#endif
2463
2464	/**
2465	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2466	* flat address variant.
2467	*
2468	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2469	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2470	* @param[in] a_GCPtrMem The memory address.
2471	* @remarks Will return/long jump on errors.
2472	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2473	*/
2474	#ifndef IEM_WITH_SETJMP
2475	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2476	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2477	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2478	#else
2479	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2480	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2481	#endif
2482
2483	/**
2484	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2485	* address variant.
2486	*
2487	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2488	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2489	* @param[in] a_GCPtrMem The memory address.
2490	* @remarks Will return/long jump on errors.
2491	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2492	*/
2493	#ifndef IEM_WITH_SETJMP
2494	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2495	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2496	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2497	#else
2498	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2499	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2500	#endif
2501
2502
2503	/* misc */
2504
2505	/**
2506	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2507	*
2508	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2509	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2510	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2511	* @param[in] a_GCPtrMem The memory address.
2512	* @remarks Will return/long jump on errors.
2513	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2514	*/
2515	#ifndef IEM_WITH_SETJMP
2516	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2517	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2518	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2519	#else
2520	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2521	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2522	#endif
2523
2524	/**
2525	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2526	*
2527	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2528	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2529	* @param[in] a_GCPtrMem The memory address.
2530	* @remarks Will return/long jump on errors.
2531	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2532	*/
2533	#ifndef IEM_WITH_SETJMP
2534	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2535	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2536	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2537	#else
2538	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2539	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2540	#endif
2541
2542
2543	/**
2544	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2545	*
2546	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2547	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2548	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2549	* @param[in] a_GCPtrMem The memory address.
2550	* @remarks Will return/long jump on errors.
2551	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2552	*/
2553	#ifndef IEM_WITH_SETJMP
2554	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2555	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2556	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2557	#else
2558	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2559	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2560	#endif
2561
2562	/**
2563	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2564	*
2565	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2566	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2567	* @param[in] a_GCPtrMem The memory address.
2568	* @remarks Will return/long jump on errors.
2569	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2570	*/
2571	#ifndef IEM_WITH_SETJMP
2572	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2573	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2574	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2575	#else
2576	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2577	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2578	#endif
2579
2580
2581
2582	/* commit + unmap */
2583
2584	/** Commits the memory and unmaps guest memory previously mapped RW.
2585	* @remarks May return.
2586	* @note Implictly frees the a_bMapInfo variable.
2587	*/
2588	#ifndef IEM_WITH_SETJMP
2589	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2590	#else
2591	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2592	#endif
2593
2594	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2595	* @remarks May return.
2596	* @note Implictly frees the a_bMapInfo variable.
2597	*/
2598	#ifndef IEM_WITH_SETJMP
2599	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2600	#else
2601	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2602	#endif
2603
2604	/** Commits the memory and unmaps guest memory previously mapped W.
2605	* @remarks May return.
2606	* @note Implictly frees the a_bMapInfo variable.
2607	*/
2608	#ifndef IEM_WITH_SETJMP
2609	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2610	#else
2611	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2612	#endif
2613
2614	/** Commits the memory and unmaps guest memory previously mapped R.
2615	* @remarks May return.
2616	* @note Implictly frees the a_bMapInfo variable.
2617	*/
2618	#ifndef IEM_WITH_SETJMP
2619	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2620	#else
2621	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2622	#endif
2623
2624
2625	/** Commits the memory and unmaps the guest memory unless the FPU status word
2626	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2627	* that would cause FLD not to store.
2628	*
2629	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2630	* store, while \#P will not.
2631	*
2632	* @remarks May in theory return - for now.
2633	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2634	*/
2635	#ifndef IEM_WITH_SETJMP
2636	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2637	if ( !(a_u16FSW & X86_FSW_ES) \
2638	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2639	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2640	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2641	else \
2642	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2643	} while (0)
2644	#else
2645	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2646	if ( !(a_u16FSW & X86_FSW_ES) \
2647	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2648	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2649	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2650	else \
2651	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2652	} while (0)
2653	#endif
2654
2655	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2656	* @note Implictly frees the a_bMapInfo variable. */
2657	#ifndef IEM_WITH_SETJMP
2658	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2659	#else
2660	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2661	#endif
2662
2663
2664
2665	/** Calculate efficient address from R/M. */
2666	#ifndef IEM_WITH_SETJMP
2667	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2668	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2669	#else
2670	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2671	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2672	#endif
2673
2674
2675	/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2676	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2677	#define IEM_MC_NATIVE_ELSE() } else {
2678	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2679
2680	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2681	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2682	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2683	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2684	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2685	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2686	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2687	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2688	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2689
2690	/** This can be used to direct the register allocator when dealing with
2691	* x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2692	#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2693
2694
2695	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2696	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2697	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2698	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2699	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2700	#define IEM_MC_CALL_AIMPL_3(a_rcType, a_rc, a_pfn, a0, a1, a2) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2))
2701	#define IEM_MC_CALL_AIMPL_4(a_rcType, a_rc, a_pfn, a0, a1, a2, a3) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2), (a3))
2702
2703
2704	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2705	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2706	*/
2707	#ifdef VBOX_STRICT
2708	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2709	do { \
2710	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2711	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2712	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2713	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2714	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2715	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2716	if (rcStrictHlp == VINF_SUCCESS) \
2717	{ \
2718	uint64_t const fRipMask = (pVCpu->iem.s.fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT ? UINT64_MAX : UINT32_MAX; \
2719	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2720	\|\| ( ((uRipBefore + cbInstr) & fRipMask) == pVCpu->cpum.GstCtx.rip \
2721	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2722	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2723	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2724	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2725	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2726	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, (uRipBefore + cbInstr) & fRipMask)); \
2727	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2728	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2729	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2730	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2731	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2732	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2733	else \
2734	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2735	== (fEflBefore & ~(X86_EFL_RF)), \
2736	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2737	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2738	{ \
2739	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2740	AssertMsg( fExecBefore == fExecRecalc \
2741	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2742	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2743	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2744	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2745	} \
2746	} \
2747	return rcStrictHlp; \
2748	} while (0)
2749	#else
2750	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2751	#endif
2752
2753	/**
2754	* Defers the rest of the instruction emulation to a C implementation routine
2755	* and returns, only taking the standard parameters.
2756	*
2757	* @param a_fFlags IEM_CIMPL_F_XXX.
2758	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2759	* in the native recompiler.
2760	* @param a_pfnCImpl The pointer to the C routine.
2761	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2762	*/
2763	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2764	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2765
2766	/**
2767	* Defers the rest of instruction emulation to a C implementation routine and
2768	* returns, taking one argument in addition to the standard ones.
2769	*
2770	* @param a_fFlags IEM_CIMPL_F_XXX.
2771	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2772	* in the native recompiler.
2773	* @param a_pfnCImpl The pointer to the C routine.
2774	* @param a0 The argument.
2775	*/
2776	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2777	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2778
2779	/**
2780	* Defers the rest of the instruction emulation to a C implementation routine
2781	* and returns, taking two arguments in addition to the standard ones.
2782	*
2783	* @param a_fFlags IEM_CIMPL_F_XXX.
2784	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2785	* in the native recompiler.
2786	* @param a_pfnCImpl The pointer to the C routine.
2787	* @param a0 The first extra argument.
2788	* @param a1 The second extra argument.
2789	*/
2790	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2791	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2792
2793	/**
2794	* Defers the rest of the instruction emulation to a C implementation routine
2795	* and returns, taking three arguments in addition to the standard ones.
2796	*
2797	* @param a_fFlags IEM_CIMPL_F_XXX.
2798	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2799	* in the native recompiler.
2800	* @param a_pfnCImpl The pointer to the C routine.
2801	* @param a0 The first extra argument.
2802	* @param a1 The second extra argument.
2803	* @param a2 The third extra argument.
2804	*/
2805	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2806	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2807
2808	/**
2809	* Defers the rest of the instruction emulation to a C implementation routine
2810	* and returns, taking four arguments in addition to the standard ones.
2811	*
2812	* @param a_fFlags IEM_CIMPL_F_XXX.
2813	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2814	* in the native recompiler.
2815	* @param a_pfnCImpl The pointer to the C routine.
2816	* @param a0 The first extra argument.
2817	* @param a1 The second extra argument.
2818	* @param a2 The third extra argument.
2819	* @param a3 The fourth extra argument.
2820	*/
2821	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2822	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2823
2824	/**
2825	* Defers the rest of the instruction emulation to a C implementation routine
2826	* and returns, taking two arguments in addition to the standard ones.
2827	*
2828	* @param a_fFlags IEM_CIMPL_F_XXX.
2829	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2830	* in the native recompiler.
2831	* @param a_pfnCImpl The pointer to the C routine.
2832	* @param a0 The first extra argument.
2833	* @param a1 The second extra argument.
2834	* @param a2 The third extra argument.
2835	* @param a3 The fourth extra argument.
2836	* @param a4 The fifth extra argument.
2837	*/
2838	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2839	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2840
2841	/**
2842	* Defers the entire instruction emulation to a C implementation routine and
2843	* returns, only taking the standard parameters.
2844	*
2845	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2846	*
2847	* @param a_fFlags IEM_CIMPL_F_XXX.
2848	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2849	* in the native recompiler.
2850	* @param a_pfnCImpl The pointer to the C routine.
2851	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2852	*/
2853	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2854	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2855
2856	/**
2857	* Defers the entire instruction emulation to a C implementation routine and
2858	* returns, taking one argument in addition to the standard ones.
2859	*
2860	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2861	*
2862	* @param a_fFlags IEM_CIMPL_F_XXX.
2863	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2864	* in the native recompiler.
2865	* @param a_pfnCImpl The pointer to the C routine.
2866	* @param a0 The argument.
2867	*/
2868	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2869	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2870
2871	/**
2872	* Defers the entire instruction emulation to a C implementation routine and
2873	* returns, taking two arguments in addition to the standard ones.
2874	*
2875	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2876	*
2877	* @param a_fFlags IEM_CIMPL_F_XXX.
2878	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2879	* in the native recompiler.
2880	* @param a_pfnCImpl The pointer to the C routine.
2881	* @param a0 The first extra argument.
2882	* @param a1 The second extra argument.
2883	*/
2884	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2885	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2886
2887	/**
2888	* Defers the entire instruction emulation to a C implementation routine and
2889	* returns, taking three arguments in addition to the standard ones.
2890	*
2891	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2892	*
2893	* @param a_fFlags IEM_CIMPL_F_XXX.
2894	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2895	* in the native recompiler.
2896	* @param a_pfnCImpl The pointer to the C routine.
2897	* @param a0 The first extra argument.
2898	* @param a1 The second extra argument.
2899	* @param a2 The third extra argument.
2900	*/
2901	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2902	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2903
2904
2905	/**
2906	* Calls a FPU assembly implementation taking one visible argument.
2907	*
2908	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2909	* @param a0 The first extra argument.
2910	*/
2911	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2912	do { \
2913	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2914	} while (0)
2915
2916	/**
2917	* Calls a FPU assembly implementation taking two visible arguments.
2918	*
2919	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2920	* @param a0 The first extra argument.
2921	* @param a1 The second extra argument.
2922	*/
2923	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2924	do { \
2925	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2926	} while (0)
2927
2928	/**
2929	* Calls a FPU assembly implementation taking three visible arguments.
2930	*
2931	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2932	* @param a0 The first extra argument.
2933	* @param a1 The second extra argument.
2934	* @param a2 The third extra argument.
2935	*/
2936	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2937	do { \
2938	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2939	} while (0)
2940
2941	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2942	do { \
2943	(a_FpuData).FSW = (a_FSW); \
2944	(a_FpuData).r80Result = *(a_pr80Value); \
2945	} while (0)
2946
2947	/** Pushes FPU result onto the stack. */
2948	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2949	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2950	/** Pushes FPU result onto the stack and sets the FPUDP. */
2951	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2952	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2953
2954	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2955	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2956	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2957
2958	/** Stores FPU result in a stack register. */
2959	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2960	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2961	/** Stores FPU result in a stack register and pops the stack. */
2962	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2963	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2964	/** Stores FPU result in a stack register and sets the FPUDP. */
2965	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2966	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2967	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2968	* stack. */
2969	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2970	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2971
2972	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2973	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2974	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2975	/** Free a stack register (for FFREE and FFREEP). */
2976	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2977	iemFpuStackFree(pVCpu, a_iStReg)
2978	/** Increment the FPU stack pointer. */
2979	#define IEM_MC_FPU_STACK_INC_TOP() \
2980	iemFpuStackIncTop(pVCpu)
2981	/** Decrement the FPU stack pointer. */
2982	#define IEM_MC_FPU_STACK_DEC_TOP() \
2983	iemFpuStackDecTop(pVCpu)
2984
2985	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2986	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2987	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2988	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2989	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2990	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2991	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2992	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2993	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2994	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2995	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2996	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2997	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2998	* stack. */
2999	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3000	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3001	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
3002	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
3003	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3004
3005	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
3006	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
3007	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
3008	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3009	* stack. */
3010	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
3011	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
3012	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3013	* FPUDS. */
3014	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3015	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3016	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3017	* FPUDS. Pops stack. */
3018	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3019	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3020	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3021	* stack twice. */
3022	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
3023	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
3024	/** Raises a FPU stack underflow exception for an instruction pushing a result
3025	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
3026	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
3027	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
3028	/** Raises a FPU stack underflow exception for an instruction pushing a result
3029	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
3030	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
3031	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
3032
3033	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3034	* FPUIP, FPUCS and FOP. */
3035	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
3036	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
3037	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3038	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
3039	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3040	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3041	/** Prepares for using the FPU state.
3042	* Ensures that we can use the host FPU in the current context (RC+R0.
3043	* Ensures the guest FPU state in the CPUMCTX is up to date. */
3044	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
3045	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
3046	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
3047	/** Actualizes the guest FPU state so it can be accessed and modified. */
3048	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
3049
3050	/** Prepares for using the SSE state.
3051	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
3052	* Ensures the guest SSE state in the CPUMCTX is up to date. */
3053	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
3054	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3055	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3056	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3057	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3058
3059	/** Prepares for using the AVX state.
3060	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3061	* Ensures the guest AVX state in the CPUMCTX is up to date.
3062	* @note This will include the AVX512 state too when support for it is added
3063	* due to the zero extending feature of VEX instruction. */
3064	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3065	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3066	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3067	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3068	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3069
3070	/**
3071	* Calls a MMX assembly implementation taking two visible arguments.
3072	*
3073	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3074	* @param a0 The first extra argument.
3075	* @param a1 The second extra argument.
3076	*/
3077	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3078	do { \
3079	IEM_MC_PREPARE_FPU_USAGE(); \
3080	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3081	} while (0)
3082
3083	/**
3084	* Calls a MMX assembly implementation taking three visible arguments.
3085	*
3086	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3087	* @param a0 The first extra argument.
3088	* @param a1 The second extra argument.
3089	* @param a2 The third extra argument.
3090	*/
3091	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3092	do { \
3093	IEM_MC_PREPARE_FPU_USAGE(); \
3094	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3095	} while (0)
3096
3097
3098	/**
3099	* Calls a SSE assembly implementation taking two visible arguments.
3100	*
3101	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3102	* @param a0 The first extra argument.
3103	* @param a1 The second extra argument.
3104	*
3105	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3106	* which is unmasked in the guest's MXCSR.
3107	*/
3108	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3109	do { \
3110	IEM_MC_PREPARE_SSE_USAGE(); \
3111	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3112	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3113	(a0), (a1)); \
3114	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3115	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3116	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3117	{ /* probable */ } \
3118	else \
3119	{ \
3120	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3121	return iemRaiseSimdFpException(pVCpu); \
3122	return iemRaiseUndefinedOpcode(pVCpu); \
3123	} \
3124	} while (0)
3125
3126	/**
3127	* Calls a SSE assembly implementation taking three visible arguments.
3128	*
3129	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3130	* @param a0 The first extra argument.
3131	* @param a1 The second extra argument.
3132	* @param a2 The third extra argument.
3133	*
3134	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3135	* which is unmasked in the guest's MXCSR.
3136	*/
3137	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3138	do { \
3139	IEM_MC_PREPARE_SSE_USAGE(); \
3140	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3141	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3142	(a0), (a1), (a2)); \
3143	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3144	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3145	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3146	{ /* probable */ } \
3147	else \
3148	{ \
3149	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3150	return iemRaiseSimdFpException(pVCpu); \
3151	return iemRaiseUndefinedOpcode(pVCpu); \
3152	} \
3153	} while (0)
3154
3155
3156	/**
3157	* Calls a AVX assembly implementation taking two visible arguments.
3158	*
3159	* There is one implicit zero'th argument, a pointer to the extended state.
3160	*
3161	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3162	* @param a0 The first extra argument.
3163	* @param a1 The second extra argument.
3164	*
3165	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3166	* which is unmasked in the guest's MXCSR.
3167	*/
3168	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3169	do { \
3170	IEM_MC_PREPARE_AVX_USAGE(); \
3171	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3172	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3173	(a0), (a1)); \
3174	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3175	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3176	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3177	{ /* probable */ } \
3178	else \
3179	{ \
3180	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3181	return iemRaiseSimdFpException(pVCpu); \
3182	return iemRaiseUndefinedOpcode(pVCpu); \
3183	} \
3184	} while (0)
3185
3186	/**
3187	* Calls a AVX assembly implementation taking three visible arguments.
3188	*
3189	* There is one implicit zero'th argument, a pointer to the extended state.
3190	*
3191	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3192	* @param a0 The first extra argument.
3193	* @param a1 The second extra argument.
3194	* @param a2 The third extra argument.
3195	*
3196	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3197	* which is unmasked in the guest's MXCSR.
3198	*/
3199	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3200	do { \
3201	IEM_MC_PREPARE_AVX_USAGE(); \
3202	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3203	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3204	(a0), (a1), (a2)); \
3205	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3206	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3207	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3208	{ /* probable */ } \
3209	else \
3210	{ \
3211	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3212	return iemRaiseSimdFpException(pVCpu); \
3213	return iemRaiseUndefinedOpcode(pVCpu); \
3214	} \
3215	} while (0)
3216
3217	/** @note Not for IOPL or IF testing. */
3218	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3219	/** @note Not for IOPL or IF testing. */
3220	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3221	/** @note Not for IOPL or IF testing. */
3222	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3223	/** @note Not for IOPL or IF testing. */
3224	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3225	/** @note Not for IOPL or IF testing. */
3226	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3227	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3228	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3229	/** @note Not for IOPL or IF testing. */
3230	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3231	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3232	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3233	/** @note Not for IOPL or IF testing. */
3234	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3235	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3236	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3237	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3238	/** @note Not for IOPL or IF testing. */
3239	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3240	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3241	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3242	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3243	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3244	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3245	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3246	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3247	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3248	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3249	/** @note Not for IOPL or IF testing. */
3250	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3251	if ( pVCpu->cpum.GstCtx.cx != 1 \
3252	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3253	/** @note Not for IOPL or IF testing. */
3254	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3255	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3256	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3257	/** @note Not for IOPL or IF testing. */
3258	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3259	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3260	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3261	/** @note Not for IOPL or IF testing. */
3262	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3263	if ( pVCpu->cpum.GstCtx.cx != 1 \
3264	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3265	/** @note Not for IOPL or IF testing. */
3266	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3267	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3268	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3269	/** @note Not for IOPL or IF testing. */
3270	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3271	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3272	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3273	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3274	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3275
3276	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3277	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3278	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3279	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3280	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3281	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3282	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3283	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3284	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3285	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3286	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3287	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3288	#define IEM_MC_IF_FCW_IM() \
3289	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3290
3291	#define IEM_MC_ELSE() } else {
3292	#define IEM_MC_ENDIF() } do {} while (0)
3293
3294
3295	/** Recompiler debugging: Flush guest register shadow copies. */
3296	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3297
3298	/** @} */
3299
3300	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3301

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 105447

Download in other formats: