IEMMc.h@ 101954

Last change on this file since 101954 was 101954, checked in by vboxsync, 13 months ago
VMM/IEM: Eliminated IEM_MC_ASSIGN_U8_SX_U64 and IEM_MC_ASSIGN_U32_SX_U64. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 134.2 KB

Line
1	/* $Id: IEMMc.h 101954 2023-11-08 02:31:04Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) return iemRegRipJumpU16AndFinishClearningRF((pVCpu), (a_u16NewIP))
76	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
77	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) return iemRegRipJumpU32AndFinishClearningRF((pVCpu), (a_u32NewIP))
78	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
79	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) return iemRegRipJumpU64AndFinishClearningRF((pVCpu), (a_u64NewIP))
80
81	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
82	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
83	do { \
84	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
85	{ /* probable */ } \
86	else return iemRaiseDeviceNotAvailable(pVCpu); \
87	} while (0)
88	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
89	do { \
90	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
91	{ /* probable */ } \
92	else return iemRaiseDeviceNotAvailable(pVCpu); \
93	} while (0)
94	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
95	do { \
96	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
97	{ /* probable */ } \
98	else return iemRaiseMathFault(pVCpu); \
99	} while (0)
100	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
101	do { \
102	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
103	be reduced to a single compare branch in the more probably code path. */ \
104	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
105	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
106	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
107	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
108	{ /* probable */ } \
109	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
110	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
111	return iemRaiseUndefinedOpcode(pVCpu); \
112	else \
113	return iemRaiseDeviceNotAvailable(pVCpu); \
114	} while (0)
115	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
116	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
117	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
118	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
119	do { \
120	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
121	single compare branch in the more probable code path. */ \
122	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
123	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
124	== X86_CR4_OSFXSR)) \
125	{ /* likely */ } \
126	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
127	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
128	return iemRaiseUndefinedOpcode(pVCpu); \
129	else \
130	return iemRaiseDeviceNotAvailable(pVCpu); \
131	} while (0)
132	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
133	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
134	do { \
135	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
136	single compare branch in the more probable code path. */ \
137	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
138	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
139	{ /* probable */ } \
140	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
141	return iemRaiseUndefinedOpcode(pVCpu); \
142	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
143	return iemRaiseDeviceNotAvailable(pVCpu); \
144	else \
145	return iemRaiseMathFault(pVCpu); \
146	} while (0)
147	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
148	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
149	* count the CPL into the TB key. However it is safe enough for now, as
150	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
151	* emitted for it. */
152	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
153	do { \
154	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
155	else return iemRaiseGeneralProtectionFault0(pVCpu); \
156	} while (0)
157	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
158	do { \
159	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
160	else return iemRaiseGeneralProtectionFault0(pVCpu); \
161	} while (0)
162	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
163	do { \
164	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
165	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
166	{ /* probable */ } \
167	else return iemRaiseUndefinedOpcode(pVCpu); \
168	} while (0)
169	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
170	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
171	do { \
172	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
173	else return iemRaiseGeneralProtectionFault0(pVCpu); \
174	} while (0)
175	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
176	do { \
177	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
178	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
179	{ /* probable */ } \
180	else \
181	{ \
182	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
183	return iemRaiseSimdFpException(pVCpu); \
184	return iemRaiseUndefinedOpcode(pVCpu); \
185	} \
186	} while (0)
187	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
188	do { \
189	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
190	return iemRaiseSimdFpException(pVCpu); \
191	return iemRaiseUndefinedOpcode(pVCpu); \
192	} while (0)
193
194
195	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
196	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
197	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
198	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
199	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
200	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
201	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
202	/** @note IEMAllInstPython.py duplicates the expansion. */
203	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
204	uint32_t a_Name; \
205	uint32_t *a_pName = &a_Name
206	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
207	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
208
209	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
210
211	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
212	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
213	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
214	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
215	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
216	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
217	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
218	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
219	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
220	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
221	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
222	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
223	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
224	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
225	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
226	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
227	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
228	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
229	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
230	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
231	} while (0)
232	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
233	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
234	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
235	} while (0)
236	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
237	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
238	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
239	} while (0)
240	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
241	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
242	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
243	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
244	} while (0)
245	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
246	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
247	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
248	} while (0)
249	/** @note Not for IOPL or IF testing or modification. */
250	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
251	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
252	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
253	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
254
255	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
256	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
257	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
258	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
259	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
260	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
261	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
262	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
263	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
264	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
265	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
266	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
267	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
268	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
269	} while (0)
270	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
271	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
272	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
273	} while (0)
274	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
275	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
276
277
278	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
279	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
280	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
281	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
282	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
283	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
284	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
285	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
286	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
287	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
288	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
289	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
290	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
291	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
292	/** @note Not for IOPL or IF testing or modification.
293	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
294	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
295	#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
296
297	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
298	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
299	do { \
300	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
301	*pu32Reg += (a_u32Value); \
302	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
303	} while (0)
304	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
305
306	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
307	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
308	do { \
309	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
310	*pu32Reg -= (a_u8Const); \
311	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
312	} while (0)
313	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
314	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
315
316	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
317	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
318	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
319	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
320	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
321	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
322	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
323
324	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
325	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
326	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
327	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
328
329	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
330	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
331	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
332
333	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
334	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
335	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
336
337	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
338	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
339	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
340
341	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
342
343	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
344	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
345	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
346
347	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
348
349	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
350
351	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
352	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
353	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
354	do { \
355	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
356	*pu32Reg &= (a_u32Value); \
357	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
358	} while (0)
359	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
360
361	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
362	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
363	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
364	do { \
365	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
366	*pu32Reg \|= (a_u32Value); \
367	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
368	} while (0)
369	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
370
371	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
372	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
373	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
374
375	/** @note Not for IOPL or IF modification. */
376	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
377	/** @note Not for IOPL or IF modification. */
378	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
379	/** @note Not for IOPL or IF modification. */
380	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
381
382	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
383
384	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
385	#define IEM_MC_FPU_TO_MMX_MODE() do { \
386	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
387	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
388	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
389	} while (0)
390
391	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
392	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
393	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
394	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
395	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
396	} while (0)
397
398	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
399	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
400	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
401	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
402	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
403	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
404	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
405	} while (0)
406	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
407	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
408	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
409	} while (0)
410	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
411	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
412	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
413	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
414	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
415	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
416	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
417	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
418	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
419	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
420
421	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
422	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
423	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
424	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
425	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
426	} while (0)
427	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
428	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
429	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
430	} while (0)
431	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
432	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
433	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
434	} while (0)
435	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
436	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
437	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
438	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
439	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
440	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
441	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
442	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iByte)]; } while (0)
443	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
444	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
445	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
446	} while (0)
447	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
448	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
449	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
450	} while (0)
451	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
452	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
453	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
454	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
455	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
456	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
457	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
458	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
459	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
460	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
461	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
462	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
463
464	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
465	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
466	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
467	} while (0)
468
469	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
470	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
471	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
472	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
473	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
474	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
475	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
476	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
477	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
478	} while (0)
479	#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
480	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
481
482	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
483	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
484	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
485	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
486	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
487	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
488	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
489	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
490	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
491	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
492	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
493	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
494	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
495	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
496	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
497	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
498	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
499	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
500	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
501	} while (0)
502	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
503	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
504	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
505	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
506	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
507	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
508	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
509	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
510	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
511	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
512	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
513	} while (0)
514	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
515	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
516	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
517	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
518	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
519	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
520	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
521	} while (0)
522	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
523	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
524	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
525	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
526	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
527	} while (0)
528
529	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
530	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
531	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
532	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
533	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
534	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
535	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
536	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
537	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
538	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
539	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
540	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
541	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
542	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
543	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
544	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
545	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
546	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
547	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
548	} while (0)
549
550	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
551	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
552	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
553	} while (0)
554	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
555	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
556	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
557	} while (0)
558	#define IEM_MC_FETCH_YREG_2ND_U64(a_u64Dst, a_iYRegSrc) \
559	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
560	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
561	} while (0)
562	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
563	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
564	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
565	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
566	} while (0)
567	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
568	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
569	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
570	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
571	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
572	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
573	} while (0)
574
575	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
576	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
577	if ((a_iDQword) == 0) \
578	{ \
579	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
580	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
581	} \
582	else \
583	{ \
584	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
585	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
586	} \
587	} while (0)
588
589	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
590	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
591	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
595	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
596	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
597	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
598	} while (0)
599	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
600	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
601	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
602	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
603	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
604	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
605	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
606	} while (0)
607	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
608	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
611	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
612	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
613	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
614	} while (0)
615	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
616	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
617	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
618	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
619	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
620	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
621	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
622	} while (0)
623
624	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
625	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
626	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
627	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
628	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
630	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
631	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
632	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
633	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
634	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
635	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
636	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
637	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
638	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
639	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
640	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
641	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
642	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
643	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
644	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
645	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
646	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
647	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
648	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
649	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
650	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
651	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
652	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
653	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
654	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
655	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
656	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
657	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
658	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
659	} while (0)
660	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
661	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
662	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
663	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
664	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
665	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
666	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
667	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
668	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
669	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
670	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
671	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
672	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
673	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
674	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
675	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
676	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
677	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
678	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
679	} while (0)
680	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
681	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
682	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
683	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
684	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
685	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
686	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
687	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
688	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
689	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
690	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
691	} while (0)
692	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
693	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
694	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
695	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
696	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
697	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
698	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
699	} while (0)
700	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
701	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
702	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
703	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
704	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
705	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
706	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
707	} while (0)
708
709	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
710	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
711	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
712	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
713	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
714	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
715	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
716	do { uintptr_t const iYRegTmp = (a_iYReg); \
717	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
718	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
719	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
720	} while (0)
721
722	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
723	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
724	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
725	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
726	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
727	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
728	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
729	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
730	} while (0)
731	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
732	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
733	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
734	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
735	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
736	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
737	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
738	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
739	} while (0)
740	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
741	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
742	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
743	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
744	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
745	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
746	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
747	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
748	} while (0)
749
750	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
751	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
752	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
753	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
754	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
755	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
756	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
757	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
758	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
759	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
760	} while (0)
761	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
762	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
763	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
764	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
765	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
766	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
767	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
768	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
769	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
770	} while (0)
771	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
772	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
773	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
774	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
775	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
776	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
777	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
778	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
779	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
780	} while (0)
781	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
782	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
783	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
784	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
785	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
787	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
788	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
789	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
790	} while (0)
791	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
792	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
793	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
796	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
797	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
798	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
799	} while (0)
800	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
801	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
802	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
803	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
804	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
807	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
808	} while (0)
809
810	#ifndef IEM_WITH_SETJMP
811	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
812	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
813	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
814	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
815	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
816	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
817	#else
818	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
819	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
820	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
821	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
822	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
823	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
824
825	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
826	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
827	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
828	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
829	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
830	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
831	#endif
832
833	#ifndef IEM_WITH_SETJMP
834	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
835	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
836	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
837	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
838	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
839	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
840	#else
841	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
842	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
843	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
844	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
845	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
846	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
847
848	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
849	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
850	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
851	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
852	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
853	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
854	#endif
855
856	#ifndef IEM_WITH_SETJMP
857	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
858	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
859	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
860	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
861	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
862	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
863	#else
864	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
865	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
866	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
867	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
868	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
869	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
870
871	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
872	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
873	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
874	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
875	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
876	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
877	#endif
878
879	#ifdef SOME_UNUSED_FUNCTION
880	# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
881	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
882	#endif
883
884	#ifndef IEM_WITH_SETJMP
885	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
886	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
887	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
888	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
889	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
890	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
891	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
892	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
893	#else
894	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
895	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
896	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
897	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
898	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
899	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
900	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
901	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
902
903	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
904	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
905	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
906	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
907	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
908	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
909	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
910	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
911	#endif
912
913	#ifndef IEM_WITH_SETJMP
914	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
915	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
916	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
917	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
918	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
919	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
920	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
921	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
922	#else
923	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
924	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
925	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
926	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
927	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
928	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
929	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
930	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
931
932	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
933	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
934	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
935	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
936	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
937	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), UINT8_MAX, (a_GCPtrMem))
938	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
939	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), UINT8_MAX, (a_GCPtrMem))
940	#endif
941
942	#ifndef IEM_WITH_SETJMP
943	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
944	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
945	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
946	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
947	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
948	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
949
950	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
951	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
952	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
953	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
954	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
955	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
956	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
957	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
958	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
959	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
960	#else
961	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
962	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
963	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
964	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
965	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
966	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
967
968	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
969	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
970	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
971	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
972	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
973	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
974	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
975	(a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
976	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
977	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
978
979	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
980	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
981	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
982	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
983	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
984	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
985
986	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
987	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
988	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
989	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
990	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
991	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
992	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
993	(a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
994	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
995	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem))
996	#endif
997
998	#ifndef IEM_WITH_SETJMP
999	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1000	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1001	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1002	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1003	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1004	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1005
1006	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1007	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1008	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1009	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1010	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1011	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1012	#else
1013	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1014	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1015	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1016	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1017	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1018	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1019
1020	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1021	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1022	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1023	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1024	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1025	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1026
1027	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1028	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1029	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1030	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1031	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1032	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1033
1034	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1035	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1036	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1037	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1038	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1039	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1040	#endif
1041
1042
1043
1044	#ifndef IEM_WITH_SETJMP
1045	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1046	do { \
1047	uint8_t u8Tmp; \
1048	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1049	(a_u16Dst) = u8Tmp; \
1050	} while (0)
1051	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1052	do { \
1053	uint8_t u8Tmp; \
1054	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1055	(a_u32Dst) = u8Tmp; \
1056	} while (0)
1057	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1058	do { \
1059	uint8_t u8Tmp; \
1060	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1061	(a_u64Dst) = u8Tmp; \
1062	} while (0)
1063	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1064	do { \
1065	uint16_t u16Tmp; \
1066	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1067	(a_u32Dst) = u16Tmp; \
1068	} while (0)
1069	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1070	do { \
1071	uint16_t u16Tmp; \
1072	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1073	(a_u64Dst) = u16Tmp; \
1074	} while (0)
1075	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1076	do { \
1077	uint32_t u32Tmp; \
1078	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1079	(a_u64Dst) = u32Tmp; \
1080	} while (0)
1081	#else /* IEM_WITH_SETJMP */
1082	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1083	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1084	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1085	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1086	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1087	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1088	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1089	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1090	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1091	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1092	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1093	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1094
1095	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1096	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1097	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1098	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1099	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1100	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1101	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1102	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1103	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1104	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1105	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1106	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1107	#endif /* IEM_WITH_SETJMP */
1108
1109	#ifndef IEM_WITH_SETJMP
1110	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1111	do { \
1112	uint8_t u8Tmp; \
1113	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1114	(a_u16Dst) = (int8_t)u8Tmp; \
1115	} while (0)
1116	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1117	do { \
1118	uint8_t u8Tmp; \
1119	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1120	(a_u32Dst) = (int8_t)u8Tmp; \
1121	} while (0)
1122	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1123	do { \
1124	uint8_t u8Tmp; \
1125	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1126	(a_u64Dst) = (int8_t)u8Tmp; \
1127	} while (0)
1128	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1129	do { \
1130	uint16_t u16Tmp; \
1131	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1132	(a_u32Dst) = (int16_t)u16Tmp; \
1133	} while (0)
1134	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1135	do { \
1136	uint16_t u16Tmp; \
1137	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1138	(a_u64Dst) = (int16_t)u16Tmp; \
1139	} while (0)
1140	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1141	do { \
1142	uint32_t u32Tmp; \
1143	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1144	(a_u64Dst) = (int32_t)u32Tmp; \
1145	} while (0)
1146	#else /* IEM_WITH_SETJMP */
1147	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1148	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1149	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1150	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1151	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1152	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1153	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1154	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1155	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1156	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1157	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1158	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1159
1160	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1161	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1162	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1163	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1164	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1165	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1166	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1167	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1168	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1169	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1170	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1171	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1172	#endif /* IEM_WITH_SETJMP */
1173
1174	#ifndef IEM_WITH_SETJMP
1175	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1176	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1177	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1178	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1179	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1180	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1181	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1182	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1183	#else
1184	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1185	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1186	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1187	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1188	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1189	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1190	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1191	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1192
1193	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1194	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1195	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1196	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1197	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1198	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1199	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1200	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1201	#endif
1202
1203	#ifndef IEM_WITH_SETJMP
1204	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1205	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1206	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1207	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1208	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1209	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1210	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1211	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1212	#else
1213	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1214	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1215	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1216	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1217	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1218	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1219	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1220	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1221
1222	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1223	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1224	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1225	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1226	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1227	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1228	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1229	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1230	#endif
1231
1232	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1233	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1234	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1235	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1236	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1237	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1238	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1239	do { \
1240	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1241	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1242	} while (0)
1243	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1244	do { \
1245	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1246	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1247	} while (0)
1248
1249	#ifndef IEM_WITH_SETJMP
1250	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1251	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1252	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1253	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1254	#else
1255	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1256	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1257	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1258	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1259
1260	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1261	iemMemStoreDataU128Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1262	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1263	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1264	#endif
1265
1266	#ifndef IEM_WITH_SETJMP
1267	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1268	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1269	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1270	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1271	#else
1272	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1273	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1274	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1275	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1276
1277	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1278	iemMemStoreDataU256Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1279	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1280	iemMemStoreDataU256AlignedAvxJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1281	#endif
1282
1283	/* Regular stack push and pop: */
1284	#ifndef IEM_WITH_SETJMP
1285	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1286	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1287	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1288	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1289
1290	# define IEM_MC_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1291	# define IEM_MC_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1292	# define IEM_MC_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1293	#else
1294	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1295	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1296	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1297	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1298
1299	# define IEM_MC_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemStackPopU16Jmp(pVCpu))
1300	# define IEM_MC_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemStackPopU32Jmp(pVCpu))
1301	# define IEM_MC_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemStackPopU64Jmp(pVCpu))
1302	#endif
1303
1304	/* 32-bit flat stack push and pop: */
1305	#ifndef IEM_WITH_SETJMP
1306	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1307	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1308	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1309
1310	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1311	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1312	#else
1313	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1314	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1315	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1316
1317	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat32StackPopU16Jmp(pVCpu))
1318	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemFlat32StackPopU32Jmp(pVCpu))
1319	#endif
1320
1321	/* 64-bit flat stack push and pop: */
1322	#ifndef IEM_WITH_SETJMP
1323	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1324	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1325
1326	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1327	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1328	#else
1329	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1330	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1331
1332	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat64StackPopU16Jmp(pVCpu))
1333	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemFlat64StackPopU64Jmp(pVCpu))
1334	#endif
1335
1336
1337	/** Maps guest memory for direct or bounce buffered access.
1338	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1339	* @remarks May return.
1340	* @deprecated
1341	*/
1342	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
1343	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), \
1344	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1345
1346	/** Flat variant of IEM_MC_MEM_MAP.
1347	* @deprecated
1348	*/
1349	#define IEM_MC_MEM_FLAT_MAP(a_pMem, a_fAccess, a_GCPtrMem, a_iArg) \
1350	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), UINT8_MAX, \
1351	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1352
1353	/** Maps guest memory for direct or bounce buffered access.
1354	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1355	* @remarks May return.
1356	* @deprecated
1357	*/
1358	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_cbAlign, a_iArg) \
1359	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), \
1360	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1361
1362	/** Flat variant of IEM_MC_MEM_MAP_EX.
1363	* @deprecated
1364	*/
1365	#define IEM_MC_MEM_FLAT_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_GCPtrMem, a_cbAlign, a_iArg) \
1366	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), UINT8_MAX, \
1367	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1368
1369	/** Commits the memory and unmaps the guest memory.
1370	* @remarks May return.
1371	* @deprecated
1372	*/
1373	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
1374	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
1375
1376
1377	/* 8-bit */
1378
1379	/**
1380	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1381	*
1382	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1383	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1384	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1385	* @param[in] a_GCPtrMem The memory address.
1386	* @remarks Will return/long jump on errors.
1387	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1388	*/
1389	#ifndef IEM_WITH_SETJMP
1390	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1391	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1392	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1393	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1394	} while (0)
1395	#else
1396	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1397	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1398	#endif
1399
1400	/**
1401	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1402	*
1403	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1404	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1405	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1406	* @param[in] a_GCPtrMem The memory address.
1407	* @remarks Will return/long jump on errors.
1408	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1409	*/
1410	#ifndef IEM_WITH_SETJMP
1411	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1412	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1413	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1414	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1415	} while (0)
1416	#else
1417	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1418	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1419	#endif
1420
1421	/**
1422	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1423	*
1424	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1425	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1426	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1427	* @param[in] a_GCPtrMem The memory address.
1428	* @remarks Will return/long jump on errors.
1429	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1430	*/
1431	#ifndef IEM_WITH_SETJMP
1432	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1433	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1434	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1435	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1436	} while (0)
1437	#else
1438	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1439	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1440	#endif
1441
1442	/**
1443	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1444	* address variant.
1445	*
1446	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1447	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1448	* @param[in] a_GCPtrMem The memory address.
1449	* @remarks Will return/long jump on errors.
1450	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1451	*/
1452	#ifndef IEM_WITH_SETJMP
1453	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1454	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1455	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1456	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1457	} while (0)
1458	#else
1459	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1460	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1461	#endif
1462
1463	/**
1464	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1465	* address variant.
1466	*
1467	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1468	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1469	* @param[in] a_GCPtrMem The memory address.
1470	* @remarks Will return/long jump on errors.
1471	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1472	*/
1473	#ifndef IEM_WITH_SETJMP
1474	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1475	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1476	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1477	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1478	} while (0)
1479	#else
1480	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1481	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1482	#endif
1483
1484	/**
1485	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1486	* address variant.
1487	*
1488	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1489	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1490	* @param[in] a_GCPtrMem The memory address.
1491	* @remarks Will return/long jump on errors.
1492	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1493	*/
1494	#ifndef IEM_WITH_SETJMP
1495	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1496	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1497	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1498	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1499	} while (0)
1500	#else
1501	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1502	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1503	#endif
1504
1505
1506	/* 16-bit */
1507
1508	/**
1509	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1510	*
1511	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1512	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1513	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1514	* @param[in] a_GCPtrMem The memory address.
1515	* @remarks Will return/long jump on errors.
1516	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1517	*/
1518	#ifndef IEM_WITH_SETJMP
1519	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1520	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1521	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1522	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1523	} while (0)
1524	#else
1525	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1526	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1527	#endif
1528
1529	/**
1530	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1531	*
1532	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1533	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1534	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1535	* @param[in] a_GCPtrMem The memory address.
1536	* @remarks Will return/long jump on errors.
1537	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1538	*/
1539	#ifndef IEM_WITH_SETJMP
1540	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1541	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1542	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1543	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1544	} while (0)
1545	#else
1546	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1547	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1548	#endif
1549
1550	/**
1551	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1552	*
1553	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1554	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1555	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1556	* @param[in] a_GCPtrMem The memory address.
1557	* @remarks Will return/long jump on errors.
1558	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1559	*/
1560	#ifndef IEM_WITH_SETJMP
1561	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1562	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1563	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1564	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1565	} while (0)
1566	#else
1567	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1568	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1569	#endif
1570
1571	/**
1572	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1573	* address variant.
1574	*
1575	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1576	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1577	* @param[in] a_GCPtrMem The memory address.
1578	* @remarks Will return/long jump on errors.
1579	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1580	*/
1581	#ifndef IEM_WITH_SETJMP
1582	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1583	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1584	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1585	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1586	} while (0)
1587	#else
1588	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1589	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1590	#endif
1591
1592	/**
1593	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1594	* address variant.
1595	*
1596	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1597	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1598	* @param[in] a_GCPtrMem The memory address.
1599	* @remarks Will return/long jump on errors.
1600	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1601	*/
1602	#ifndef IEM_WITH_SETJMP
1603	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1604	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1605	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1606	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1607	} while (0)
1608	#else
1609	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1610	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1611	#endif
1612
1613	/**
1614	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1615	* address variant.
1616	*
1617	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1618	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1619	* @param[in] a_GCPtrMem The memory address.
1620	* @remarks Will return/long jump on errors.
1621	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1622	*/
1623	#ifndef IEM_WITH_SETJMP
1624	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1625	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1626	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1627	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1628	} while (0)
1629	#else
1630	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1631	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1632	#endif
1633
1634
1635	/* 32-bit */
1636
1637	/**
1638	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1639	*
1640	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1641	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1642	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1643	* @param[in] a_GCPtrMem The memory address.
1644	* @remarks Will return/long jump on errors.
1645	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1646	*/
1647	#ifndef IEM_WITH_SETJMP
1648	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1649	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1650	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1651	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1652	} while (0)
1653	#else
1654	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1655	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1656	#endif
1657
1658	/**
1659	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1660	*
1661	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1662	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1663	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1664	* @param[in] a_GCPtrMem The memory address.
1665	* @remarks Will return/long jump on errors.
1666	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1667	*/
1668	#ifndef IEM_WITH_SETJMP
1669	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1670	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1671	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1672	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1673	} while (0)
1674	#else
1675	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1676	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1677	#endif
1678
1679	/**
1680	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1681	*
1682	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1683	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1684	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1685	* @param[in] a_GCPtrMem The memory address.
1686	* @remarks Will return/long jump on errors.
1687	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1688	*/
1689	#ifndef IEM_WITH_SETJMP
1690	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1691	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1692	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1693	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1694	} while (0)
1695	#else
1696	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1697	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1698	#endif
1699
1700	/**
1701	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
1702	* flat address variant.
1703	*
1704	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1705	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1706	* @param[in] a_GCPtrMem The memory address.
1707	* @remarks Will return/long jump on errors.
1708	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1709	*/
1710	#ifndef IEM_WITH_SETJMP
1711	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1712	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1713	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1714	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1715	} while (0)
1716	#else
1717	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1718	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1719	#endif
1720
1721	/**
1722	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
1723	* address variant.
1724	*
1725	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1726	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1727	* @param[in] a_GCPtrMem The memory address.
1728	* @remarks Will return/long jump on errors.
1729	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1730	*/
1731	#ifndef IEM_WITH_SETJMP
1732	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1733	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1734	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1735	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1736	} while (0)
1737	#else
1738	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1739	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1740	#endif
1741
1742	/**
1743	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
1744	* address variant.
1745	*
1746	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1747	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1748	* @param[in] a_GCPtrMem The memory address.
1749	* @remarks Will return/long jump on errors.
1750	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1751	*/
1752	#ifndef IEM_WITH_SETJMP
1753	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1754	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1755	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1756	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1757	} while (0)
1758	#else
1759	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1760	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1761	#endif
1762
1763
1764	/* 64-bit */
1765
1766	/**
1767	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
1768	*
1769	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1770	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1771	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1772	* @param[in] a_GCPtrMem The memory address.
1773	* @remarks Will return/long jump on errors.
1774	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1775	*/
1776	#ifndef IEM_WITH_SETJMP
1777	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1778	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1779	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1780	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1781	} while (0)
1782	#else
1783	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1784	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1785	#endif
1786
1787	/**
1788	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
1789	*
1790	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1791	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1792	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1793	* @param[in] a_GCPtrMem The memory address.
1794	* @remarks Will return/long jump on errors.
1795	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1796	*/
1797	#ifndef IEM_WITH_SETJMP
1798	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1799	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1800	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1801	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1802	} while (0)
1803	#else
1804	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1805	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1806	#endif
1807
1808	/**
1809	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
1810	*
1811	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1812	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1813	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1814	* @param[in] a_GCPtrMem The memory address.
1815	* @remarks Will return/long jump on errors.
1816	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1817	*/
1818	#ifndef IEM_WITH_SETJMP
1819	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1820	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1821	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1822	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1823	} while (0)
1824	#else
1825	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1826	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1827	#endif
1828
1829	/**
1830	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
1831	* flat address variant.
1832	*
1833	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1834	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1835	* @param[in] a_GCPtrMem The memory address.
1836	* @remarks Will return/long jump on errors.
1837	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1838	*/
1839	#ifndef IEM_WITH_SETJMP
1840	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1841	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1842	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1843	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1844	} while (0)
1845	#else
1846	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1847	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1848	#endif
1849
1850	/**
1851	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
1852	* address variant.
1853	*
1854	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1855	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1856	* @param[in] a_GCPtrMem The memory address.
1857	* @remarks Will return/long jump on errors.
1858	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1859	*/
1860	#ifndef IEM_WITH_SETJMP
1861	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1862	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1863	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1864	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1865	} while (0)
1866	#else
1867	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1868	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1869	#endif
1870
1871	/**
1872	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
1873	* address variant.
1874	*
1875	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1876	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1877	* @param[in] a_GCPtrMem The memory address.
1878	* @remarks Will return/long jump on errors.
1879	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1880	*/
1881	#ifndef IEM_WITH_SETJMP
1882	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1883	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1884	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1885	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1886	} while (0)
1887	#else
1888	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1889	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1890	#endif
1891
1892
1893	/* commit + unmap */
1894
1895	/** Commits the memory and unmaps guest memory previously mapped RW.
1896	* @remarks May return.
1897	*/
1898	#ifndef IEM_WITH_SETJMP
1899	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) do { \
1900	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4)) ); \
1901	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_RW)); \
1902	} while (0)
1903	#else
1904	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) \
1905	iemMemCommitAndUnmapRwJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1906	#endif
1907
1908	/** Commits the memory and unmaps guest memory previously mapped W.
1909	* @remarks May return.
1910	*/
1911	#ifndef IEM_WITH_SETJMP
1912	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) do { \
1913	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_WRITE << 4)) ); \
1914	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W)); \
1915	} while (0)
1916	#else
1917	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) \
1918	iemMemCommitAndUnmapWoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1919	#endif
1920
1921	/** Commits the memory and unmaps guest memory previously mapped R.
1922	* @remarks May return.
1923	*/
1924	#ifndef IEM_WITH_SETJMP
1925	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) do { \
1926	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_READ << 4)) ); \
1927	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (void *)(a_pvMem), IEM_ACCESS_DATA_R)); \
1928	} while (0)
1929	#else
1930	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) \
1931	iemMemCommitAndUnmapRoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1932	#endif
1933
1934
1935	/** Commits the memory and unmaps the guest memory unless the FPU status word
1936	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
1937	* that would cause FLD not to store.
1938	*
1939	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
1940	* store, while \#P will not.
1941	*
1942	* @remarks May in theory return - for now.
1943	*/
1944	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
1945	do { \
1946	if ( !(a_u16FSW & X86_FSW_ES) \
1947	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
1948	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
1949	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
1950	} while (0)
1951
1952
1953
1954	/** Calculate efficient address from R/M. */
1955	#ifndef IEM_WITH_SETJMP
1956	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1957	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
1958	#else
1959	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1960	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
1961	#endif
1962
1963	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
1964	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
1965	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
1966	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
1967	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
1968	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
1969	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
1970
1971
1972	/** @def IEM_MC_CALL_CIMPL_HLP_RET
1973	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
1974	*/
1975	#ifdef VBOX_STRICT
1976	#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
1977	do { \
1978	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
1979	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
1980	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
1981	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
1982	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
1983	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
1984	if (rcStrictHlp == VINF_SUCCESS) \
1985	{ \
1986	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
1987	\|\| ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
1988	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
1989	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
1990	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
1991	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
1992	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
1993	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
1994	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
1995	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
1996	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
1997	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
1998	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
1999	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2000	else \
2001	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2002	== (fEflBefore & ~(X86_EFL_RF)), \
2003	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2004	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2005	{ \
2006	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2007	AssertMsg( fExecBefore == fExecRecalc \
2008	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2009	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2010	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2011	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2012	} \
2013	} \
2014	return rcStrictHlp; \
2015	} while (0)
2016	#else
2017	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2018	#endif
2019
2020	/**
2021	* Defers the rest of the instruction emulation to a C implementation routine
2022	* and returns, only taking the standard parameters.
2023	*
2024	* @param a_fFlags IEM_CIMPL_F_XXX.
2025	* @param a_pfnCImpl The pointer to the C routine.
2026	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2027	*/
2028	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_pfnCImpl) \
2029	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2030
2031	/**
2032	* Defers the rest of instruction emulation to a C implementation routine and
2033	* returns, taking one argument in addition to the standard ones.
2034	*
2035	* @param a_fFlags IEM_CIMPL_F_XXX.
2036	* @param a_pfnCImpl The pointer to the C routine.
2037	* @param a0 The argument.
2038	*/
2039	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_pfnCImpl, a0) \
2040	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2041
2042	/**
2043	* Defers the rest of the instruction emulation to a C implementation routine
2044	* and returns, taking two arguments in addition to the standard ones.
2045	*
2046	* @param a_fFlags IEM_CIMPL_F_XXX.
2047	* @param a_pfnCImpl The pointer to the C routine.
2048	* @param a0 The first extra argument.
2049	* @param a1 The second extra argument.
2050	*/
2051	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_pfnCImpl, a0, a1) \
2052	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2053
2054	/**
2055	* Defers the rest of the instruction emulation to a C implementation routine
2056	* and returns, taking three arguments in addition to the standard ones.
2057	*
2058	* @param a_fFlags IEM_CIMPL_F_XXX.
2059	* @param a_pfnCImpl The pointer to the C routine.
2060	* @param a0 The first extra argument.
2061	* @param a1 The second extra argument.
2062	* @param a2 The third extra argument.
2063	*/
2064	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2065	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2066
2067	/**
2068	* Defers the rest of the instruction emulation to a C implementation routine
2069	* and returns, taking four arguments in addition to the standard ones.
2070	*
2071	* @param a_fFlags IEM_CIMPL_F_XXX.
2072	* @param a_pfnCImpl The pointer to the C routine.
2073	* @param a0 The first extra argument.
2074	* @param a1 The second extra argument.
2075	* @param a2 The third extra argument.
2076	* @param a3 The fourth extra argument.
2077	*/
2078	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
2079	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2080
2081	/**
2082	* Defers the rest of the instruction emulation to a C implementation routine
2083	* and returns, taking two arguments in addition to the standard ones.
2084	*
2085	* @param a_fFlags IEM_CIMPL_F_XXX.
2086	* @param a_pfnCImpl The pointer to the C routine.
2087	* @param a0 The first extra argument.
2088	* @param a1 The second extra argument.
2089	* @param a2 The third extra argument.
2090	* @param a3 The fourth extra argument.
2091	* @param a4 The fifth extra argument.
2092	*/
2093	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
2094	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2095
2096	/**
2097	* Defers the entire instruction emulation to a C implementation routine and
2098	* returns, only taking the standard parameters.
2099	*
2100	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2101	*
2102	* @param a_fFlags IEM_CIMPL_F_XXX.
2103	* @param a_pfnCImpl The pointer to the C routine.
2104	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2105	*/
2106	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) \
2107	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2108
2109	/**
2110	* Defers the entire instruction emulation to a C implementation routine and
2111	* returns, taking one argument in addition to the standard ones.
2112	*
2113	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2114	*
2115	* @param a_fFlags IEM_CIMPL_F_XXX.
2116	* @param a_pfnCImpl The pointer to the C routine.
2117	* @param a0 The argument.
2118	*/
2119	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_pfnCImpl, a0) \
2120	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2121
2122	/**
2123	* Defers the entire instruction emulation to a C implementation routine and
2124	* returns, taking two arguments in addition to the standard ones.
2125	*
2126	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2127	*
2128	* @param a_fFlags IEM_CIMPL_F_XXX.
2129	* @param a_pfnCImpl The pointer to the C routine.
2130	* @param a0 The first extra argument.
2131	* @param a1 The second extra argument.
2132	*/
2133	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_pfnCImpl, a0, a1) \
2134	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2135
2136	/**
2137	* Defers the entire instruction emulation to a C implementation routine and
2138	* returns, taking three arguments in addition to the standard ones.
2139	*
2140	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2141	*
2142	* @param a_fFlags IEM_CIMPL_F_XXX.
2143	* @param a_pfnCImpl The pointer to the C routine.
2144	* @param a0 The first extra argument.
2145	* @param a1 The second extra argument.
2146	* @param a2 The third extra argument.
2147	*/
2148	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2149	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2150
2151
2152	/**
2153	* Calls a FPU assembly implementation taking one visible argument.
2154	*
2155	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2156	* @param a0 The first extra argument.
2157	*/
2158	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2159	do { \
2160	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2161	} while (0)
2162
2163	/**
2164	* Calls a FPU assembly implementation taking two visible arguments.
2165	*
2166	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2167	* @param a0 The first extra argument.
2168	* @param a1 The second extra argument.
2169	*/
2170	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2171	do { \
2172	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2173	} while (0)
2174
2175	/**
2176	* Calls a FPU assembly implementation taking three visible arguments.
2177	*
2178	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2179	* @param a0 The first extra argument.
2180	* @param a1 The second extra argument.
2181	* @param a2 The third extra argument.
2182	*/
2183	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2184	do { \
2185	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2186	} while (0)
2187
2188	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2189	do { \
2190	(a_FpuData).FSW = (a_FSW); \
2191	(a_FpuData).r80Result = *(a_pr80Value); \
2192	} while (0)
2193
2194	/** Pushes FPU result onto the stack. */
2195	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2196	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2197	/** Pushes FPU result onto the stack and sets the FPUDP. */
2198	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2199	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2200
2201	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2202	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2203	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2204
2205	/** Stores FPU result in a stack register. */
2206	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2207	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2208	/** Stores FPU result in a stack register and pops the stack. */
2209	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2210	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2211	/** Stores FPU result in a stack register and sets the FPUDP. */
2212	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2213	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2214	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2215	* stack. */
2216	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2217	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2218
2219	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2220	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2221	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2222	/** Free a stack register (for FFREE and FFREEP). */
2223	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2224	iemFpuStackFree(pVCpu, a_iStReg)
2225	/** Increment the FPU stack pointer. */
2226	#define IEM_MC_FPU_STACK_INC_TOP() \
2227	iemFpuStackIncTop(pVCpu)
2228	/** Decrement the FPU stack pointer. */
2229	#define IEM_MC_FPU_STACK_DEC_TOP() \
2230	iemFpuStackDecTop(pVCpu)
2231
2232	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2233	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2234	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2235	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2236	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2237	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2238	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2239	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2240	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2241	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2242	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2243	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2244	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2245	* stack. */
2246	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2247	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2248	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2249	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2250	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2251
2252	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2253	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2254	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2255	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2256	* stack. */
2257	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2258	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2259	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2260	* FPUDS. */
2261	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2262	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2263	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2264	* FPUDS. Pops stack. */
2265	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2266	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2267	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2268	* stack twice. */
2269	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2270	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2271	/** Raises a FPU stack underflow exception for an instruction pushing a result
2272	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
2273	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2274	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2275	/** Raises a FPU stack underflow exception for an instruction pushing a result
2276	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2277	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2278	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2279
2280	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2281	* FPUIP, FPUCS and FOP. */
2282	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2283	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2284	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2285	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2286	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2287	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2288	/** Prepares for using the FPU state.
2289	* Ensures that we can use the host FPU in the current context (RC+R0.
2290	* Ensures the guest FPU state in the CPUMCTX is up to date. */
2291	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2292	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2293	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2294	/** Actualizes the guest FPU state so it can be accessed and modified. */
2295	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2296
2297	/** Stores SSE SIMD result updating MXCSR. */
2298	#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
2299	iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
2300	/** Updates MXCSR. */
2301	#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
2302	iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
2303
2304	/** Prepares for using the SSE state.
2305	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2306	* Ensures the guest SSE state in the CPUMCTX is up to date. */
2307	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2308	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2309	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
2310	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
2311	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
2312
2313	/** Prepares for using the AVX state.
2314	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
2315	* Ensures the guest AVX state in the CPUMCTX is up to date.
2316	* @note This will include the AVX512 state too when support for it is added
2317	* due to the zero extending feature of VEX instruction. */
2318	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
2319	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2320	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
2321	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
2322	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
2323
2324	/**
2325	* Calls a MMX assembly implementation taking two visible arguments.
2326	*
2327	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2328	* @param a0 The first extra argument.
2329	* @param a1 The second extra argument.
2330	*/
2331	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
2332	do { \
2333	IEM_MC_PREPARE_FPU_USAGE(); \
2334	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2335	} while (0)
2336
2337	/**
2338	* Calls a MMX assembly implementation taking three visible arguments.
2339	*
2340	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2341	* @param a0 The first extra argument.
2342	* @param a1 The second extra argument.
2343	* @param a2 The third extra argument.
2344	*/
2345	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2346	do { \
2347	IEM_MC_PREPARE_FPU_USAGE(); \
2348	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2349	} while (0)
2350
2351
2352	/**
2353	* Calls a SSE assembly implementation taking two visible arguments.
2354	*
2355	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2356	* @param a0 The first extra argument.
2357	* @param a1 The second extra argument.
2358	*/
2359	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
2360	do { \
2361	IEM_MC_PREPARE_SSE_USAGE(); \
2362	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2363	} while (0)
2364
2365	/**
2366	* Calls a SSE assembly implementation taking three visible arguments.
2367	*
2368	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2369	* @param a0 The first extra argument.
2370	* @param a1 The second extra argument.
2371	* @param a2 The third extra argument.
2372	*/
2373	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2374	do { \
2375	IEM_MC_PREPARE_SSE_USAGE(); \
2376	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2377	} while (0)
2378
2379
2380	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
2381	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ...
2382	* @note IEMAllInstPython.py duplicates the expansion. */
2383	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
2384	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
2385
2386	/**
2387	* Calls a AVX assembly implementation taking two visible arguments.
2388	*
2389	* There is one implicit zero'th argument, a pointer to the extended state.
2390	*
2391	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2392	* @param a1 The first extra argument.
2393	* @param a2 The second extra argument.
2394	*/
2395	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
2396	do { \
2397	IEM_MC_PREPARE_AVX_USAGE(); \
2398	a_pfnAImpl(pXState, (a1), (a2)); \
2399	} while (0)
2400
2401	/**
2402	* Calls a AVX assembly implementation taking three visible arguments.
2403	*
2404	* There is one implicit zero'th argument, a pointer to the extended state.
2405	*
2406	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2407	* @param a1 The first extra argument.
2408	* @param a2 The second extra argument.
2409	* @param a3 The third extra argument.
2410	*/
2411	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
2412	do { \
2413	IEM_MC_PREPARE_AVX_USAGE(); \
2414	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
2415	} while (0)
2416
2417	/** @note Not for IOPL or IF testing. */
2418	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
2419	/** @note Not for IOPL or IF testing. */
2420	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
2421	/** @note Not for IOPL or IF testing. */
2422	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
2423	/** @note Not for IOPL or IF testing. */
2424	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
2425	/** @note Not for IOPL or IF testing. */
2426	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
2427	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2428	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2429	/** @note Not for IOPL or IF testing. */
2430	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
2431	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2432	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2433	/** @note Not for IOPL or IF testing. */
2434	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
2435	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2436	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2437	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2438	/** @note Not for IOPL or IF testing. */
2439	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
2440	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2441	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2442	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2443	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
2444	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
2445	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
2446	/** @note Not for IOPL or IF testing. */
2447	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2448	if ( pVCpu->cpum.GstCtx.cx != 0 \
2449	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2450	/** @note Not for IOPL or IF testing. */
2451	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2452	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2453	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2454	/** @note Not for IOPL or IF testing. */
2455	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2456	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2457	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2458	/** @note Not for IOPL or IF testing. */
2459	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2460	if ( pVCpu->cpum.GstCtx.cx != 0 \
2461	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2462	/** @note Not for IOPL or IF testing. */
2463	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2464	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2465	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2466	/** @note Not for IOPL or IF testing. */
2467	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2468	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2469	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2470	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
2471	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
2472
2473	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
2474	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
2475	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2476	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2477	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
2478	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
2479	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2480	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2481	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
2482	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
2483	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
2484	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
2485	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
2486	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
2487	#define IEM_MC_IF_FCW_IM() \
2488	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
2489	#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
2490	if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
2491	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
2492
2493	#define IEM_MC_ELSE() } else {
2494	#define IEM_MC_ENDIF() } do {} while (0)
2495
2496	/** @} */
2497
2498	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
2499

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 101954

Download in other formats: