IEMMc.h@ 101399

Last change on this file since 101399 was 101387, checked in by vboxsync, 16 months ago
VMM/IEM: Added a new class of threaded function variants, the 16f/32f/64f variants that will clear RF (and vbox internal friends) and check for TF (and vbox internal friends). The variants w/o the 'f' after the bitcount will skip this test+branch. The motivation of this was to deal with this issue that the threaded recompiler level rather than try optimize away the test+branch++ code when generating native code, make the IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC32 a very simple place to start emitting native code (compared to IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC32_WITH_FLAGS). bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 137.8 KB

Line
1	/* $Id: IEMMc.h 101387 2023-10-07 23:34:54Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Advances RIP, finishes the instruction and returns.
59	* This may include raising debug exceptions and such. */
60	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
61	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
62	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
63	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
65	* @note only usable in 16-bit op size mode. */
66	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
67	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
68	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
69	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
70	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) return iemRegRipJumpU16AndFinishClearningRF((pVCpu), (a_u16NewIP))
73	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
74	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) return iemRegRipJumpU32AndFinishClearningRF((pVCpu), (a_u32NewIP))
75	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
76	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) return iemRegRipJumpU64AndFinishClearningRF((pVCpu), (a_u64NewIP))
77
78	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
79	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
80	do { \
81	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
82	{ /* probable */ } \
83	else return iemRaiseDeviceNotAvailable(pVCpu); \
84	} while (0)
85	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
86	do { \
87	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
88	{ /* probable */ } \
89	else return iemRaiseDeviceNotAvailable(pVCpu); \
90	} while (0)
91	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
92	do { \
93	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
94	{ /* probable */ } \
95	else return iemRaiseMathFault(pVCpu); \
96	} while (0)
97	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
98	do { \
99	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
100	be reduced to a single compare branch in the more probably code path. */ \
101	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
102	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
103	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
104	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
105	{ /* probable */ } \
106	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
107	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
108	return iemRaiseUndefinedOpcode(pVCpu); \
109	else \
110	return iemRaiseDeviceNotAvailable(pVCpu); \
111	} while (0)
112	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
113	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
114	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
115	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
116	do { \
117	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
118	single compare branch in the more probable code path. */ \
119	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
120	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
121	== X86_CR4_OSFXSR)) \
122	{ /* likely */ } \
123	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
124	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
125	return iemRaiseUndefinedOpcode(pVCpu); \
126	else \
127	return iemRaiseDeviceNotAvailable(pVCpu); \
128	} while (0)
129	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
130	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
131	do { \
132	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
133	single compare branch in the more probable code path. */ \
134	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
135	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
136	{ /* probable */ } \
137	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
138	return iemRaiseUndefinedOpcode(pVCpu); \
139	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
140	return iemRaiseDeviceNotAvailable(pVCpu); \
141	else \
142	return iemRaiseMathFault(pVCpu); \
143	} while (0)
144	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
145	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
146	* count the CPL into the TB key. However it is safe enough for now, as
147	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
148	* emitted for it. */
149	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
150	do { \
151	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
152	else return iemRaiseGeneralProtectionFault0(pVCpu); \
153	} while (0)
154	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
155	do { \
156	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
157	else return iemRaiseGeneralProtectionFault0(pVCpu); \
158	} while (0)
159	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
160	do { \
161	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
162	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
163	{ /* probable */ } \
164	else return iemRaiseUndefinedOpcode(pVCpu); \
165	} while (0)
166	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
167	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
168	do { \
169	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
170	else return iemRaiseGeneralProtectionFault0(pVCpu); \
171	} while (0)
172	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
173	do { \
174	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
175	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
176	{ /* probable */ } \
177	else \
178	{ \
179	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
180	return iemRaiseSimdFpException(pVCpu); \
181	return iemRaiseUndefinedOpcode(pVCpu); \
182	} \
183	} while (0)
184	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
185	do { \
186	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
187	return iemRaiseSimdFpException(pVCpu); \
188	return iemRaiseUndefinedOpcode(pVCpu); \
189	} while (0)
190
191
192	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
193	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
194	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
195	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
196	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
197	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
198	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
199	uint32_t a_Name; \
200	uint32_t *a_pName = &a_Name
201	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
202	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
203
204	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
205	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
206	#define IEM_MC_ASSIGN_U8_SX_U64(a_u64VarOrArg, a_u8CVariableOrConst) \
207	(a_u64VarOrArg) = (int8_t)(a_u8CVariableOrConst)
208	#define IEM_MC_ASSIGN_U32_SX_U64(a_u64VarOrArg, a_u32CVariableOrConst) \
209	(a_u64VarOrArg) = (int32_t)(a_u32CVariableOrConst)
210
211	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
212	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
213	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
214	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
215	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
216	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
217	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
218	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
219	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
220	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
221	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
222	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
223	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
224	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
225	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
226	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
227	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
228	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
229	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
230	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
231	} while (0)
232	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
233	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
234	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
235	} while (0)
236	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
237	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
238	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
239	} while (0)
240	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
241	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
242	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
243	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
244	} while (0)
245	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
246	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
247	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
248	} while (0)
249	/** @note Not for IOPL or IF testing or modification. */
250	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
251	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
252	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
253	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
254
255	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
256	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
257	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
258	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
259	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
260	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
261	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
262	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
263	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
264	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
265	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
266	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
267	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
268	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
269	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
270	} while (0)
271	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
272	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
273	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
274	} while (0)
275	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
276	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
277
278
279	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
280	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
281	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
282	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
283	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
284	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
285	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
286	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
287	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
288	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
289	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
290	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
291	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
292	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
293	/** @note Not for IOPL or IF testing or modification.
294	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
295	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
296	#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
297
298	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
299	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
300	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
301	do { \
302	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
303	*pu32Reg += (a_u32Value); \
304	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
305	} while (0)
306	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
307
308	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
309	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
310	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
311	do { \
312	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
313	*pu32Reg -= (a_u32Value); \
314	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
315	} while (0)
316	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
317	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
318
319	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
320	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
321	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
322	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
323	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
324	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
325	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
326
327	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
328	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
329	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
330	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
331
332	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
333	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
334	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
335
336	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
337	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
338	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
339
340	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
341	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
342	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
343
344	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
345
346	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
347	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
348	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
349
350	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
351
352	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
353
354	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
355	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
356	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
357	do { \
358	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
359	*pu32Reg &= (a_u32Value); \
360	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
361	} while (0)
362	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
363
364	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
365	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
366	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
367	do { \
368	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
369	*pu32Reg \|= (a_u32Value); \
370	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
371	} while (0)
372	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
373
374	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
375	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
376	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
377
378	/** @note Not for IOPL or IF modification. */
379	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
380	/** @note Not for IOPL or IF modification. */
381	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
382	/** @note Not for IOPL or IF modification. */
383	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
384
385	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
386
387	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
388	#define IEM_MC_FPU_TO_MMX_MODE() do { \
389	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
390	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
391	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
392	} while (0)
393
394	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
395	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
396	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
397	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
398	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
399	} while (0)
400
401	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
402	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
403	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
404	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
405	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
406	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
407	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
408	} while (0)
409	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
410	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
411	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
412	} while (0)
413	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
414	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
415	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
416	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
417	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
418	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
419	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
420	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
421	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
422	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
423
424	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
425	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
426	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
427	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
428	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
429	} while (0)
430	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
431	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
432	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
433	} while (0)
434	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
435	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
436	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
437	} while (0)
438	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
439	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
440	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
441	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
442	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
443	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
444	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
445	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iByte)]; } while (0)
446	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
447	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
448	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
449	} while (0)
450	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
451	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
452	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
453	} while (0)
454	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
455	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
456	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
457	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
458	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
459	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
460	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
461	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
462	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
463	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
464	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
465	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
466
467	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
468	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
469	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
470	} while (0)
471
472	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
473	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
474	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
475	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
476	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
477	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
478	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
479	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
480	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
481	} while (0)
482	#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
483	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
484
485	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
486	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
487	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
488	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
489	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
490	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
491	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
492	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
493	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
494	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
495	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
496	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
497	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
498	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
499	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
500	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
501	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
502	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
503	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
504	} while (0)
505	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
506	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
507	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
508	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
509	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
510	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
511	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
512	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
513	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
514	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
515	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
516	} while (0)
517	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
518	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
519	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
520	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
521	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
522	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
523	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
524	} while (0)
525	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
526	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
527	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
528	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
529	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
530	} while (0)
531
532	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
533	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
534	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
535	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
536	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
537	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
538	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
539	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
540	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
541	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
542	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
543	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
544	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
545	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
546	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
547	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
548	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
549	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
550	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
551	} while (0)
552
553	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
554	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
555	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
556	} while (0)
557	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
558	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
559	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
560	} while (0)
561	#define IEM_MC_FETCH_YREG_2ND_U64(a_u64Dst, a_iYRegSrc) \
562	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
563	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
564	} while (0)
565	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
566	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
567	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
568	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
569	} while (0)
570	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
571	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
572	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
573	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
574	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
575	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
576	} while (0)
577
578	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
579	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
580	if ((a_iDQword) == 0) \
581	{ \
582	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
583	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
584	} \
585	else \
586	{ \
587	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
588	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
589	} \
590	} while (0)
591
592	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
593	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
594	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
598	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
599	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
600	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
601	} while (0)
602	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
603	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
605	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
606	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
607	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
608	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
609	} while (0)
610	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
611	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
612	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
613	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
614	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
615	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
616	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
617	} while (0)
618	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
619	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
620	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
621	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
622	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
623	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
624	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
625	} while (0)
626
627	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
628	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
630	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
631	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
632	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
633	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
634	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
635	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
636	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
637	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
638	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
639	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
640	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
641	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
642	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
643	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
644	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
645	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
646	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
647	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
648	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
649	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
650	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
651	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
652	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
653	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
654	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
655	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
656	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
657	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
658	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
659	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
660	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
661	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
662	} while (0)
663	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
664	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
665	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
666	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
667	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
668	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
669	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
670	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
671	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
672	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
673	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
674	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
675	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
676	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
677	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
678	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
679	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
680	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
681	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
682	} while (0)
683	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
684	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
685	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
686	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
687	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
688	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
689	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
690	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
691	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
692	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
693	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
694	} while (0)
695	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
696	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
697	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
698	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
699	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
700	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
701	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
702	} while (0)
703	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
704	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
705	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
706	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
707	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
708	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
709	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
710	} while (0)
711
712	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
713	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
714	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
715	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
716	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
717	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
718	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
719	do { uintptr_t const iYRegTmp = (a_iYReg); \
720	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
721	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
722	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
723	} while (0)
724
725	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
726	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
727	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
728	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
729	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
730	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
731	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
732	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
733	} while (0)
734	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
735	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
736	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
737	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
738	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
739	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
740	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
741	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
742	} while (0)
743	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
744	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
745	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
747	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
749	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
750	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
751	} while (0)
752
753	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
754	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
755	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
756	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
757	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
758	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
759	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
760	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
761	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
762	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
763	} while (0)
764	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
765	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
766	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
767	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
768	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
769	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
770	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
771	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
772	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
773	} while (0)
774	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
775	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
776	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
777	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
778	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
779	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
780	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
781	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
782	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
783	} while (0)
784	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
785	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
786	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
787	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
790	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
791	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
792	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
793	} while (0)
794	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
795	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
796	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
798	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
799	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
800	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
801	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
802	} while (0)
803	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
804	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
805	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
806	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
807	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
809	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
810	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
811	} while (0)
812
813	#ifndef IEM_WITH_SETJMP
814	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
815	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
816	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
817	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
818	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
819	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
820	#else
821	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
822	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
823	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
824	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
825	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
826	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
827
828	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
829	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
830	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
831	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
832	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
833	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
834	#endif
835
836	#ifndef IEM_WITH_SETJMP
837	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
838	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
839	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
840	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
841	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
842	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
843	#else
844	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
845	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
846	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
847	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
848	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
849	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
850
851	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
852	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
853	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
854	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
855	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
856	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
857	#endif
858
859	#ifndef IEM_WITH_SETJMP
860	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
861	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
862	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
863	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
864	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
865	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
866	#else
867	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
868	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
869	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
870	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
871	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
872	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
873
874	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
875	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
876	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
877	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
878	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
879	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
880	#endif
881
882	#ifdef SOME_UNUSED_FUNCTION
883	# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
884	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
885	#endif
886
887	#ifndef IEM_WITH_SETJMP
888	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
889	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
890	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
891	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
892	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
893	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
894	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
895	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
896	#else
897	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
898	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
899	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
900	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
901	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
902	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
903	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
904	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
905
906	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
907	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
908	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
909	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
910	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
911	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
912	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
913	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
914	#endif
915
916	#ifndef IEM_WITH_SETJMP
917	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
918	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
919	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
920	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
921	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
922	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
923	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
924	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
925	#else
926	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
927	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
928	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
929	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
930	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
931	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
932	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
933	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
934
935	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
936	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
937	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
938	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
939	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
940	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), UINT8_MAX, (a_GCPtrMem))
941	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
942	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), UINT8_MAX, (a_GCPtrMem))
943	#endif
944
945	#ifndef IEM_WITH_SETJMP
946	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
947	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
948	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
949	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
950	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
951	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
952
953	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
954	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
955	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
956	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
957	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
958	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
959	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
960	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
961	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
962	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
963	#else
964	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
965	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
966	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
967	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
968	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
969	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
970
971	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
972	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
973	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
974	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
975	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
976	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
977	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
978	(a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
979	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
980	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
981
982	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
983	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
984	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
985	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
986	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
987	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
988
989	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
990	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
991	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
992	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
993	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
994	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
995	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
996	(a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
997	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
998	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem))
999	#endif
1000
1001	#ifndef IEM_WITH_SETJMP
1002	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1003	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1004	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1005	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1006	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1007	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1008
1009	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1010	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1011	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1012	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1013	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1014	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1015	#else
1016	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1017	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1018	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1019	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1020	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1021	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1022
1023	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1024	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1025	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1026	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1027	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1028	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1029
1030	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1031	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1032	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1033	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1034	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1035	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1036
1037	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1038	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1039	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1040	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1041	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1042	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1043	#endif
1044
1045
1046
1047	#ifndef IEM_WITH_SETJMP
1048	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1049	do { \
1050	uint8_t u8Tmp; \
1051	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1052	(a_u16Dst) = u8Tmp; \
1053	} while (0)
1054	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1055	do { \
1056	uint8_t u8Tmp; \
1057	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1058	(a_u32Dst) = u8Tmp; \
1059	} while (0)
1060	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1061	do { \
1062	uint8_t u8Tmp; \
1063	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1064	(a_u64Dst) = u8Tmp; \
1065	} while (0)
1066	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1067	do { \
1068	uint16_t u16Tmp; \
1069	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1070	(a_u32Dst) = u16Tmp; \
1071	} while (0)
1072	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1073	do { \
1074	uint16_t u16Tmp; \
1075	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1076	(a_u64Dst) = u16Tmp; \
1077	} while (0)
1078	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1079	do { \
1080	uint32_t u32Tmp; \
1081	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1082	(a_u64Dst) = u32Tmp; \
1083	} while (0)
1084	#else /* IEM_WITH_SETJMP */
1085	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1086	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1087	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1088	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1089	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1090	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1091	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1092	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1093	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1094	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1095	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1096	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1097
1098	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1099	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1100	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1101	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1102	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1103	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1104	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1105	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1106	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1107	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1108	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1109	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1110	#endif /* IEM_WITH_SETJMP */
1111
1112	#ifndef IEM_WITH_SETJMP
1113	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1114	do { \
1115	uint8_t u8Tmp; \
1116	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1117	(a_u16Dst) = (int8_t)u8Tmp; \
1118	} while (0)
1119	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1120	do { \
1121	uint8_t u8Tmp; \
1122	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1123	(a_u32Dst) = (int8_t)u8Tmp; \
1124	} while (0)
1125	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1126	do { \
1127	uint8_t u8Tmp; \
1128	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1129	(a_u64Dst) = (int8_t)u8Tmp; \
1130	} while (0)
1131	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1132	do { \
1133	uint16_t u16Tmp; \
1134	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1135	(a_u32Dst) = (int16_t)u16Tmp; \
1136	} while (0)
1137	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1138	do { \
1139	uint16_t u16Tmp; \
1140	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1141	(a_u64Dst) = (int16_t)u16Tmp; \
1142	} while (0)
1143	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1144	do { \
1145	uint32_t u32Tmp; \
1146	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1147	(a_u64Dst) = (int32_t)u32Tmp; \
1148	} while (0)
1149	#else /* IEM_WITH_SETJMP */
1150	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1151	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1152	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1153	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1154	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1155	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1156	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1157	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1158	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1159	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1160	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1161	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1162
1163	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1164	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1165	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1166	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1167	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1168	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1169	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1170	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1171	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1172	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1173	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1174	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1175	#endif /* IEM_WITH_SETJMP */
1176
1177	#ifndef IEM_WITH_SETJMP
1178	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1179	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1180	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1181	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1182	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1183	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1184	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1185	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1186	#else
1187	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1188	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1189	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1190	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1191	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1192	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1193	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1194	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1195
1196	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1197	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1198	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1199	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1200	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1201	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1202	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1203	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1204	#endif
1205
1206	#ifndef IEM_WITH_SETJMP
1207	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1208	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1209	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1210	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1211	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1212	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1213	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1214	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1215	#else
1216	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1217	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1218	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1219	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1220	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1221	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1222	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1223	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1224
1225	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1226	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1227	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1228	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1229	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1230	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1231	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1232	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1233	#endif
1234
1235	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1236	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1237	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1238	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1239	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1240	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1241	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1242	do { \
1243	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1244	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1245	} while (0)
1246	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1247	do { \
1248	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1249	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1250	} while (0)
1251
1252	#ifndef IEM_WITH_SETJMP
1253	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1254	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1255	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1256	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1257	#else
1258	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1259	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1260	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1261	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1262
1263	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1264	iemMemStoreDataU128Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1265	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1266	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1267	#endif
1268
1269	#ifndef IEM_WITH_SETJMP
1270	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1271	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1272	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1273	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1274	#else
1275	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1276	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1277	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1278	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1279
1280	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1281	iemMemStoreDataU256Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1282	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1283	iemMemStoreDataU256AlignedAvxJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1284	#endif
1285
1286	/* Regular stack push and pop: */
1287	#ifndef IEM_WITH_SETJMP
1288	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1289	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1290	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1291	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1292
1293	# define IEM_MC_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1294	# define IEM_MC_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1295	# define IEM_MC_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1296	#else
1297	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1298	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1299	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1300	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1301
1302	# define IEM_MC_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemStackPopU16Jmp(pVCpu))
1303	# define IEM_MC_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemStackPopU32Jmp(pVCpu))
1304	# define IEM_MC_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemStackPopU64Jmp(pVCpu))
1305	#endif
1306
1307	/* 32-bit flat stack push and pop: */
1308	#ifndef IEM_WITH_SETJMP
1309	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1310	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1311	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1312
1313	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1314	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1315	#else
1316	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1317	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1318	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1319
1320	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat32StackPopU16Jmp(pVCpu))
1321	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemFlat32StackPopU32Jmp(pVCpu))
1322	#endif
1323
1324	/* 64-bit flat stack push and pop: */
1325	#ifndef IEM_WITH_SETJMP
1326	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1327	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1328
1329	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1330	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1331	#else
1332	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1333	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1334
1335	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat64StackPopU16Jmp(pVCpu))
1336	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemFlat64StackPopU64Jmp(pVCpu))
1337	#endif
1338
1339
1340	/** Maps guest memory for direct or bounce buffered access.
1341	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1342	* @remarks May return.
1343	* @deprecated
1344	*/
1345	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
1346	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), \
1347	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1348
1349	/** Flat variant of IEM_MC_MEM_MAP.
1350	* @deprecated
1351	*/
1352	#define IEM_MC_MEM_FLAT_MAP(a_pMem, a_fAccess, a_GCPtrMem, a_iArg) \
1353	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), UINT8_MAX, \
1354	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1355
1356	/** Maps guest memory for direct or bounce buffered access.
1357	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1358	* @remarks May return.
1359	* @deprecated
1360	*/
1361	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_cbAlign, a_iArg) \
1362	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), \
1363	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1364
1365	/** Flat variant of IEM_MC_MEM_MAP_EX.
1366	* @deprecated
1367	*/
1368	#define IEM_MC_MEM_FLAT_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_GCPtrMem, a_cbAlign, a_iArg) \
1369	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), UINT8_MAX, \
1370	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1371
1372	/** Commits the memory and unmaps the guest memory.
1373	* @remarks May return.
1374	* @deprecated
1375	*/
1376	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
1377	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
1378
1379
1380	/* 8-bit */
1381
1382	/**
1383	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1384	*
1385	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1386	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1387	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1388	* @param[in] a_GCPtrMem The memory address.
1389	* @remarks Will return/long jump on errors.
1390	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1391	*/
1392	#ifndef IEM_WITH_SETJMP
1393	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1394	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1395	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1396	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1397	} while (0)
1398	#else
1399	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1400	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1401	#endif
1402
1403	/**
1404	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1405	*
1406	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1407	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1408	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1409	* @param[in] a_GCPtrMem The memory address.
1410	* @remarks Will return/long jump on errors.
1411	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1412	*/
1413	#ifndef IEM_WITH_SETJMP
1414	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1415	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1416	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1417	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1418	} while (0)
1419	#else
1420	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1421	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1422	#endif
1423
1424	/**
1425	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1426	*
1427	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1428	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1429	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1430	* @param[in] a_GCPtrMem The memory address.
1431	* @remarks Will return/long jump on errors.
1432	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1433	*/
1434	#ifndef IEM_WITH_SETJMP
1435	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1436	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1437	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1438	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1439	} while (0)
1440	#else
1441	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1442	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1443	#endif
1444
1445	/**
1446	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1447	* address variant.
1448	*
1449	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1450	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1451	* @param[in] a_GCPtrMem The memory address.
1452	* @remarks Will return/long jump on errors.
1453	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1454	*/
1455	#ifndef IEM_WITH_SETJMP
1456	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1457	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1458	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1459	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1460	} while (0)
1461	#else
1462	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1463	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1464	#endif
1465
1466	/**
1467	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1468	* address variant.
1469	*
1470	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1471	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1472	* @param[in] a_GCPtrMem The memory address.
1473	* @remarks Will return/long jump on errors.
1474	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1475	*/
1476	#ifndef IEM_WITH_SETJMP
1477	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1478	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1479	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1480	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1481	} while (0)
1482	#else
1483	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1484	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1485	#endif
1486
1487	/**
1488	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1489	* address variant.
1490	*
1491	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1492	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1493	* @param[in] a_GCPtrMem The memory address.
1494	* @remarks Will return/long jump on errors.
1495	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1496	*/
1497	#ifndef IEM_WITH_SETJMP
1498	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1499	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1500	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1501	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1502	} while (0)
1503	#else
1504	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1505	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1506	#endif
1507
1508
1509	/* 16-bit */
1510
1511	/**
1512	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1513	*
1514	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1515	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1516	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1517	* @param[in] a_GCPtrMem The memory address.
1518	* @remarks Will return/long jump on errors.
1519	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1520	*/
1521	#ifndef IEM_WITH_SETJMP
1522	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1523	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1524	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1525	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1526	} while (0)
1527	#else
1528	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1529	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1530	#endif
1531
1532	/**
1533	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1534	*
1535	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1536	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1537	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1538	* @param[in] a_GCPtrMem The memory address.
1539	* @remarks Will return/long jump on errors.
1540	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1541	*/
1542	#ifndef IEM_WITH_SETJMP
1543	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1544	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1545	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1546	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1547	} while (0)
1548	#else
1549	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1550	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1551	#endif
1552
1553	/**
1554	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1555	*
1556	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1557	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1558	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1559	* @param[in] a_GCPtrMem The memory address.
1560	* @remarks Will return/long jump on errors.
1561	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1562	*/
1563	#ifndef IEM_WITH_SETJMP
1564	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1565	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1566	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1567	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1568	} while (0)
1569	#else
1570	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1571	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1572	#endif
1573
1574	/**
1575	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1576	* address variant.
1577	*
1578	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1579	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1580	* @param[in] a_GCPtrMem The memory address.
1581	* @remarks Will return/long jump on errors.
1582	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1583	*/
1584	#ifndef IEM_WITH_SETJMP
1585	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1586	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1587	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1588	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1589	} while (0)
1590	#else
1591	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1592	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1593	#endif
1594
1595	/**
1596	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1597	* address variant.
1598	*
1599	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1600	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1601	* @param[in] a_GCPtrMem The memory address.
1602	* @remarks Will return/long jump on errors.
1603	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1604	*/
1605	#ifndef IEM_WITH_SETJMP
1606	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1607	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1608	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1609	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1610	} while (0)
1611	#else
1612	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1613	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1614	#endif
1615
1616	/**
1617	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1618	* address variant.
1619	*
1620	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1621	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1622	* @param[in] a_GCPtrMem The memory address.
1623	* @remarks Will return/long jump on errors.
1624	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1625	*/
1626	#ifndef IEM_WITH_SETJMP
1627	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1628	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1629	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1630	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1631	} while (0)
1632	#else
1633	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1634	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1635	#endif
1636
1637
1638	/* 32-bit */
1639
1640	/**
1641	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1642	*
1643	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1644	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1645	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1646	* @param[in] a_GCPtrMem The memory address.
1647	* @remarks Will return/long jump on errors.
1648	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1649	*/
1650	#ifndef IEM_WITH_SETJMP
1651	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1652	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1653	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1654	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1655	} while (0)
1656	#else
1657	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1658	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1659	#endif
1660
1661	/**
1662	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1663	*
1664	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1665	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1666	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1667	* @param[in] a_GCPtrMem The memory address.
1668	* @remarks Will return/long jump on errors.
1669	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1670	*/
1671	#ifndef IEM_WITH_SETJMP
1672	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1673	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1674	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1675	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1676	} while (0)
1677	#else
1678	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1679	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1680	#endif
1681
1682	/**
1683	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1684	*
1685	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1686	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1687	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1688	* @param[in] a_GCPtrMem The memory address.
1689	* @remarks Will return/long jump on errors.
1690	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1691	*/
1692	#ifndef IEM_WITH_SETJMP
1693	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1694	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1695	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1696	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1697	} while (0)
1698	#else
1699	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1700	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1701	#endif
1702
1703	/**
1704	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
1705	* flat address variant.
1706	*
1707	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1708	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1709	* @param[in] a_GCPtrMem The memory address.
1710	* @remarks Will return/long jump on errors.
1711	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1712	*/
1713	#ifndef IEM_WITH_SETJMP
1714	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1715	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1716	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1717	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1718	} while (0)
1719	#else
1720	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1721	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1722	#endif
1723
1724	/**
1725	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
1726	* address variant.
1727	*
1728	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1729	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1730	* @param[in] a_GCPtrMem The memory address.
1731	* @remarks Will return/long jump on errors.
1732	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1733	*/
1734	#ifndef IEM_WITH_SETJMP
1735	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1736	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1737	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1738	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1739	} while (0)
1740	#else
1741	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1742	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1743	#endif
1744
1745	/**
1746	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
1747	* address variant.
1748	*
1749	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1750	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1751	* @param[in] a_GCPtrMem The memory address.
1752	* @remarks Will return/long jump on errors.
1753	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1754	*/
1755	#ifndef IEM_WITH_SETJMP
1756	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1757	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1758	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1759	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1760	} while (0)
1761	#else
1762	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1763	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1764	#endif
1765
1766
1767	/* 64-bit */
1768
1769	/**
1770	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
1771	*
1772	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1773	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1774	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1775	* @param[in] a_GCPtrMem The memory address.
1776	* @remarks Will return/long jump on errors.
1777	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1778	*/
1779	#ifndef IEM_WITH_SETJMP
1780	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1781	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1782	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1783	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1784	} while (0)
1785	#else
1786	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1787	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1788	#endif
1789
1790	/**
1791	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
1792	*
1793	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1794	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1795	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1796	* @param[in] a_GCPtrMem The memory address.
1797	* @remarks Will return/long jump on errors.
1798	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1799	*/
1800	#ifndef IEM_WITH_SETJMP
1801	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1802	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1803	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1804	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1805	} while (0)
1806	#else
1807	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1808	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1809	#endif
1810
1811	/**
1812	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
1813	*
1814	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1815	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1816	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1817	* @param[in] a_GCPtrMem The memory address.
1818	* @remarks Will return/long jump on errors.
1819	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1820	*/
1821	#ifndef IEM_WITH_SETJMP
1822	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1823	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1824	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1825	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1826	} while (0)
1827	#else
1828	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1829	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1830	#endif
1831
1832	/**
1833	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
1834	* flat address variant.
1835	*
1836	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1837	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1838	* @param[in] a_GCPtrMem The memory address.
1839	* @remarks Will return/long jump on errors.
1840	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1841	*/
1842	#ifndef IEM_WITH_SETJMP
1843	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1844	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1845	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1846	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1847	} while (0)
1848	#else
1849	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1850	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1851	#endif
1852
1853	/**
1854	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
1855	* address variant.
1856	*
1857	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1858	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1859	* @param[in] a_GCPtrMem The memory address.
1860	* @remarks Will return/long jump on errors.
1861	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1862	*/
1863	#ifndef IEM_WITH_SETJMP
1864	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1865	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1866	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1867	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1868	} while (0)
1869	#else
1870	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1871	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1872	#endif
1873
1874	/**
1875	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
1876	* address variant.
1877	*
1878	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1879	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1880	* @param[in] a_GCPtrMem The memory address.
1881	* @remarks Will return/long jump on errors.
1882	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1883	*/
1884	#ifndef IEM_WITH_SETJMP
1885	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1886	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1887	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1888	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1889	} while (0)
1890	#else
1891	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1892	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1893	#endif
1894
1895
1896	/* commit + unmap */
1897
1898	/** Commits the memory and unmaps guest memory previously mapped RW.
1899	* @remarks May return.
1900	*/
1901	#ifndef IEM_WITH_SETJMP
1902	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) do { \
1903	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4)) ); \
1904	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_RW)); \
1905	} while (0)
1906	#else
1907	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) \
1908	iemMemCommitAndUnmapRwJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1909	#endif
1910
1911	/** Commits the memory and unmaps guest memory previously mapped W.
1912	* @remarks May return.
1913	*/
1914	#ifndef IEM_WITH_SETJMP
1915	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) do { \
1916	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_WRITE << 4)) ); \
1917	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W)); \
1918	} while (0)
1919	#else
1920	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) \
1921	iemMemCommitAndUnmapWoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1922	#endif
1923
1924	/** Commits the memory and unmaps guest memory previously mapped R.
1925	* @remarks May return.
1926	*/
1927	#ifndef IEM_WITH_SETJMP
1928	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) do { \
1929	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_READ << 4)) ); \
1930	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (void *)(a_pvMem), IEM_ACCESS_DATA_R)); \
1931	} while (0)
1932	#else
1933	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) \
1934	iemMemCommitAndUnmapRoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1935	#endif
1936
1937
1938	/** Commits the memory and unmaps the guest memory unless the FPU status word
1939	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
1940	* that would cause FLD not to store.
1941	*
1942	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
1943	* store, while \#P will not.
1944	*
1945	* @remarks May in theory return - for now.
1946	*/
1947	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
1948	do { \
1949	if ( !(a_u16FSW & X86_FSW_ES) \
1950	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
1951	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
1952	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
1953	} while (0)
1954
1955
1956
1957	/** Calculate efficient address from R/M. */
1958	#ifndef IEM_WITH_SETJMP
1959	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1960	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
1961	#else
1962	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1963	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
1964	#endif
1965
1966	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
1967	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
1968	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
1969	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
1970	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
1971	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
1972	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
1973
1974	/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
1975	*
1976	* These clues are mainly for the recompiler, so that it can emit correct code.
1977	*
1978	* They are processed by the python script and which also automatically
1979	* calculates flags for MC blocks based on the statements, extending the use of
1980	* these flags to describe MC block behavior to the recompiler core. The python
1981	* script pass the flags to the IEM_MC2_END_EMIT_CALLS macro, but mainly for
1982	* error checking purposes. The script emits the necessary fEndTb = true and
1983	* similar statements as this reduces compile time a tiny bit.
1984	*
1985	* @{ */
1986	/** Flag set if direct branch, clear if absolute or indirect. */
1987	#define IEM_CIMPL_F_BRANCH_DIRECT RT_BIT_32(0)
1988	/** Flag set if indirect branch, clear if direct or relative.
1989	* This is also used for all system control transfers (SYSCALL, SYSRET, INT, ++)
1990	* as well as for return instructions (RET, IRET, RETF). */
1991	#define IEM_CIMPL_F_BRANCH_INDIRECT RT_BIT_32(1)
1992	/** Flag set if relative branch, clear if absolute or indirect. */
1993	#define IEM_CIMPL_F_BRANCH_RELATIVE RT_BIT_32(2)
1994	/** Flag set if conditional branch, clear if unconditional. */
1995	#define IEM_CIMPL_F_BRANCH_CONDITIONAL RT_BIT_32(3)
1996	/** Flag set if it's a far branch (changes CS). */
1997	#define IEM_CIMPL_F_BRANCH_FAR RT_BIT_32(4)
1998	/** Convenience: Testing any kind of branch. */
1999	#define IEM_CIMPL_F_BRANCH_ANY (IEM_CIMPL_F_BRANCH_DIRECT \| IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_RELATIVE)
2000
2001	/** Execution flags may change (IEMCPU::fExec). */
2002	#define IEM_CIMPL_F_MODE RT_BIT_32(5)
2003	/** May change significant portions of RFLAGS. */
2004	#define IEM_CIMPL_F_RFLAGS RT_BIT_32(6)
2005	/** May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS. */
2006	#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(7)
2007	/** May trigger interrupt shadowing. */
2008	#define IEM_CIMPL_F_INHIBIT_SHADOW RT_BIT_32(8)
2009	/** May enable interrupts, so recheck IRQ immediately afterwards executing
2010	* the instruction. */
2011	#define IEM_CIMPL_F_CHECK_IRQ_AFTER RT_BIT_32(9)
2012	/** May disable interrupts, so recheck IRQ immediately before executing the
2013	* instruction. */
2014	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE RT_BIT_32(10)
2015	/** Convenience: Check for IRQ both before and after an instruction. */
2016	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE_AND_AFTER (IEM_CIMPL_F_CHECK_IRQ_BEFORE \| IEM_CIMPL_F_CHECK_IRQ_AFTER)
2017	/** May trigger a VM exit (treated like IEM_CIMPL_F_MODE atm). */
2018	#define IEM_CIMPL_F_VMEXIT RT_BIT_32(11)
2019	/** May modify FPU state.
2020	* @todo Not sure if this is useful yet. */
2021	#define IEM_CIMPL_F_FPU RT_BIT_32(12)
2022	/** REP prefixed instruction which may yield before updating PC.
2023	* @todo Not sure if this is useful, REP functions now return non-zero
2024	* status if they don't update the PC. */
2025	#define IEM_CIMPL_F_REP RT_BIT_32(13)
2026	/** I/O instruction.
2027	* @todo Not sure if this is useful yet. */
2028	#define IEM_CIMPL_F_IO RT_BIT_32(14)
2029	/** Force end of TB after the instruction. */
2030	#define IEM_CIMPL_F_END_TB RT_BIT_32(15)
2031	/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
2032	#define IEM_CIMPL_F_XCPT \
2033	(IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_FAR \| IEM_CIMPL_F_MODE \| IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_VMEXIT)
2034	/** @} */
2035
2036	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2037	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2038	*/
2039	#ifdef VBOX_STRICT
2040	#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2041	do { \
2042	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2043	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2044	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2045	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2046	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2047	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2048	if (rcStrictHlp == VINF_SUCCESS) \
2049	{ \
2050	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2051	\|\| ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
2052	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2053	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2054	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2055	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2056	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2057	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
2058	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2059	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2060	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2061	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2062	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2063	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2064	else \
2065	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2066	== (fEflBefore & ~(X86_EFL_RF)), \
2067	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2068	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2069	{ \
2070	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2071	AssertMsg( fExecBefore == fExecRecalc \
2072	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2073	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2074	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2075	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2076	} \
2077	} \
2078	return rcStrictHlp; \
2079	} while (0)
2080	#else
2081	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2082	#endif
2083
2084	/**
2085	* Defers the rest of the instruction emulation to a C implementation routine
2086	* and returns, only taking the standard parameters.
2087	*
2088	* @param a_fFlags IEM_CIMPL_F_XXX.
2089	* @param a_pfnCImpl The pointer to the C routine.
2090	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2091	*/
2092	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_pfnCImpl) \
2093	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2094
2095	/**
2096	* Defers the rest of instruction emulation to a C implementation routine and
2097	* returns, taking one argument in addition to the standard ones.
2098	*
2099	* @param a_fFlags IEM_CIMPL_F_XXX.
2100	* @param a_pfnCImpl The pointer to the C routine.
2101	* @param a0 The argument.
2102	*/
2103	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_pfnCImpl, a0) \
2104	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2105
2106	/**
2107	* Defers the rest of the instruction emulation to a C implementation routine
2108	* and returns, taking two arguments in addition to the standard ones.
2109	*
2110	* @param a_fFlags IEM_CIMPL_F_XXX.
2111	* @param a_pfnCImpl The pointer to the C routine.
2112	* @param a0 The first extra argument.
2113	* @param a1 The second extra argument.
2114	*/
2115	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_pfnCImpl, a0, a1) \
2116	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2117
2118	/**
2119	* Defers the rest of the instruction emulation to a C implementation routine
2120	* and returns, taking three arguments in addition to the standard ones.
2121	*
2122	* @param a_fFlags IEM_CIMPL_F_XXX.
2123	* @param a_pfnCImpl The pointer to the C routine.
2124	* @param a0 The first extra argument.
2125	* @param a1 The second extra argument.
2126	* @param a2 The third extra argument.
2127	*/
2128	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2129	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2130
2131	/**
2132	* Defers the rest of the instruction emulation to a C implementation routine
2133	* and returns, taking four arguments in addition to the standard ones.
2134	*
2135	* @param a_fFlags IEM_CIMPL_F_XXX.
2136	* @param a_pfnCImpl The pointer to the C routine.
2137	* @param a0 The first extra argument.
2138	* @param a1 The second extra argument.
2139	* @param a2 The third extra argument.
2140	* @param a3 The fourth extra argument.
2141	*/
2142	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
2143	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2144
2145	/**
2146	* Defers the rest of the instruction emulation to a C implementation routine
2147	* and returns, taking two arguments in addition to the standard ones.
2148	*
2149	* @param a_fFlags IEM_CIMPL_F_XXX.
2150	* @param a_pfnCImpl The pointer to the C routine.
2151	* @param a0 The first extra argument.
2152	* @param a1 The second extra argument.
2153	* @param a2 The third extra argument.
2154	* @param a3 The fourth extra argument.
2155	* @param a4 The fifth extra argument.
2156	*/
2157	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
2158	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2159
2160	/**
2161	* Defers the entire instruction emulation to a C implementation routine and
2162	* returns, only taking the standard parameters.
2163	*
2164	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2165	*
2166	* @param a_fFlags IEM_CIMPL_F_XXX.
2167	* @param a_pfnCImpl The pointer to the C routine.
2168	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2169	*/
2170	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) \
2171	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2172
2173	/**
2174	* Defers the entire instruction emulation to a C implementation routine and
2175	* returns, taking one argument in addition to the standard ones.
2176	*
2177	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2178	*
2179	* @param a_fFlags IEM_CIMPL_F_XXX.
2180	* @param a_pfnCImpl The pointer to the C routine.
2181	* @param a0 The argument.
2182	*/
2183	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_pfnCImpl, a0) \
2184	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2185
2186	/**
2187	* Defers the entire instruction emulation to a C implementation routine and
2188	* returns, taking two arguments in addition to the standard ones.
2189	*
2190	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2191	*
2192	* @param a_fFlags IEM_CIMPL_F_XXX.
2193	* @param a_pfnCImpl The pointer to the C routine.
2194	* @param a0 The first extra argument.
2195	* @param a1 The second extra argument.
2196	*/
2197	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_pfnCImpl, a0, a1) \
2198	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2199
2200	/**
2201	* Defers the entire instruction emulation to a C implementation routine and
2202	* returns, taking three arguments in addition to the standard ones.
2203	*
2204	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2205	*
2206	* @param a_fFlags IEM_CIMPL_F_XXX.
2207	* @param a_pfnCImpl The pointer to the C routine.
2208	* @param a0 The first extra argument.
2209	* @param a1 The second extra argument.
2210	* @param a2 The third extra argument.
2211	*/
2212	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2213	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2214
2215
2216	/**
2217	* Calls a FPU assembly implementation taking one visible argument.
2218	*
2219	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2220	* @param a0 The first extra argument.
2221	*/
2222	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2223	do { \
2224	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2225	} while (0)
2226
2227	/**
2228	* Calls a FPU assembly implementation taking two visible arguments.
2229	*
2230	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2231	* @param a0 The first extra argument.
2232	* @param a1 The second extra argument.
2233	*/
2234	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2235	do { \
2236	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2237	} while (0)
2238
2239	/**
2240	* Calls a FPU assembly implementation taking three visible arguments.
2241	*
2242	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2243	* @param a0 The first extra argument.
2244	* @param a1 The second extra argument.
2245	* @param a2 The third extra argument.
2246	*/
2247	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2248	do { \
2249	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2250	} while (0)
2251
2252	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2253	do { \
2254	(a_FpuData).FSW = (a_FSW); \
2255	(a_FpuData).r80Result = *(a_pr80Value); \
2256	} while (0)
2257
2258	/** Pushes FPU result onto the stack. */
2259	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2260	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2261	/** Pushes FPU result onto the stack and sets the FPUDP. */
2262	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2263	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2264
2265	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2266	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2267	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2268
2269	/** Stores FPU result in a stack register. */
2270	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2271	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2272	/** Stores FPU result in a stack register and pops the stack. */
2273	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2274	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2275	/** Stores FPU result in a stack register and sets the FPUDP. */
2276	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2277	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2278	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2279	* stack. */
2280	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2281	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2282
2283	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2284	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2285	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2286	/** Free a stack register (for FFREE and FFREEP). */
2287	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2288	iemFpuStackFree(pVCpu, a_iStReg)
2289	/** Increment the FPU stack pointer. */
2290	#define IEM_MC_FPU_STACK_INC_TOP() \
2291	iemFpuStackIncTop(pVCpu)
2292	/** Decrement the FPU stack pointer. */
2293	#define IEM_MC_FPU_STACK_DEC_TOP() \
2294	iemFpuStackDecTop(pVCpu)
2295
2296	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2297	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2298	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2299	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2300	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2301	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2302	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2303	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2304	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2305	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2306	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2307	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2308	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2309	* stack. */
2310	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2311	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2312	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2313	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2314	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2315
2316	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2317	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2318	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2319	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2320	* stack. */
2321	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2322	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2323	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2324	* FPUDS. */
2325	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2326	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2327	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2328	* FPUDS. Pops stack. */
2329	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2330	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2331	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2332	* stack twice. */
2333	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2334	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2335	/** Raises a FPU stack underflow exception for an instruction pushing a result
2336	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
2337	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2338	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2339	/** Raises a FPU stack underflow exception for an instruction pushing a result
2340	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2341	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2342	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2343
2344	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2345	* FPUIP, FPUCS and FOP. */
2346	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2347	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2348	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2349	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2350	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2351	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2352	/** Prepares for using the FPU state.
2353	* Ensures that we can use the host FPU in the current context (RC+R0.
2354	* Ensures the guest FPU state in the CPUMCTX is up to date. */
2355	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2356	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2357	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2358	/** Actualizes the guest FPU state so it can be accessed and modified. */
2359	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2360
2361	/** Stores SSE SIMD result updating MXCSR. */
2362	#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
2363	iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
2364	/** Updates MXCSR. */
2365	#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
2366	iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
2367
2368	/** Prepares for using the SSE state.
2369	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2370	* Ensures the guest SSE state in the CPUMCTX is up to date. */
2371	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2372	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2373	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
2374	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
2375	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
2376
2377	/** Prepares for using the AVX state.
2378	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
2379	* Ensures the guest AVX state in the CPUMCTX is up to date.
2380	* @note This will include the AVX512 state too when support for it is added
2381	* due to the zero extending feature of VEX instruction. */
2382	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
2383	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2384	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
2385	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
2386	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
2387
2388	/**
2389	* Calls a MMX assembly implementation taking two visible arguments.
2390	*
2391	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2392	* @param a0 The first extra argument.
2393	* @param a1 The second extra argument.
2394	*/
2395	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
2396	do { \
2397	IEM_MC_PREPARE_FPU_USAGE(); \
2398	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2399	} while (0)
2400
2401	/**
2402	* Calls a MMX assembly implementation taking three visible arguments.
2403	*
2404	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2405	* @param a0 The first extra argument.
2406	* @param a1 The second extra argument.
2407	* @param a2 The third extra argument.
2408	*/
2409	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2410	do { \
2411	IEM_MC_PREPARE_FPU_USAGE(); \
2412	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2413	} while (0)
2414
2415
2416	/**
2417	* Calls a SSE assembly implementation taking two visible arguments.
2418	*
2419	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2420	* @param a0 The first extra argument.
2421	* @param a1 The second extra argument.
2422	*/
2423	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
2424	do { \
2425	IEM_MC_PREPARE_SSE_USAGE(); \
2426	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2427	} while (0)
2428
2429	/**
2430	* Calls a SSE assembly implementation taking three visible arguments.
2431	*
2432	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2433	* @param a0 The first extra argument.
2434	* @param a1 The second extra argument.
2435	* @param a2 The third extra argument.
2436	*/
2437	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2438	do { \
2439	IEM_MC_PREPARE_SSE_USAGE(); \
2440	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2441	} while (0)
2442
2443
2444	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
2445	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
2446	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
2447	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
2448
2449	/**
2450	* Calls a AVX assembly implementation taking two visible arguments.
2451	*
2452	* There is one implicit zero'th argument, a pointer to the extended state.
2453	*
2454	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2455	* @param a1 The first extra argument.
2456	* @param a2 The second extra argument.
2457	*/
2458	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
2459	do { \
2460	IEM_MC_PREPARE_AVX_USAGE(); \
2461	a_pfnAImpl(pXState, (a1), (a2)); \
2462	} while (0)
2463
2464	/**
2465	* Calls a AVX assembly implementation taking three visible arguments.
2466	*
2467	* There is one implicit zero'th argument, a pointer to the extended state.
2468	*
2469	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2470	* @param a1 The first extra argument.
2471	* @param a2 The second extra argument.
2472	* @param a3 The third extra argument.
2473	*/
2474	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
2475	do { \
2476	IEM_MC_PREPARE_AVX_USAGE(); \
2477	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
2478	} while (0)
2479
2480	/** @note Not for IOPL or IF testing. */
2481	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
2482	/** @note Not for IOPL or IF testing. */
2483	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
2484	/** @note Not for IOPL or IF testing. */
2485	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
2486	/** @note Not for IOPL or IF testing. */
2487	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
2488	/** @note Not for IOPL or IF testing. */
2489	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
2490	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2491	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2492	/** @note Not for IOPL or IF testing. */
2493	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
2494	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2495	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2496	/** @note Not for IOPL or IF testing. */
2497	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
2498	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2499	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2500	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2501	/** @note Not for IOPL or IF testing. */
2502	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
2503	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2504	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2505	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2506	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
2507	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
2508	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
2509	/** @note Not for IOPL or IF testing. */
2510	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2511	if ( pVCpu->cpum.GstCtx.cx != 0 \
2512	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2513	/** @note Not for IOPL or IF testing. */
2514	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2515	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2516	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2517	/** @note Not for IOPL or IF testing. */
2518	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2519	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2520	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2521	/** @note Not for IOPL or IF testing. */
2522	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2523	if ( pVCpu->cpum.GstCtx.cx != 0 \
2524	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2525	/** @note Not for IOPL or IF testing. */
2526	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2527	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2528	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2529	/** @note Not for IOPL or IF testing. */
2530	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2531	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2532	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2533	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
2534	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
2535
2536	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
2537	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
2538	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2539	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2540	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
2541	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
2542	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2543	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2544	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
2545	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
2546	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
2547	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
2548	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
2549	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
2550	#define IEM_MC_IF_FCW_IM() \
2551	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
2552	#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
2553	if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
2554	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
2555
2556	#define IEM_MC_ELSE() } else {
2557	#define IEM_MC_ENDIF() } do {} while (0)
2558
2559	/** @} */
2560
2561	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
2562

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 101399

Download in other formats: