IEMMc.h@ 95421

Last change on this file since 95421 was 95421, checked in by vboxsync, 2 years ago
VMM/IEM: fld, fbld and fstp are 8 byte aligned as far as #AC is concerned on an 10980xe. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 73.1 KB

Line
1	/* $Id: IEMMc.h 95421 2022-06-29 02:41:48Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
19	#define VMM_INCLUDED_SRC_include_IEMMc_h
20	#ifndef RT_WITHOUT_PRAGMA_ONCE
21	# pragma once
22	#endif
23
24
25	/** @name "Microcode" macros.
26	*
27	* The idea is that we should be able to use the same code to interpret
28	* instructions as well as recompiler instructions. Thus this obfuscation.
29	*
30	* @{
31	*/
32	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
33	#define IEM_MC_END() }
34	#define IEM_MC_PAUSE() do {} while (0)
35	#define IEM_MC_CONTINUE() do {} while (0)
36
37	/** Internal macro. */
38	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
39	do \
40	{ \
41	VBOXSTRICTRC rcStrict2 = a_Expr; \
42	if (rcStrict2 != VINF_SUCCESS) \
43	return rcStrict2; \
44	} while (0)
45
46
47	#define IEM_MC_ADVANCE_RIP() iemRegUpdateRipAndClearRF(pVCpu)
48	#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pVCpu, a_i8))
49	#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pVCpu, a_i16))
50	#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pVCpu, a_i32))
51	#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u16NewIP)))
52	#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u32NewIP)))
53	#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u64NewIP)))
54	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
55	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
56	do { \
57	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
58	return iemRaiseDeviceNotAvailable(pVCpu); \
59	} while (0)
60	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
61	do { \
62	if ((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)) \
63	return iemRaiseDeviceNotAvailable(pVCpu); \
64	} while (0)
65	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
66	do { \
67	if (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES) \
68	return iemRaiseMathFault(pVCpu); \
69	} while (0)
70	#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT() \
71	do { \
72	if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
73	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
74	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx2) \
75	return iemRaiseUndefinedOpcode(pVCpu); \
76	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
77	return iemRaiseDeviceNotAvailable(pVCpu); \
78	} while (0)
79	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
80	do { \
81	if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
82	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
83	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) \
84	return iemRaiseUndefinedOpcode(pVCpu); \
85	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
86	return iemRaiseDeviceNotAvailable(pVCpu); \
87	} while (0)
88	#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT() \
89	do { \
90	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
91	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
92	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse41) \
93	return iemRaiseUndefinedOpcode(pVCpu); \
94	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
95	return iemRaiseDeviceNotAvailable(pVCpu); \
96	} while (0)
97	#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT() \
98	do { \
99	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
100	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
101	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse3) \
102	return iemRaiseUndefinedOpcode(pVCpu); \
103	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
104	return iemRaiseDeviceNotAvailable(pVCpu); \
105	} while (0)
106	#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
107	do { \
108	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
109	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
110	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) \
111	return iemRaiseUndefinedOpcode(pVCpu); \
112	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
113	return iemRaiseDeviceNotAvailable(pVCpu); \
114	} while (0)
115	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
116	do { \
117	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
118	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
119	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) \
120	return iemRaiseUndefinedOpcode(pVCpu); \
121	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
122	return iemRaiseDeviceNotAvailable(pVCpu); \
123	} while (0)
124	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
125	do { \
126	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
127	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMmx) \
128	return iemRaiseUndefinedOpcode(pVCpu); \
129	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
130	return iemRaiseDeviceNotAvailable(pVCpu); \
131	if (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES) \
132	return iemRaiseMathFault(pVCpu); \
133	} while (0)
134	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
135	do { \
136	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
137	\|\| ( !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse \
138	&& !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAmdMmxExts) ) \
139	return iemRaiseUndefinedOpcode(pVCpu); \
140	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
141	return iemRaiseDeviceNotAvailable(pVCpu); \
142	} while (0)
143	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
144	do { \
145	if (pVCpu->iem.s.uCpl != 0) \
146	return iemRaiseGeneralProtectionFault0(pVCpu); \
147	} while (0)
148	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
149	do { \
150	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
151	else return iemRaiseGeneralProtectionFault0(pVCpu); \
152	} while (0)
153	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
154	do { \
155	if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT \
156	\|\| !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFsGsBase \
157	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE)) \
158	return iemRaiseUndefinedOpcode(pVCpu); \
159	} while (0)
160	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
161	do { \
162	if (!IEM_IS_CANONICAL(a_u64Addr)) \
163	return iemRaiseGeneralProtectionFault0(pVCpu); \
164	} while (0)
165
166
167	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
168	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
169	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
170	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
171	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
172	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
173	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
174	uint32_t a_Name; \
175	uint32_t *a_pName = &a_Name
176	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
177	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
178
179	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
180	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
181
182	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
183	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
184	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
185	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
186	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
187	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
188	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
189	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
190	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
191	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
192	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
193	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
194	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
195	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
196	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
197	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
198	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
199	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
200	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
201	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
202	} while (0)
203	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
204	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
205	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
206	} while (0)
207	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
208	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
209	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
210	} while (0)
211	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
212	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
213	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
214	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
215	} while (0)
216	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
217	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
218	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
219	} while (0)
220	/** @note Not for IOPL or IF testing or modification. */
221	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
222	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
223	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
224	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
225
226	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
227	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
228	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
229	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
230	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
231	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
232	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
233	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
234	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
235	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
236	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
237	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
238	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
239	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
240	} while (0)
241	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
242	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
243	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
244	} while (0)
245	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
246	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
247
248
249	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
250	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
251	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
252	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
253	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
254	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
255	/** @note Not for IOPL or IF testing or modification. */
256	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.u
257
258	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
259	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
260	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
261	do { \
262	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
263	*pu32Reg += (a_u32Value); \
264	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
265	} while (0)
266	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
267
268	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
269	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
270	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
271	do { \
272	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
273	*pu32Reg -= (a_u32Value); \
274	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
275	} while (0)
276	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
277	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
278
279	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
280	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
281	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
282	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
283	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
284	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
285	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
286
287	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
288	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
289	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
290	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
291
292	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
293	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
294	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
295
296	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
297	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
298	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
299
300	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
301	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
302	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
303
304	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
305	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
306	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
307
308	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
309
310	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
311
312	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
313	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
314	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
315	do { \
316	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
317	*pu32Reg &= (a_u32Value); \
318	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
319	} while (0)
320	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
321
322	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
323	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
324	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
325	do { \
326	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
327	*pu32Reg \|= (a_u32Value); \
328	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
329	} while (0)
330	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
331
332
333	/** @note Not for IOPL or IF modification. */
334	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
335	/** @note Not for IOPL or IF modification. */
336	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
337	/** @note Not for IOPL or IF modification. */
338	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
339
340	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
341
342	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
343	#define IEM_MC_FPU_TO_MMX_MODE() do { \
344	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
345	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
346	} while (0)
347
348	/** Switches the FPU state from MMX mode (FTW=0xffff). */
349	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
350	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
351	} while (0)
352
353	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
354	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
355	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
356	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
357	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
358	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
359	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
360	} while (0)
361	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
362	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
363	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
364	} while (0)
365	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
366	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
367	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
368	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
369	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
370	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
371
372	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
373	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
374	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
375	} while (0)
376	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
377	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; } while (0)
378	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
379	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0]; } while (0)
380	#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg) \
381	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; } while (0)
382	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
383	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
384	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
385	} while (0)
386	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value) \
387	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); } while (0)
388	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
389	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
390	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
391	} while (0)
392	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value) \
393	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = (a_u32Value); } while (0)
394	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
395	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
396	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
397	} while (0)
398	#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
399	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
400	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
401	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
402	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
403	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
404	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
405	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
406	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
407	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
408	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
409	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
410	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
411	} while (0)
412
413	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
414	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
415	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
416	} while (0)
417	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
418	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
419	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
420	} while (0)
421	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
422	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
423	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
424	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
425	} while (0)
426	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
427	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
428	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
429	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
430	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
431	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
432	} while (0)
433
434	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
435	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
436	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
437	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
438	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
439	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
440	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
441	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
442	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
443	} while (0)
444	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
445	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
446	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
447	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
448	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
449	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
450	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
451	} while (0)
452	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
453	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
454	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
455	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
456	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
457	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
458	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
459	} while (0)
460	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
461	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
462	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
463	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
464	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
465	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
466	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
467	} while (0)
468
469	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
470	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
471	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
472	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
473	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
474	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
475	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
476	do { uintptr_t const iYRegTmp = (a_iYReg); \
477	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
478	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
479	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
480	} while (0)
481
482	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
483	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
484	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
485	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
486	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
487	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
488	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
489	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
490	} while (0)
491	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
492	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
493	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
494	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
495	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
496	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
497	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
498	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
499	} while (0)
500	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
501	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
502	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
503	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
504	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
505	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
506	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
507	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
508	} while (0)
509
510	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
511	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
512	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
513	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
514	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
515	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
516	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
517	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
518	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
519	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
520	} while (0)
521	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
522	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
523	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
524	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
525	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
526	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
527	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
528	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
529	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
530	} while (0)
531	#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
532	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
533	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
534	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
535	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
536	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
537	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
538	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
539	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
540	} while (0)
541	#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
542	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
543	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
544	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
545	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
546	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
547	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
548	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
549	} while (0)
550
551	#ifndef IEM_WITH_SETJMP
552	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
553	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
554	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
555	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
556	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
557	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
558	#else
559	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
560	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
561	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
562	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
563	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
564	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
565	#endif
566
567	#ifndef IEM_WITH_SETJMP
568	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
569	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
570	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
571	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
572	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
573	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
574	#else
575	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
576	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
577	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
578	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
579	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
580	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
581	#endif
582
583	#ifndef IEM_WITH_SETJMP
584	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
585	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
586	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
587	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
588	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
589	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
590	#else
591	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
592	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
593	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
594	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
595	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
596	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
597	#endif
598
599	#ifdef SOME_UNUSED_FUNCTION
600	# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
601	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
602	#endif
603
604	#ifndef IEM_WITH_SETJMP
605	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
606	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
607	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
608	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
609	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
610	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
611	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
612	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
613	#else
614	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
615	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
616	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
617	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
618	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
619	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
620	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
621	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
622	#endif
623
624	#ifndef IEM_WITH_SETJMP
625	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
626	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
627	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
628	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
629	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
630	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
631	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
632	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
633	#else
634	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
635	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
636	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
637	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
638	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
639	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
640	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
641	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
642	#endif
643
644	#ifndef IEM_WITH_SETJMP
645	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
646	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
647	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
648	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
649	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
650	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
651	#else
652	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
653	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
654	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
655	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
656	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
657	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
658	#endif
659
660	#ifndef IEM_WITH_SETJMP
661	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
662	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
663	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
664	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
665	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
666	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
667	#else
668	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
669	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
670	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
671	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
672	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
673	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
674	#endif
675
676
677
678	#ifndef IEM_WITH_SETJMP
679	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
680	do { \
681	uint8_t u8Tmp; \
682	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
683	(a_u16Dst) = u8Tmp; \
684	} while (0)
685	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
686	do { \
687	uint8_t u8Tmp; \
688	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
689	(a_u32Dst) = u8Tmp; \
690	} while (0)
691	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
692	do { \
693	uint8_t u8Tmp; \
694	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
695	(a_u64Dst) = u8Tmp; \
696	} while (0)
697	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
698	do { \
699	uint16_t u16Tmp; \
700	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
701	(a_u32Dst) = u16Tmp; \
702	} while (0)
703	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
704	do { \
705	uint16_t u16Tmp; \
706	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
707	(a_u64Dst) = u16Tmp; \
708	} while (0)
709	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
710	do { \
711	uint32_t u32Tmp; \
712	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
713	(a_u64Dst) = u32Tmp; \
714	} while (0)
715	#else /* IEM_WITH_SETJMP */
716	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
717	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
718	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
719	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
720	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
721	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
722	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
723	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
724	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
725	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
726	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
727	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
728	#endif /* IEM_WITH_SETJMP */
729
730	#ifndef IEM_WITH_SETJMP
731	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
732	do { \
733	uint8_t u8Tmp; \
734	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
735	(a_u16Dst) = (int8_t)u8Tmp; \
736	} while (0)
737	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
738	do { \
739	uint8_t u8Tmp; \
740	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
741	(a_u32Dst) = (int8_t)u8Tmp; \
742	} while (0)
743	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
744	do { \
745	uint8_t u8Tmp; \
746	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
747	(a_u64Dst) = (int8_t)u8Tmp; \
748	} while (0)
749	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
750	do { \
751	uint16_t u16Tmp; \
752	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
753	(a_u32Dst) = (int16_t)u16Tmp; \
754	} while (0)
755	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
756	do { \
757	uint16_t u16Tmp; \
758	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
759	(a_u64Dst) = (int16_t)u16Tmp; \
760	} while (0)
761	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
762	do { \
763	uint32_t u32Tmp; \
764	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
765	(a_u64Dst) = (int32_t)u32Tmp; \
766	} while (0)
767	#else /* IEM_WITH_SETJMP */
768	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
769	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
770	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
771	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
772	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
773	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
774	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
775	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
776	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
777	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
778	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
779	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
780	#endif /* IEM_WITH_SETJMP */
781
782	#ifndef IEM_WITH_SETJMP
783	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
784	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
785	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
786	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
787	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
788	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
789	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
790	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
791	#else
792	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
793	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
794	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
795	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
796	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
797	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
798	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
799	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
800	#endif
801
802	#ifndef IEM_WITH_SETJMP
803	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
804	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
805	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
806	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
807	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
808	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
809	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
810	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
811	#else
812	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
813	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
814	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
815	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
816	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
817	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
818	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
819	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
820	#endif
821
822	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
823	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
824	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
825	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
826	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
827	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
828	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
829	do { \
830	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
831	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
832	} while (0)
833	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
834	do { \
835	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
836	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
837	} while (0)
838
839	#ifndef IEM_WITH_SETJMP
840	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
841	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
842	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
843	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
844	#else
845	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
846	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
847	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
848	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
849	#endif
850
851	#ifndef IEM_WITH_SETJMP
852	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
853	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
854	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
855	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
856	#else
857	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
858	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
859	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
860	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
861	#endif
862
863
864	#define IEM_MC_PUSH_U16(a_u16Value) \
865	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
866	#define IEM_MC_PUSH_U32(a_u32Value) \
867	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
868	#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
869	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
870	#define IEM_MC_PUSH_U64(a_u64Value) \
871	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
872
873	#define IEM_MC_POP_U16(a_pu16Value) \
874	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
875	#define IEM_MC_POP_U32(a_pu32Value) \
876	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
877	#define IEM_MC_POP_U64(a_pu64Value) \
878	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
879
880	/** Maps guest memory for direct or bounce buffered access.
881	* The purpose is to pass it to an operand implementation, thus the a_iArg.
882	* @remarks May return.
883	*/
884	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
885	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), \
886	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
887
888	/** Maps guest memory for direct or bounce buffered access.
889	* The purpose is to pass it to an operand implementation, thus the a_iArg.
890	* @remarks May return.
891	*/
892	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_cbAlign, a_iArg) \
893	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), \
894	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
895
896	/** Commits the memory and unmaps the guest memory.
897	* @remarks May return.
898	*/
899	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
900	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
901
902	/** Commits the memory and unmaps the guest memory unless the FPU status word
903	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
904	* that would cause FLD not to store.
905	*
906	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
907	* store, while \#P will not.
908	*
909	* @remarks May in theory return - for now.
910	*/
911	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
912	do { \
913	if ( !(a_u16FSW & X86_FSW_ES) \
914	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
915	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
916	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
917	} while (0)
918
919	/** Calculate efficient address from R/M. */
920	#ifndef IEM_WITH_SETJMP
921	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
922	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (bRm), (cbImm), &(a_GCPtrEff)))
923	#else
924	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
925	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (bRm), (cbImm)))
926	#endif
927
928	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
929	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
930	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
931	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
932	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
933	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
934	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
935
936	/**
937	* Defers the rest of the instruction emulation to a C implementation routine
938	* and returns, only taking the standard parameters.
939	*
940	* @param a_pfnCImpl The pointer to the C routine.
941	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
942	*/
943	#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
944
945	/**
946	* Defers the rest of instruction emulation to a C implementation routine and
947	* returns, taking one argument in addition to the standard ones.
948	*
949	* @param a_pfnCImpl The pointer to the C routine.
950	* @param a0 The argument.
951	*/
952	#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
953
954	/**
955	* Defers the rest of the instruction emulation to a C implementation routine
956	* and returns, taking two arguments in addition to the standard ones.
957	*
958	* @param a_pfnCImpl The pointer to the C routine.
959	* @param a0 The first extra argument.
960	* @param a1 The second extra argument.
961	*/
962	#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
963
964	/**
965	* Defers the rest of the instruction emulation to a C implementation routine
966	* and returns, taking three arguments in addition to the standard ones.
967	*
968	* @param a_pfnCImpl The pointer to the C routine.
969	* @param a0 The first extra argument.
970	* @param a1 The second extra argument.
971	* @param a2 The third extra argument.
972	*/
973	#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
974
975	/**
976	* Defers the rest of the instruction emulation to a C implementation routine
977	* and returns, taking four arguments in addition to the standard ones.
978	*
979	* @param a_pfnCImpl The pointer to the C routine.
980	* @param a0 The first extra argument.
981	* @param a1 The second extra argument.
982	* @param a2 The third extra argument.
983	* @param a3 The fourth extra argument.
984	*/
985	#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3)
986
987	/**
988	* Defers the rest of the instruction emulation to a C implementation routine
989	* and returns, taking two arguments in addition to the standard ones.
990	*
991	* @param a_pfnCImpl The pointer to the C routine.
992	* @param a0 The first extra argument.
993	* @param a1 The second extra argument.
994	* @param a2 The third extra argument.
995	* @param a3 The fourth extra argument.
996	* @param a4 The fifth extra argument.
997	*/
998	#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4)
999
1000	/**
1001	* Defers the entire instruction emulation to a C implementation routine and
1002	* returns, only taking the standard parameters.
1003	*
1004	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1005	*
1006	* @param a_pfnCImpl The pointer to the C routine.
1007	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
1008	*/
1009	#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
1010
1011	/**
1012	* Defers the entire instruction emulation to a C implementation routine and
1013	* returns, taking one argument in addition to the standard ones.
1014	*
1015	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1016	*
1017	* @param a_pfnCImpl The pointer to the C routine.
1018	* @param a0 The argument.
1019	*/
1020	#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
1021
1022	/**
1023	* Defers the entire instruction emulation to a C implementation routine and
1024	* returns, taking two arguments in addition to the standard ones.
1025	*
1026	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1027	*
1028	* @param a_pfnCImpl The pointer to the C routine.
1029	* @param a0 The first extra argument.
1030	* @param a1 The second extra argument.
1031	*/
1032	#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
1033
1034	/**
1035	* Defers the entire instruction emulation to a C implementation routine and
1036	* returns, taking three arguments in addition to the standard ones.
1037	*
1038	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1039	*
1040	* @param a_pfnCImpl The pointer to the C routine.
1041	* @param a0 The first extra argument.
1042	* @param a1 The second extra argument.
1043	* @param a2 The third extra argument.
1044	*/
1045	#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
1046
1047	/**
1048	* Calls a FPU assembly implementation taking one visible argument.
1049	*
1050	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1051	* @param a0 The first extra argument.
1052	*/
1053	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
1054	do { \
1055	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
1056	} while (0)
1057
1058	/**
1059	* Calls a FPU assembly implementation taking two visible arguments.
1060	*
1061	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1062	* @param a0 The first extra argument.
1063	* @param a1 The second extra argument.
1064	*/
1065	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
1066	do { \
1067	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1068	} while (0)
1069
1070	/**
1071	* Calls a FPU assembly implementation taking three visible arguments.
1072	*
1073	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1074	* @param a0 The first extra argument.
1075	* @param a1 The second extra argument.
1076	* @param a2 The third extra argument.
1077	*/
1078	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1079	do { \
1080	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1081	} while (0)
1082
1083	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
1084	do { \
1085	(a_FpuData).FSW = (a_FSW); \
1086	(a_FpuData).r80Result = *(a_pr80Value); \
1087	} while (0)
1088
1089	/** Pushes FPU result onto the stack. */
1090	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
1091	iemFpuPushResult(pVCpu, &a_FpuData)
1092	/** Pushes FPU result onto the stack and sets the FPUDP. */
1093	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
1094	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
1095
1096	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
1097	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
1098	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo)
1099
1100	/** Stores FPU result in a stack register. */
1101	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
1102	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg)
1103	/** Stores FPU result in a stack register and pops the stack. */
1104	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
1105	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg)
1106	/** Stores FPU result in a stack register and sets the FPUDP. */
1107	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
1108	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
1109	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
1110	* stack. */
1111	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
1112	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
1113
1114	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
1115	#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
1116	iemFpuUpdateOpcodeAndIp(pVCpu)
1117	/** Free a stack register (for FFREE and FFREEP). */
1118	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
1119	iemFpuStackFree(pVCpu, a_iStReg)
1120	/** Increment the FPU stack pointer. */
1121	#define IEM_MC_FPU_STACK_INC_TOP() \
1122	iemFpuStackIncTop(pVCpu)
1123	/** Decrement the FPU stack pointer. */
1124	#define IEM_MC_FPU_STACK_DEC_TOP() \
1125	iemFpuStackDecTop(pVCpu)
1126
1127	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
1128	#define IEM_MC_UPDATE_FSW(a_u16FSW) \
1129	iemFpuUpdateFSW(pVCpu, a_u16FSW)
1130	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
1131	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
1132	iemFpuUpdateFSW(pVCpu, a_u16FSW)
1133	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
1134	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
1135	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
1136	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
1137	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
1138	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW)
1139	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
1140	* stack. */
1141	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
1142	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
1143	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
1144	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
1145	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW)
1146
1147	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
1148	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
1149	iemFpuStackUnderflow(pVCpu, a_iStDst)
1150	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
1151	* stack. */
1152	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
1153	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst)
1154	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
1155	* FPUDS. */
1156	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
1157	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
1158	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
1159	* FPUDS. Pops stack. */
1160	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
1161	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
1162	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
1163	* stack twice. */
1164	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
1165	iemFpuStackUnderflowThenPopPop(pVCpu)
1166	/** Raises a FPU stack underflow exception for an instruction pushing a result
1167	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
1168	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
1169	iemFpuStackPushUnderflow(pVCpu)
1170	/** Raises a FPU stack underflow exception for an instruction pushing a result
1171	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
1172	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
1173	iemFpuStackPushUnderflowTwo(pVCpu)
1174
1175	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
1176	* FPUIP, FPUCS and FOP. */
1177	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
1178	iemFpuStackPushOverflow(pVCpu)
1179	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
1180	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
1181	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
1182	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff)
1183	/** Prepares for using the FPU state.
1184	* Ensures that we can use the host FPU in the current context (RC+R0.
1185	* Ensures the guest FPU state in the CPUMCTX is up to date. */
1186	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
1187	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
1188	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
1189	/** Actualizes the guest FPU state so it can be accessed and modified. */
1190	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
1191
1192	/** Prepares for using the SSE state.
1193	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
1194	* Ensures the guest SSE state in the CPUMCTX is up to date. */
1195	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
1196	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
1197	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
1198	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
1199	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
1200
1201	/** Prepares for using the AVX state.
1202	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
1203	* Ensures the guest AVX state in the CPUMCTX is up to date.
1204	* @note This will include the AVX512 state too when support for it is added
1205	* due to the zero extending feature of VEX instruction. */
1206	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
1207	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
1208	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
1209	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
1210	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
1211
1212	/**
1213	* Calls a MMX assembly implementation taking two visible arguments.
1214	*
1215	* @param a_pfnAImpl Pointer to the assembly MMX routine.
1216	* @param a0 The first extra argument.
1217	* @param a1 The second extra argument.
1218	*/
1219	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
1220	do { \
1221	IEM_MC_PREPARE_FPU_USAGE(); \
1222	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1223	} while (0)
1224
1225	/**
1226	* Calls a MMX assembly implementation taking three visible arguments.
1227	*
1228	* @param a_pfnAImpl Pointer to the assembly MMX routine.
1229	* @param a0 The first extra argument.
1230	* @param a1 The second extra argument.
1231	* @param a2 The third extra argument.
1232	*/
1233	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1234	do { \
1235	IEM_MC_PREPARE_FPU_USAGE(); \
1236	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1237	} while (0)
1238
1239
1240	/**
1241	* Calls a SSE assembly implementation taking two visible arguments.
1242	*
1243	* @param a_pfnAImpl Pointer to the assembly SSE routine.
1244	* @param a0 The first extra argument.
1245	* @param a1 The second extra argument.
1246	*/
1247	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
1248	do { \
1249	IEM_MC_PREPARE_SSE_USAGE(); \
1250	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1251	} while (0)
1252
1253	/**
1254	* Calls a SSE assembly implementation taking three visible arguments.
1255	*
1256	* @param a_pfnAImpl Pointer to the assembly SSE routine.
1257	* @param a0 The first extra argument.
1258	* @param a1 The second extra argument.
1259	* @param a2 The third extra argument.
1260	*/
1261	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1262	do { \
1263	IEM_MC_PREPARE_SSE_USAGE(); \
1264	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1265	} while (0)
1266
1267
1268	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
1269	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
1270	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
1271	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
1272
1273	/**
1274	* Calls a AVX assembly implementation taking two visible arguments.
1275	*
1276	* There is one implicit zero'th argument, a pointer to the extended state.
1277	*
1278	* @param a_pfnAImpl Pointer to the assembly AVX routine.
1279	* @param a1 The first extra argument.
1280	* @param a2 The second extra argument.
1281	*/
1282	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
1283	do { \
1284	IEM_MC_PREPARE_AVX_USAGE(); \
1285	a_pfnAImpl(pXState, (a1), (a2)); \
1286	} while (0)
1287
1288	/**
1289	* Calls a AVX assembly implementation taking three visible arguments.
1290	*
1291	* There is one implicit zero'th argument, a pointer to the extended state.
1292	*
1293	* @param a_pfnAImpl Pointer to the assembly AVX routine.
1294	* @param a1 The first extra argument.
1295	* @param a2 The second extra argument.
1296	* @param a3 The third extra argument.
1297	*/
1298	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
1299	do { \
1300	IEM_MC_PREPARE_AVX_USAGE(); \
1301	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
1302	} while (0)
1303
1304	/** @note Not for IOPL or IF testing. */
1305	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
1306	/** @note Not for IOPL or IF testing. */
1307	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
1308	/** @note Not for IOPL or IF testing. */
1309	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
1310	/** @note Not for IOPL or IF testing. */
1311	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
1312	/** @note Not for IOPL or IF testing. */
1313	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
1314	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1315	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1316	/** @note Not for IOPL or IF testing. */
1317	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
1318	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1319	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1320	/** @note Not for IOPL or IF testing. */
1321	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
1322	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
1323	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1324	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1325	/** @note Not for IOPL or IF testing. */
1326	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
1327	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
1328	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1329	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1330	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
1331	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
1332	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
1333	/** @note Not for IOPL or IF testing. */
1334	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1335	if ( pVCpu->cpum.GstCtx.cx != 0 \
1336	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1337	/** @note Not for IOPL or IF testing. */
1338	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1339	if ( pVCpu->cpum.GstCtx.ecx != 0 \
1340	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1341	/** @note Not for IOPL or IF testing. */
1342	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1343	if ( pVCpu->cpum.GstCtx.rcx != 0 \
1344	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1345	/** @note Not for IOPL or IF testing. */
1346	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1347	if ( pVCpu->cpum.GstCtx.cx != 0 \
1348	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1349	/** @note Not for IOPL or IF testing. */
1350	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1351	if ( pVCpu->cpum.GstCtx.ecx != 0 \
1352	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1353	/** @note Not for IOPL or IF testing. */
1354	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1355	if ( pVCpu->cpum.GstCtx.rcx != 0 \
1356	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1357	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
1358	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
1359
1360	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
1361	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[X86_FSW_TOP_GET_ST(pVCpu->cpum.GstCtx.XState.x87.FSW, a_iSt)].r80; } while (0)
1362	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
1363	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
1364	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
1365	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
1366	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
1367	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
1368	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
1369	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
1370	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
1371	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
1372	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
1373	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
1374	#define IEM_MC_IF_FCW_IM() \
1375	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
1376
1377	#define IEM_MC_ELSE() } else {
1378	#define IEM_MC_ENDIF() } do {} while (0)
1379
1380	/** @} */
1381
1382	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
1383

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 95421

Download in other formats: