IEMAllN8veRecompFuncs.h@ 103827

Last change on this file since 103827 was 103826, checked in by vboxsync, 9 months ago
VMM/IEM: Fixed missing variable index unpacking in iemNativeEmitMemCommitAndUnmap. bugref:10370
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 336.7 KB

Line
1	/* $Id: IEMAllN8veRecompFuncs.h 103826 2024-03-13 12:49:44Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler - Inlined Bits.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#define IEMNATIVE_INCL_TABLE_FUNCTION_PROTOTYPES
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/cpum.h>
39	#include <VBox/vmm/dbgf.h>
40	#include "IEMInternal.h"
41	#include <VBox/vmm/vmcc.h>
42	#include <VBox/log.h>
43	#include <VBox/err.h>
44	#include <VBox/dis.h>
45	#include <VBox/param.h>
46	#include <iprt/assert.h>
47	#include <iprt/heap.h>
48	#include <iprt/mem.h>
49	#include <iprt/string.h>
50	#if defined(RT_ARCH_AMD64)
51	# include <iprt/x86.h>
52	#elif defined(RT_ARCH_ARM64)
53	# include <iprt/armv8.h>
54	#endif
55
56	#include "IEMInline.h"
57	#include "IEMThreadedFunctions.h"
58	#include "IEMN8veRecompiler.h"
59	#include "IEMN8veRecompilerEmit.h"
60	#include "IEMN8veRecompilerTlbLookup.h"
61	#include "IEMNativeFunctions.h"
62
63
64	/*
65	* Narrow down configs here to avoid wasting time on unused configs here.
66	* Note! Same checks in IEMAllThrdRecompiler.cpp.
67	*/
68
69	#ifndef IEM_WITH_CODE_TLB
70	# error The code TLB must be enabled for the recompiler.
71	#endif
72
73	#ifndef IEM_WITH_DATA_TLB
74	# error The data TLB must be enabled for the recompiler.
75	#endif
76
77	#ifndef IEM_WITH_SETJMP
78	# error The setjmp approach must be enabled for the recompiler.
79	#endif
80
81
82
83	/*********************************************************************************************************************************
84	* Code emitters for flushing pending guest register writes and sanity checks *
85	*********************************************************************************************************************************/
86
87	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
88	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
89	DECL_INLINE_THROW(uint32_t) iemNativePcAdjustCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off)
90	{
91	/* Compare the shadow with the context value, they should match. */
92	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_REG_FIXED_TMP1, IEMNATIVE_REG_FIXED_PC_DBG);
93	off = iemNativeEmitAddGprImm(pReNative, off, IEMNATIVE_REG_FIXED_TMP1, pReNative->Core.offPc);
94	off = iemNativeEmitGuestRegValueCheck(pReNative, off, IEMNATIVE_REG_FIXED_TMP1, kIemNativeGstReg_Pc);
95	return off;
96	}
97	# endif
98	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
99
100	/**
101	* Flushes delayed write of a specific guest register.
102	*
103	* This must be called prior to calling CImpl functions and any helpers that use
104	* the guest state (like raising exceptions) and such.
105	*
106	* This optimization has not yet been implemented. The first target would be
107	* RIP updates, since these are the most common ones.
108	*/
109	DECL_INLINE_THROW(uint32_t)
110	iemNativeRegFlushPendingSpecificWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREGREF enmClass, uint8_t idxReg)
111	{
112	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
113	/* If for whatever reason it is possible to reference the PC register at some point we need to do the writeback here first. */
114	#endif
115
116	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
117	if ( enmClass == kIemNativeGstRegRef_XReg
118	&& pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(idxReg))
119	{
120	off = iemNativeSimdRegFlushPendingWrite(pReNative, off, IEMNATIVEGSTSIMDREG_SIMD(idxReg));
121	/* Flush the shadows as the register needs to be reloaded (there is no guarantee right now, that the referenced register doesn't change). */
122	uint8_t const idxHstSimdReg = pReNative->Core.aidxGstSimdRegShadows[idxReg];
123
124	iemNativeSimdRegClearGstSimdRegShadowing(pReNative, idxHstSimdReg, off);
125	iemNativeSimdRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTSIMDREG_SIMD(idxReg)));
126	}
127	#endif
128	RT_NOREF(pReNative, enmClass, idxReg);
129	return off;
130	}
131
132
133
134	/*********************************************************************************************************************************
135	* Emitters for IEM_MC_BEGIN and IEM_MC_END. *
136	*********************************************************************************************************************************/
137
138	#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) \
139	{ \
140	Assert(pReNative->Core.bmVars == 0); \
141	Assert(pReNative->Core.u64ArgVars == UINT64_MAX); \
142	Assert(pReNative->Core.bmStack == 0); \
143	pReNative->fMc = (a_fMcFlags); \
144	pReNative->fCImpl = (a_fCImplFlags); \
145	pReNative->cArgs = ((a_cArgs) + iemNativeArgGetHiddenArgCount(pReNative))
146
147	/** We have to get to the end in recompilation mode, as otherwise we won't
148	* generate code for all the IEM_MC_IF_XXX branches. */
149	#define IEM_MC_END() \
150	iemNativeVarFreeAll(pReNative); \
151	} return off
152
153
154
155	/*********************************************************************************************************************************
156	* Native Emitter Support. *
157	*********************************************************************************************************************************/
158
159	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (RT_ARCH_VAL & (a_fSupportedHosts)) {
160
161	#define IEM_MC_NATIVE_ELSE() } else {
162
163	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
164
165
166	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter) \
167	off = a_fnEmitter(pReNative, off)
168
169	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) \
170	off = a_fnEmitter(pReNative, off, (a0))
171
172	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) \
173	off = a_fnEmitter(pReNative, off, (a0), (a1))
174
175	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) \
176	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2))
177
178	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) \
179	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2), (a3))
180
181	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) \
182	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2), (a3), (a4))
183
184	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) \
185	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2), (a3), (a4), (a5))
186
187	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) \
188	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2), (a3), (a4), (a5), (a6))
189
190	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) \
191	off = a_fnEmitter(pReNative, off, (a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7))
192
193
194
195	/*********************************************************************************************************************************
196	* Emitters for standalone C-implementation deferals (IEM_MC_DEFER_TO_CIMPL_XXXX) *
197	*********************************************************************************************************************************/
198
199	#define IEM_MC_DEFER_TO_CIMPL_0_RET_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
200	pReNative->fMc = 0; \
201	pReNative->fCImpl = (a_fFlags); \
202	return iemNativeEmitCImplCall0(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, a_cbInstr) /** @todo not used ... */
203
204
205	#define IEM_MC_DEFER_TO_CIMPL_1_RET_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
206	pReNative->fMc = 0; \
207	pReNative->fCImpl = (a_fFlags); \
208	return iemNativeEmitCImplCall1(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, a_cbInstr, a0)
209
210	DECL_INLINE_THROW(uint32_t) iemNativeEmitCImplCall1(PIEMRECOMPILERSTATE pReNative, uint32_t off,
211	uint8_t idxInstr, uint64_t a_fGstShwFlush,
212	uintptr_t pfnCImpl, uint8_t cbInstr, uint64_t uArg0)
213	{
214	return iemNativeEmitCImplCall(pReNative, off, idxInstr, a_fGstShwFlush, pfnCImpl, cbInstr, 1, uArg0, 0, 0);
215	}
216
217
218	#define IEM_MC_DEFER_TO_CIMPL_2_RET_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
219	pReNative->fMc = 0; \
220	pReNative->fCImpl = (a_fFlags); \
221	return iemNativeEmitCImplCall2(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, \
222	(uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1)
223
224	DECL_INLINE_THROW(uint32_t) iemNativeEmitCImplCall2(PIEMRECOMPILERSTATE pReNative, uint32_t off,
225	uint8_t idxInstr, uint64_t a_fGstShwFlush,
226	uintptr_t pfnCImpl, uint8_t cbInstr, uint64_t uArg0, uint64_t uArg1)
227	{
228	return iemNativeEmitCImplCall(pReNative, off, idxInstr, a_fGstShwFlush, pfnCImpl, cbInstr, 2, uArg0, uArg1, 0);
229	}
230
231
232	#define IEM_MC_DEFER_TO_CIMPL_3_RET_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
233	pReNative->fMc = 0; \
234	pReNative->fCImpl = (a_fFlags); \
235	return iemNativeEmitCImplCall3(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, \
236	(uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1, a2)
237
238	DECL_INLINE_THROW(uint32_t) iemNativeEmitCImplCall3(PIEMRECOMPILERSTATE pReNative, uint32_t off,
239	uint8_t idxInstr, uint64_t a_fGstShwFlush,
240	uintptr_t pfnCImpl, uint8_t cbInstr, uint64_t uArg0, uint64_t uArg1,
241	uint64_t uArg2)
242	{
243	return iemNativeEmitCImplCall(pReNative, off, idxInstr, a_fGstShwFlush, pfnCImpl, cbInstr, 3, uArg0, uArg1, uArg2);
244	}
245
246
247
248	/*********************************************************************************************************************************
249	* Emitters for advancing PC/RIP/EIP/IP (IEM_MC_ADVANCE_RIP_AND_FINISH_XXX) *
250	*********************************************************************************************************************************/
251
252	/** Emits the flags check for IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC64_WITH_FLAGS
253	* and the other _WITH_FLAGS MCs, see iemRegFinishClearingRF. */
254	DECL_INLINE_THROW(uint32_t)
255	iemNativeEmitFinishInstructionFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off)
256	{
257	/*
258	* If its not just X86_EFL_RF and CPUMCTX_INHIBIT_SHADOW that are set, we
259	* return with special status code and make the execution loop deal with
260	* this. If TF or CPUMCTX_DBG_HIT_DRX_MASK triggers, we have to raise an
261	* exception and won't continue execution. While CPUMCTX_DBG_DBGF_MASK
262	* could continue w/o interruption, it probably will drop into the
263	* debugger, so not worth the effort of trying to services it here and we
264	* just lump it in with the handling of the others.
265	*
266	* To simplify the code and the register state management even more (wrt
267	* immediate in AND operation), we always update the flags and skip the
268	* extra check associated conditional jump.
269	*/
270	AssertCompile( (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)
271	<= UINT32_MAX);
272	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
273	AssertMsg( pReNative->idxCurCall == 0
274	\|\| IEMLIVENESS_STATE_IS_INPUT_EXPECTED(iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], IEMLIVENESSBIT_IDX_EFL_OTHER)),
275	("Efl_Other - %u\n", iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], IEMLIVENESSBIT_IDX_EFL_OTHER)));
276	#endif
277
278	/*
279	* As this code can break out of the execution loop when jumping to the ReturnWithFlags label
280	* any pending register writes must be flushed.
281	*/
282	off = iemNativeRegFlushPendingWrites(pReNative, off);
283
284	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
285	kIemNativeGstRegUse_ForUpdate, false /fNoVolatileRegs/,
286	true /fSkipLivenessAssert/);
287	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxEflReg,
288	X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK,
289	iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnWithFlags));
290	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxEflReg, ~(uint32_t)(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW));
291	off = iemNativeEmitStoreGprToVCpuU32(pReNative, off, idxEflReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
292
293	/* Free but don't flush the EFLAGS register. */
294	iemNativeRegFreeTmp(pReNative, idxEflReg);
295
296	return off;
297	}
298
299
300	/** The VINF_SUCCESS dummy. */
301	template<int const a_rcNormal>
302	DECL_FORCE_INLINE(uint32_t)
303	iemNativeEmitFinishInstructionWithStatus(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
304	{
305	AssertCompile(a_rcNormal == VINF_SUCCESS \|\| a_rcNormal == VINF_IEM_REEXEC_BREAK);
306	if (a_rcNormal != VINF_SUCCESS)
307	{
308	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
309	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
310	#else
311	RT_NOREF_PV(idxInstr);
312	#endif
313
314	/* As this code returns from the TB any pending register writes must be flushed. */
315	off = iemNativeRegFlushPendingWrites(pReNative, off);
316
317	return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreak);
318	}
319	return off;
320	}
321
322
323	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC64(a_cbInstr, a_rcNormal) \
324	off = iemNativeEmitAddToRip64AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
325	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
326
327	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_cbInstr, a_rcNormal) \
328	off = iemNativeEmitAddToRip64AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
329	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
330	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
331
332	/** Same as iemRegAddToRip64AndFinishingNoFlags. */
333	DECL_INLINE_THROW(uint32_t)
334	iemNativeEmitAddToRip64AndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
335	{
336	#if !defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) \|\| defined(IEMNATIVE_REG_FIXED_PC_DBG)
337	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
338	if (!pReNative->Core.offPc)
339	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, IEMNATIVE_REG_FIXED_PC_DBG, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
340	# endif
341
342	/* Allocate a temporary PC register. */
343	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
344
345	/* Perform the addition and store the result. */
346	off = iemNativeEmitAddGprImm8(pReNative, off, idxPcReg, cbInstr);
347	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
348
349	/* Free but don't flush the PC register. */
350	iemNativeRegFreeTmp(pReNative, idxPcReg);
351	#endif
352
353	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
354	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
355
356	pReNative->Core.offPc += cbInstr;
357	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
358	off = iemNativePcAdjustCheck(pReNative, off);
359	# endif
360	if (pReNative->cCondDepth)
361	off = iemNativeEmitPcWriteback(pReNative, off);
362	else
363	pReNative->Core.cInstrPcUpdateSkipped++;
364	#endif
365
366	return off;
367	}
368
369
370	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC32(a_cbInstr, a_rcNormal) \
371	off = iemNativeEmitAddToEip32AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
372	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
373
374	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_cbInstr, a_rcNormal) \
375	off = iemNativeEmitAddToEip32AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
376	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
377	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
378
379	/** Same as iemRegAddToEip32AndFinishingNoFlags. */
380	DECL_INLINE_THROW(uint32_t)
381	iemNativeEmitAddToEip32AndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
382	{
383	#if !defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) \|\| defined(IEMNATIVE_REG_FIXED_PC_DBG)
384	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
385	if (!pReNative->Core.offPc)
386	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, IEMNATIVE_REG_FIXED_PC_DBG, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
387	# endif
388
389	/* Allocate a temporary PC register. */
390	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
391
392	/* Perform the addition and store the result. */
393	off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxPcReg, cbInstr);
394	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
395
396	/* Free but don't flush the PC register. */
397	iemNativeRegFreeTmp(pReNative, idxPcReg);
398	#endif
399
400	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
401	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
402
403	pReNative->Core.offPc += cbInstr;
404	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
405	off = iemNativePcAdjustCheck(pReNative, off);
406	# endif
407	if (pReNative->cCondDepth)
408	off = iemNativeEmitPcWriteback(pReNative, off);
409	else
410	pReNative->Core.cInstrPcUpdateSkipped++;
411	#endif
412
413	return off;
414	}
415
416
417	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC16(a_cbInstr, a_rcNormal) \
418	off = iemNativeEmitAddToIp16AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
419	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
420
421	#define IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC16_WITH_FLAGS(a_cbInstr, a_rcNormal) \
422	off = iemNativeEmitAddToIp16AndFinishingNoFlags(pReNative, off, (a_cbInstr)); \
423	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
424	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
425
426	/** Same as iemRegAddToIp16AndFinishingNoFlags. */
427	DECL_INLINE_THROW(uint32_t)
428	iemNativeEmitAddToIp16AndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
429	{
430	#if !defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) \|\| defined(IEMNATIVE_REG_FIXED_PC_DBG)
431	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
432	if (!pReNative->Core.offPc)
433	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, IEMNATIVE_REG_FIXED_PC_DBG, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
434	# endif
435
436	/* Allocate a temporary PC register. */
437	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
438
439	/* Perform the addition and store the result. */
440	off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxPcReg, cbInstr);
441	off = iemNativeEmitClear16UpGpr(pReNative, off, idxPcReg);
442	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
443
444	/* Free but don't flush the PC register. */
445	iemNativeRegFreeTmp(pReNative, idxPcReg);
446	#endif
447
448	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
449	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
450
451	pReNative->Core.offPc += cbInstr;
452	# if defined(IEMNATIVE_REG_FIXED_PC_DBG)
453	off = iemNativePcAdjustCheck(pReNative, off);
454	# endif
455	if (pReNative->cCondDepth)
456	off = iemNativeEmitPcWriteback(pReNative, off);
457	else
458	pReNative->Core.cInstrPcUpdateSkipped++;
459	#endif
460
461	return off;
462	}
463
464
465
466	/*********************************************************************************************************************************
467	* Emitters for changing PC/RIP/EIP/IP with a relative jump (IEM_MC_REL_JMP_XXX_AND_FINISH_XXX). *
468	*********************************************************************************************************************************/
469
470	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC64(a_i8, a_cbInstr, a_enmEffOpSize, a_rcNormal) \
471	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), \
472	(a_enmEffOpSize), pCallEntry->idxInstr); \
473	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
474
475	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_i8, a_cbInstr, a_enmEffOpSize, a_rcNormal) \
476	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), \
477	(a_enmEffOpSize), pCallEntry->idxInstr); \
478	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
479	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
480
481	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC64(a_i16, a_cbInstr, a_rcNormal) \
482	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), \
483	IEMMODE_16BIT, pCallEntry->idxInstr); \
484	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
485
486	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_i16, a_cbInstr, a_rcNormal) \
487	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), \
488	IEMMODE_16BIT, pCallEntry->idxInstr); \
489	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
490	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
491
492	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC64(a_i32, a_cbInstr, a_rcNormal) \
493	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), \
494	IEMMODE_64BIT, pCallEntry->idxInstr); \
495	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
496
497	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_i32, a_cbInstr, a_rcNormal) \
498	off = iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), \
499	IEMMODE_64BIT, pCallEntry->idxInstr); \
500	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
501	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
502
503	/** Same as iemRegRip64RelativeJumpS8AndFinishNoFlags,
504	* iemRegRip64RelativeJumpS16AndFinishNoFlags and
505	* iemRegRip64RelativeJumpS32AndFinishNoFlags. */
506	DECL_INLINE_THROW(uint32_t)
507	iemNativeEmitRip64RelativeJumpAndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr,
508	int32_t offDisp, IEMMODE enmEffOpSize, uint8_t idxInstr)
509	{
510	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
511
512	/* We speculatively modify PC and may raise #GP(0), so make sure the right values are in CPUMCTX. */
513	off = iemNativeRegFlushPendingWrites(pReNative, off);
514
515	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
516	Assert(pReNative->Core.offPc == 0);
517
518	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
519	#endif
520
521	/* Allocate a temporary PC register. */
522	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
523
524	/* Perform the addition. */
525	off = iemNativeEmitAddGprImm(pReNative, off, idxPcReg, (int64_t)offDisp + cbInstr);
526
527	if (RT_LIKELY(enmEffOpSize == IEMMODE_64BIT))
528	{
529	/* Check that the address is canonical, raising #GP(0) + exit TB if it isn't. */
530	off = iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(pReNative, off, idxPcReg, idxInstr);
531	}
532	else
533	{
534	/* Just truncate the result to 16-bit IP. */
535	Assert(enmEffOpSize == IEMMODE_16BIT);
536	off = iemNativeEmitClear16UpGpr(pReNative, off, idxPcReg);
537	}
538	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
539
540	/* Free but don't flush the PC register. */
541	iemNativeRegFreeTmp(pReNative, idxPcReg);
542
543	return off;
544	}
545
546
547	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC32(a_i8, a_cbInstr, a_enmEffOpSize, a_rcNormal) \
548	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), \
549	(a_enmEffOpSize), pCallEntry->idxInstr); \
550	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
551
552	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_i8, a_cbInstr, a_enmEffOpSize, a_rcNormal) \
553	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), \
554	(a_enmEffOpSize), pCallEntry->idxInstr); \
555	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
556	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
557
558	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC32(a_i16, a_cbInstr, a_rcNormal) \
559	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), \
560	IEMMODE_16BIT, pCallEntry->idxInstr); \
561	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
562
563	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_i16, a_cbInstr, a_rcNormal) \
564	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), \
565	IEMMODE_16BIT, pCallEntry->idxInstr); \
566	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
567	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
568
569	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC32(a_i32, a_cbInstr, a_rcNormal) \
570	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), \
571	IEMMODE_32BIT, pCallEntry->idxInstr); \
572	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
573
574	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_i32, a_cbInstr, a_rcNormal) \
575	off = iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), \
576	IEMMODE_32BIT, pCallEntry->idxInstr); \
577	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
578	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
579
580	/** Same as iemRegEip32RelativeJumpS8AndFinishNoFlags,
581	* iemRegEip32RelativeJumpS16AndFinishNoFlags and
582	* iemRegEip32RelativeJumpS32AndFinishNoFlags. */
583	DECL_INLINE_THROW(uint32_t)
584	iemNativeEmitEip32RelativeJumpAndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr,
585	int32_t offDisp, IEMMODE enmEffOpSize, uint8_t idxInstr)
586	{
587	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
588
589	/* We speculatively modify PC and may raise #GP(0), so make sure the right values are in CPUMCTX. */
590	off = iemNativeRegFlushPendingWrites(pReNative, off);
591
592	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
593	Assert(pReNative->Core.offPc == 0);
594
595	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
596	#endif
597
598	/* Allocate a temporary PC register. */
599	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
600
601	/* Perform the addition. */
602	off = iemNativeEmitAddGpr32Imm(pReNative, off, idxPcReg, offDisp + cbInstr);
603
604	/* Truncate the result to 16-bit IP if the operand size is 16-bit. */
605	if (enmEffOpSize == IEMMODE_16BIT)
606	off = iemNativeEmitClear16UpGpr(pReNative, off, idxPcReg);
607
608	/* Perform limit checking, potentially raising #GP(0) and exit the TB. */
609	/** @todo we can skip this in 32-bit FLAT mode. */
610	off = iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(pReNative, off, idxPcReg, idxInstr);
611
612	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
613
614	/* Free but don't flush the PC register. */
615	iemNativeRegFreeTmp(pReNative, idxPcReg);
616
617	return off;
618	}
619
620
621	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC16(a_i8, a_cbInstr, a_rcNormal) \
622	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), pCallEntry->idxInstr); \
623	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
624
625	#define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC16_WITH_FLAGS(a_i8, a_cbInstr, a_rcNormal) \
626	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int8_t)(a_i8), pCallEntry->idxInstr); \
627	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
628	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
629
630	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC16(a_i16, a_cbInstr, a_rcNormal) \
631	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), pCallEntry->idxInstr); \
632	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
633
634	#define IEM_MC_REL_JMP_S16_AND_FINISH_THREADED_PC16_WITH_FLAGS(a_i16, a_cbInstr, a_rcNormal) \
635	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (int16_t)(a_i16), pCallEntry->idxInstr); \
636	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
637	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
638
639	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC16(a_i32, a_cbInstr, a_rcNormal) \
640	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), pCallEntry->idxInstr); \
641	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
642
643	#define IEM_MC_REL_JMP_S32_AND_FINISH_THREADED_PC16_WITH_FLAGS(a_i32, a_cbInstr, a_rcNormal) \
644	off = iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(pReNative, off, (a_cbInstr), (a_i32), pCallEntry->idxInstr); \
645	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off); \
646	off = iemNativeEmitFinishInstructionWithStatus<a_rcNormal>(pReNative, off, pCallEntry->idxInstr)
647
648	/** Same as iemRegIp16RelativeJumpS8AndFinishNoFlags. */
649	DECL_INLINE_THROW(uint32_t)
650	iemNativeEmitIp16RelativeJumpAndFinishingNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off,
651	uint8_t cbInstr, int32_t offDisp, uint8_t idxInstr)
652	{
653	/* We speculatively modify PC and may raise #GP(0), so make sure the right values are in CPUMCTX. */
654	off = iemNativeRegFlushPendingWrites(pReNative, off);
655
656	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
657	Assert(pReNative->Core.offPc == 0);
658
659	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
660	#endif
661
662	/* Allocate a temporary PC register. */
663	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
664
665	/* Perform the addition, clamp the result, check limit (may #GP(0) + exit TB) and store the result. */
666	off = iemNativeEmitAddGpr32Imm(pReNative, off, idxPcReg, offDisp + cbInstr);
667	off = iemNativeEmitClear16UpGpr(pReNative, off, idxPcReg);
668	off = iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(pReNative, off, idxPcReg, idxInstr);
669	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
670
671	/* Free but don't flush the PC register. */
672	iemNativeRegFreeTmp(pReNative, idxPcReg);
673
674	return off;
675	}
676
677
678
679	/*********************************************************************************************************************************
680	* Emitters for changing PC/RIP/EIP/IP with a indirect jump (IEM_MC_SET_RIP_UXX_AND_FINISH). *
681	*********************************************************************************************************************************/
682
683	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for pre-386 targets. */
684	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC16(a_u16NewIP) \
685	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u16NewIP), false /f64Bit/, pCallEntry->idxInstr, sizeof(uint16_t))
686
687	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for 386+ targets. */
688	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC32(a_u16NewIP) \
689	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u16NewIP), false /f64Bit/, pCallEntry->idxInstr, sizeof(uint16_t))
690
691	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for use in 64-bit code. */
692	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC64(a_u16NewIP) \
693	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u16NewIP), true /f64Bit/, pCallEntry->idxInstr, sizeof(uint16_t))
694
695	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for pre-386 targets that checks and
696	* clears flags. */
697	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC16_WITH_FLAGS(a_u16NewIP) \
698	IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC16(a_u16NewIP); \
699	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
700
701	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for 386+ targets that checks and
702	* clears flags. */
703	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_u16NewIP) \
704	IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC32(a_u16NewIP); \
705	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
706
707	/** Variant of IEM_MC_SET_RIP_U16_AND_FINISH for use in 64-bit code that checks and
708	* clears flags. */
709	#define IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_u16NewIP) \
710	IEM_MC_SET_RIP_U16_AND_FINISH_THREADED_PC64(a_u16NewIP); \
711	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
712
713	#undef IEM_MC_SET_RIP_U16_AND_FINISH
714
715
716	/** Variant of IEM_MC_SET_RIP_U32_AND_FINISH for 386+ targets. */
717	#define IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC32(a_u32NewEIP) \
718	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u32NewEIP), false /f64Bit/, pCallEntry->idxInstr, sizeof(uint32_t))
719
720	/** Variant of IEM_MC_SET_RIP_U32_AND_FINISH for use in 64-bit code. */
721	#define IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC64(a_u32NewEIP) \
722	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u32NewEIP), true /f64Bit/, pCallEntry->idxInstr, sizeof(uint32_t))
723
724	/** Variant of IEM_MC_SET_RIP_U32_AND_FINISH for 386+ targets that checks and
725	* clears flags. */
726	#define IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC32_WITH_FLAGS(a_u32NewEIP) \
727	IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC32(a_u32NewEIP); \
728	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
729
730	/** Variant of IEM_MC_SET_RIP_U32_AND_FINISH for use in 64-bit code that checks
731	* and clears flags. */
732	#define IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_u32NewEIP) \
733	IEM_MC_SET_RIP_U32_AND_FINISH_THREADED_PC64(a_u32NewEIP); \
734	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
735
736	#undef IEM_MC_SET_RIP_U32_AND_FINISH
737
738
739	/** Variant of IEM_MC_SET_RIP_U64_AND_FINISH for use in 64-bit code. */
740	#define IEM_MC_SET_RIP_U64_AND_FINISH_THREADED_PC64(a_u64NewEIP) \
741	off = iemNativeEmitRipJumpNoFlags(pReNative, off, (a_u64NewEIP), true /f64Bit/, pCallEntry->idxInstr, sizeof(uint64_t))
742
743	/** Variant of IEM_MC_SET_RIP_U64_AND_FINISH for use in 64-bit code that checks
744	* and clears flags. */
745	#define IEM_MC_SET_RIP_U64_AND_FINISH_THREADED_PC64_WITH_FLAGS(a_u64NewEIP) \
746	IEM_MC_SET_RIP_U64_AND_FINISH_THREADED_PC64(a_u64NewEIP); \
747	off = iemNativeEmitFinishInstructionFlagsCheck(pReNative, off)
748
749	#undef IEM_MC_SET_RIP_U64_AND_FINISH
750
751
752	/** Same as iemRegRipJumpU16AndFinishNoFlags,
753	* iemRegRipJumpU32AndFinishNoFlags and iemRegRipJumpU64AndFinishNoFlags. */
754	DECL_INLINE_THROW(uint32_t)
755	iemNativeEmitRipJumpNoFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarPc, bool f64Bit,
756	uint8_t idxInstr, uint8_t cbVar)
757	{
758	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarPc);
759	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarPc, cbVar);
760
761	/* We speculatively modify PC and may raise #GP(0), so make sure the right values are in CPUMCTX. */
762	off = iemNativeRegFlushPendingWrites(pReNative, off);
763
764	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
765	Assert(pReNative->Core.offPc == 0);
766
767	STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativePcUpdateTotal);
768	#endif
769
770	/* Get a register with the new PC loaded from idxVarPc.
771	Note! This ASSUMES that the high bits of the GPR is zeroed. */
772	uint8_t const idxPcReg = iemNativeVarRegisterAcquireForGuestReg(pReNative, idxVarPc, kIemNativeGstReg_Pc, &off);
773
774	/* Check limit (may #GP(0) + exit TB). */
775	if (!f64Bit)
776	/** @todo we can skip this test in FLAT 32-bit mode. */
777	off = iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(pReNative, off, idxPcReg, idxInstr);
778	/* Check that the address is canonical, raising #GP(0) + exit TB if it isn't. */
779	else if (cbVar > sizeof(uint32_t))
780	off = iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(pReNative, off, idxPcReg, idxInstr);
781
782	/* Store the result. */
783	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
784
785	iemNativeVarRegisterRelease(pReNative, idxVarPc);
786	/** @todo implictly free the variable? */
787
788	return off;
789	}
790
791
792
793	/*********************************************************************************************************************************
794	* Emitters for raising exceptions (IEM_MC_MAYBE_RAISE_XXX) *
795	*********************************************************************************************************************************/
796
797	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
798	off = iemNativeEmitMaybeRaiseDeviceNotAvailable(pReNative, off, pCallEntry->idxInstr)
799
800	/**
801	* Emits code to check if a \#NM exception should be raised.
802	*
803	* @returns New code buffer offset, UINT32_MAX on failure.
804	* @param pReNative The native recompile state.
805	* @param off The code buffer offset.
806	* @param idxInstr The current instruction.
807	*/
808	DECL_INLINE_THROW(uint32_t)
809	iemNativeEmitMaybeRaiseDeviceNotAvailable(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
810	{
811	/*
812	* Make sure we don't have any outstanding guest register writes as we may
813	* raise an #NM and all guest register must be up to date in CPUMCTX.
814	*
815	* @todo r=aeichner Can we postpone this to the RaiseNm path?
816	*/
817	off = iemNativeRegFlushPendingWrites(pReNative, off);
818
819	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
820	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
821	#else
822	RT_NOREF(idxInstr);
823	#endif
824
825	/* Allocate a temporary CR0 register. */
826	uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
827	uint8_t const idxLabelRaiseNm = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseNm);
828
829	/*
830	* if (cr0 & (X86_CR0_EM \| X86_CR0_TS) != 0)
831	* return raisexcpt();
832	*/
833	/* Test and jump. */
834	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM \| X86_CR0_TS, idxLabelRaiseNm);
835
836	/* Free but don't flush the CR0 register. */
837	iemNativeRegFreeTmp(pReNative, idxCr0Reg);
838
839	return off;
840	}
841
842
843	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
844	off = iemNativeEmitMaybeFpuException(pReNative, off, pCallEntry->idxInstr)
845
846	/**
847	* Emits code to check if a \#MF exception should be raised.
848	*
849	* @returns New code buffer offset, UINT32_MAX on failure.
850	* @param pReNative The native recompile state.
851	* @param off The code buffer offset.
852	* @param idxInstr The current instruction.
853	*/
854	DECL_INLINE_THROW(uint32_t)
855	iemNativeEmitMaybeRaiseFpuException(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
856	{
857	/*
858	* Make sure we don't have any outstanding guest register writes as we may
859	* raise an #MF and all guest register must be up to date in CPUMCTX.
860	*
861	* @todo r=aeichner Can we postpone this to the RaiseMf path?
862	*/
863	off = iemNativeRegFlushPendingWrites(pReNative, off);
864
865	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
866	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
867	#else
868	RT_NOREF(idxInstr);
869	#endif
870
871	/* Allocate a temporary FSW register. */
872	uint8_t const idxFpuFswReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFsw, kIemNativeGstRegUse_ReadOnly);
873	uint8_t const idxLabelRaiseMf = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseMf);
874
875	/*
876	* if (FSW & X86_FSW_ES != 0)
877	* return raisexcpt();
878	*/
879	/* Test and jump. */
880	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxFpuFswReg, X86_FSW_ES, idxLabelRaiseMf);
881
882	/* Free but don't flush the FSW register. */
883	iemNativeRegFreeTmp(pReNative, idxFpuFswReg);
884
885	return off;
886	}
887
888
889	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
890	off = iemNativeEmitMaybeRaiseSseRelatedXcpt(pReNative, off, pCallEntry->idxInstr)
891
892	/**
893	* Emits code to check if a SSE exception (either \#UD or \#NM) should be raised.
894	*
895	* @returns New code buffer offset, UINT32_MAX on failure.
896	* @param pReNative The native recompile state.
897	* @param off The code buffer offset.
898	* @param idxInstr The current instruction.
899	*/
900	DECL_INLINE_THROW(uint32_t)
901	iemNativeEmitMaybeRaiseSseRelatedXcpt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
902	{
903	/*
904	* Make sure we don't have any outstanding guest register writes as we may
905	* raise an \#UD or \#NM and all guest register must be up to date in CPUMCTX.
906	*
907	* @todo r=aeichner Can we postpone this to the RaiseNm/RaiseUd path?
908	*/
909	off = iemNativeRegFlushPendingWrites(pReNative, off, false /fFlushShadows/);
910
911	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
912	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
913	#else
914	RT_NOREF(idxInstr);
915	#endif
916
917	/* Allocate a temporary CR0 and CR4 register. */
918	uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
919	uint8_t const idxCr4Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4, kIemNativeGstRegUse_ReadOnly);
920	uint8_t const idxLabelRaiseNm = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseNm);
921	uint8_t const idxLabelRaiseUd = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseUd);
922
923	/** @todo r=aeichner Optimize this more later to have less compares and branches,
924	* (see IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() in IEMMc.h but check that it has some
925	* actual performance benefit first). */
926	/*
927	* if (cr0 & X86_CR0_EM)
928	* return raisexcpt();
929	*/
930	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxCr0Reg, X86_CR0_EM_BIT, idxLabelRaiseUd);
931	/*
932	* if (!(cr4 & X86_CR4_OSFXSR))
933	* return raisexcpt();
934	*/
935	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxCr4Reg, X86_CR4_OSFXSR_BIT, idxLabelRaiseUd);
936	/*
937	* if (cr0 & X86_CR0_TS)
938	* return raisexcpt();
939	*/
940	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxCr0Reg, X86_CR0_TS_BIT, idxLabelRaiseNm);
941
942	/* Free but don't flush the CR0 and CR4 register. */
943	iemNativeRegFreeTmp(pReNative, idxCr0Reg);
944	iemNativeRegFreeTmp(pReNative, idxCr4Reg);
945
946	return off;
947	}
948
949
950	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
951	off = iemNativeEmitMaybeRaiseAvxRelatedXcpt(pReNative, off, pCallEntry->idxInstr)
952
953	/**
954	* Emits code to check if a AVX exception (either \#UD or \#NM) should be raised.
955	*
956	* @returns New code buffer offset, UINT32_MAX on failure.
957	* @param pReNative The native recompile state.
958	* @param off The code buffer offset.
959	* @param idxInstr The current instruction.
960	*/
961	DECL_INLINE_THROW(uint32_t)
962	iemNativeEmitMaybeRaiseAvxRelatedXcpt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
963	{
964	/*
965	* Make sure we don't have any outstanding guest register writes as we may
966	* raise an \#UD or \#NM and all guest register must be up to date in CPUMCTX.
967	*
968	* @todo r=aeichner Can we postpone this to the RaiseNm/RaiseUd path?
969	*/
970	off = iemNativeRegFlushPendingWrites(pReNative, off, false /fFlushShadows/);
971
972	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
973	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
974	#else
975	RT_NOREF(idxInstr);
976	#endif
977
978	/* Allocate a temporary CR0, CR4 and XCR0 register. */
979	uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
980	uint8_t const idxCr4Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4, kIemNativeGstRegUse_ReadOnly);
981	uint8_t const idxXcr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Xcr0, kIemNativeGstRegUse_ReadOnly);
982	uint8_t const idxLabelRaiseNm = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseNm);
983	uint8_t const idxLabelRaiseUd = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseUd);
984
985	/** @todo r=aeichner Optimize this more later to have less compares and branches,
986	* (see IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() in IEMMc.h but check that it has some
987	* actual performance benefit first). */
988	/*
989	* if ((xcr0 & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE))
990	* return raisexcpt();
991	*/
992	const uint8_t idxRegTmp = iemNativeRegAllocTmpImm(pReNative, &off, XSAVE_C_YMM \| XSAVE_C_SSE);
993	off = iemNativeEmitAndGprByGpr(pReNative, off, idxRegTmp, idxXcr0Reg);
994	off = iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, idxRegTmp, XSAVE_C_YMM \| XSAVE_C_SSE, idxLabelRaiseUd);
995	iemNativeRegFreeTmp(pReNative, idxRegTmp);
996
997	/*
998	* if (!(cr4 & X86_CR4_OSXSAVE))
999	* return raisexcpt();
1000	*/
1001	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxCr4Reg, X86_CR4_OSXSAVE_BIT, idxLabelRaiseUd);
1002	/*
1003	* if (cr0 & X86_CR0_TS)
1004	* return raisexcpt();
1005	*/
1006	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxCr0Reg, X86_CR0_TS_BIT, idxLabelRaiseNm);
1007
1008	/* Free but don't flush the CR0, CR4 and XCR0 register. */
1009	iemNativeRegFreeTmp(pReNative, idxCr0Reg);
1010	iemNativeRegFreeTmp(pReNative, idxCr4Reg);
1011	iemNativeRegFreeTmp(pReNative, idxXcr0Reg);
1012
1013	return off;
1014	}
1015
1016
1017	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
1018	off = iemNativeEmitRaiseSseAvxSimdFpXcpt(pReNative, off, pCallEntry->idxInstr)
1019
1020	/**
1021	* Emits code to raise a SIMD floating point (either \#UD or \#XF) should be raised.
1022	*
1023	* @returns New code buffer offset, UINT32_MAX on failure.
1024	* @param pReNative The native recompile state.
1025	* @param off The code buffer offset.
1026	* @param idxInstr The current instruction.
1027	*/
1028	DECL_INLINE_THROW(uint32_t)
1029	iemNativeEmitRaiseSseAvxSimdFpXcpt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
1030	{
1031	/*
1032	* Make sure we don't have any outstanding guest register writes as we may
1033	* raise an \#UD or \#NM and all guest register must be up to date in CPUMCTX.
1034	*
1035	* @todo r=aeichner Can we postpone this to the RaiseNm/RaiseUd path?
1036	*/
1037	off = iemNativeRegFlushPendingWrites(pReNative, off);
1038
1039	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
1040	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
1041	#else
1042	RT_NOREF(idxInstr);
1043	#endif
1044
1045	/* Allocate a temporary CR4 register. */
1046	uint8_t const idxCr4Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4, kIemNativeGstRegUse_ReadOnly);
1047	uint8_t const idxLabelRaiseXf = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseXf);
1048	uint8_t const idxLabelRaiseUd = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseUd);
1049
1050	/*
1051	* if (!(cr4 & X86_CR4_OSXMMEEXCPT))
1052	* return raisexcpt();
1053	*/
1054	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxCr4Reg, X86_CR4_OSXMMEEXCPT_BIT, idxLabelRaiseXf);
1055
1056	/* raise \#UD exception unconditionally. */
1057	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelRaiseUd);
1058
1059	/* Free but don't flush the CR4 register. */
1060	iemNativeRegFreeTmp(pReNative, idxCr4Reg);
1061
1062	return off;
1063	}
1064
1065
1066
1067	/*********************************************************************************************************************************
1068	* Emitters for conditionals (IEM_MC_IF_XXX, IEM_MC_ELSE, IEM_MC_ENDIF) *
1069	*********************************************************************************************************************************/
1070
1071	/**
1072	* Pushes an IEM_MC_IF_XXX onto the condition stack.
1073	*
1074	* @returns Pointer to the condition stack entry on success, NULL on failure
1075	* (too many nestings)
1076	*/
1077	DECL_INLINE_THROW(PIEMNATIVECOND) iemNativeCondPushIf(PIEMRECOMPILERSTATE pReNative, uint32_t *poff)
1078	{
1079	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1080	poff = iemNativeRegFlushPendingWrites(pReNative, poff);
1081	#endif
1082
1083	uint32_t const idxStack = pReNative->cCondDepth;
1084	AssertStmt(idxStack < RT_ELEMENTS(pReNative->aCondStack), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_COND_TOO_DEEPLY_NESTED));
1085
1086	PIEMNATIVECOND const pEntry = &pReNative->aCondStack[idxStack];
1087	pReNative->cCondDepth = (uint8_t)(idxStack + 1);
1088
1089	uint16_t const uCondSeqNo = ++pReNative->uCondSeqNo;
1090	pEntry->fInElse = false;
1091	pEntry->idxLabelElse = iemNativeLabelCreate(pReNative, kIemNativeLabelType_Else, UINT32_MAX /offWhere/, uCondSeqNo);
1092	pEntry->idxLabelEndIf = iemNativeLabelCreate(pReNative, kIemNativeLabelType_Endif, UINT32_MAX /offWhere/, uCondSeqNo);
1093
1094	return pEntry;
1095	}
1096
1097
1098	/**
1099	* Start of the if-block, snapshotting the register and variable state.
1100	*/
1101	DECL_INLINE_THROW(void)
1102	iemNativeCondStartIfBlock(PIEMRECOMPILERSTATE pReNative, uint32_t offIfBlock, uint32_t idxLabelIf = UINT32_MAX)
1103	{
1104	Assert(offIfBlock != UINT32_MAX);
1105	Assert(pReNative->cCondDepth > 0 && pReNative->cCondDepth <= RT_ELEMENTS(pReNative->aCondStack));
1106	PIEMNATIVECOND const pEntry = &pReNative->aCondStack[pReNative->cCondDepth - 1];
1107	Assert(!pEntry->fInElse);
1108
1109	/* Define the start of the IF block if request or for disassembly purposes. */
1110	if (idxLabelIf != UINT32_MAX)
1111	iemNativeLabelDefine(pReNative, idxLabelIf, offIfBlock);
1112	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1113	else
1114	iemNativeLabelCreate(pReNative, kIemNativeLabelType_If, offIfBlock, pReNative->paLabels[pEntry->idxLabelElse].uData);
1115	#else
1116	RT_NOREF(offIfBlock);
1117	#endif
1118
1119	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1120	Assert(pReNative->Core.offPc == 0);
1121	#endif
1122
1123	/* Copy the initial state so we can restore it in the 'else' block. */
1124	pEntry->InitialState = pReNative->Core;
1125	}
1126
1127
1128	#define IEM_MC_ELSE() } while (0); \
1129	off = iemNativeEmitElse(pReNative, off); \
1130	do {
1131
1132	/** Emits code related to IEM_MC_ELSE. */
1133	DECL_INLINE_THROW(uint32_t) iemNativeEmitElse(PIEMRECOMPILERSTATE pReNative, uint32_t off)
1134	{
1135	/* Check sanity and get the conditional stack entry. */
1136	Assert(off != UINT32_MAX);
1137	Assert(pReNative->cCondDepth > 0 && pReNative->cCondDepth <= RT_ELEMENTS(pReNative->aCondStack));
1138	PIEMNATIVECOND const pEntry = &pReNative->aCondStack[pReNative->cCondDepth - 1];
1139	Assert(!pEntry->fInElse);
1140
1141	/* Jump to the endif */
1142	off = iemNativeEmitJmpToLabel(pReNative, off, pEntry->idxLabelEndIf);
1143
1144	/* Define the else label and enter the else part of the condition. */
1145	iemNativeLabelDefine(pReNative, pEntry->idxLabelElse, off);
1146	pEntry->fInElse = true;
1147
1148	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1149	Assert(pReNative->Core.offPc == 0);
1150	#endif
1151
1152	/* Snapshot the core state so we can do a merge at the endif and restore
1153	the snapshot we took at the start of the if-block. */
1154	pEntry->IfFinalState = pReNative->Core;
1155	pReNative->Core = pEntry->InitialState;
1156
1157	return off;
1158	}
1159
1160
1161	#define IEM_MC_ENDIF() } while (0); \
1162	off = iemNativeEmitEndIf(pReNative, off)
1163
1164	/** Emits code related to IEM_MC_ENDIF. */
1165	DECL_INLINE_THROW(uint32_t) iemNativeEmitEndIf(PIEMRECOMPILERSTATE pReNative, uint32_t off)
1166	{
1167	/* Check sanity and get the conditional stack entry. */
1168	Assert(off != UINT32_MAX);
1169	Assert(pReNative->cCondDepth > 0 && pReNative->cCondDepth <= RT_ELEMENTS(pReNative->aCondStack));
1170	PIEMNATIVECOND const pEntry = &pReNative->aCondStack[pReNative->cCondDepth - 1];
1171
1172	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1173	Assert(pReNative->Core.offPc == 0);
1174	#endif
1175
1176	/*
1177	* Now we have find common group with the core state at the end of the
1178	* if-final. Use the smallest common denominator and just drop anything
1179	* that isn't the same in both states.
1180	*/
1181	/** @todo We could, maybe, shuffle registers around if we thought it helpful,
1182	* which is why we're doing this at the end of the else-block.
1183	* But we'd need more info about future for that to be worth the effort. */
1184	PCIEMNATIVECORESTATE const pOther = pEntry->fInElse ? &pEntry->IfFinalState : &pEntry->InitialState;
1185	if (memcmp(&pReNative->Core, pOther, sizeof(*pOther)) != 0)
1186	{
1187	/* shadow guest stuff first. */
1188	uint64_t fGstRegs = pReNative->Core.bmGstRegShadows;
1189	if (fGstRegs)
1190	{
1191	Assert(pReNative->Core.bmHstRegsWithGstShadow != 0);
1192	do
1193	{
1194	unsigned idxGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
1195	fGstRegs &= ~RT_BIT_64(idxGstReg);
1196
1197	uint8_t const idxHstReg = pReNative->Core.aidxGstRegShadows[idxGstReg];
1198	if ( !(pOther->bmGstRegShadows & RT_BIT_64(idxGstReg))
1199	\|\| idxHstReg != pOther->aidxGstRegShadows[idxGstReg])
1200	{
1201	Log12(("iemNativeEmitEndIf: dropping gst %s from hst %s\n",
1202	g_aGstShadowInfo[idxGstReg].pszName, g_apszIemNativeHstRegNames[idxHstReg]));
1203	iemNativeRegClearGstRegShadowing(pReNative, idxHstReg, off);
1204	}
1205	} while (fGstRegs);
1206	}
1207	else
1208	Assert(pReNative->Core.bmHstRegsWithGstShadow == 0);
1209
1210	/* Check variables next. For now we must require them to be identical
1211	or stuff we can recreate. */
1212	Assert(pReNative->Core.u64ArgVars == pOther->u64ArgVars);
1213	uint32_t fVars = pReNative->Core.bmVars \| pOther->bmVars;
1214	if (fVars)
1215	{
1216	uint32_t const fVarsMustRemove = pReNative->Core.bmVars ^ pOther->bmVars;
1217	do
1218	{
1219	unsigned idxVar = ASMBitFirstSetU32(fVars) - 1;
1220	fVars &= ~RT_BIT_32(idxVar);
1221
1222	if (!(fVarsMustRemove & RT_BIT_32(idxVar)))
1223	{
1224	if (pReNative->Core.aVars[idxVar].idxReg == pOther->aVars[idxVar].idxReg)
1225	continue;
1226	if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
1227	{
1228	uint8_t const idxHstReg = pReNative->Core.aVars[idxVar].idxReg;
1229	if (idxHstReg != UINT8_MAX)
1230	{
1231	pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
1232	pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
1233	Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u/%#x\n",
1234	g_apszIemNativeHstRegNames[idxHstReg], idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
1235	}
1236	continue;
1237	}
1238	}
1239	else if (!(pReNative->Core.bmVars & RT_BIT_32(idxVar)))
1240	continue;
1241
1242	/* Irreconcilable, so drop it. */
1243	uint8_t const idxHstReg = pReNative->Core.aVars[idxVar].idxReg;
1244	if (idxHstReg != UINT8_MAX)
1245	{
1246	pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
1247	pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
1248	Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u/%#x (also dropped)\n",
1249	g_apszIemNativeHstRegNames[idxHstReg], idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
1250	}
1251	Log11(("iemNativeEmitEndIf: Freeing variable #%u/%#x\n", idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
1252	pReNative->Core.bmVars &= ~RT_BIT_32(idxVar);
1253	} while (fVars);
1254	}
1255
1256	/* Finally, check that the host register allocations matches. */
1257	AssertMsgStmt(pReNative->Core.bmHstRegs == pOther->bmHstRegs,
1258	("Core.bmHstRegs=%#x pOther->bmHstRegs=%#x - %#x\n",
1259	pReNative->Core.bmHstRegs, pOther->bmHstRegs, pReNative->Core.bmHstRegs ^ pOther->bmHstRegs),
1260	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_COND_ENDIF_RECONCILIATION_FAILED));
1261	}
1262
1263	/*
1264	* Define the endif label and maybe the else one if we're still in the 'if' part.
1265	*/
1266	if (!pEntry->fInElse)
1267	iemNativeLabelDefine(pReNative, pEntry->idxLabelElse, off);
1268	else
1269	Assert(pReNative->paLabels[pEntry->idxLabelElse].off <= off);
1270	iemNativeLabelDefine(pReNative, pEntry->idxLabelEndIf, off);
1271
1272	/* Pop the conditional stack.*/
1273	pReNative->cCondDepth -= 1;
1274
1275	return off;
1276	}
1277
1278
1279	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) \
1280	off = iemNativeEmitIfEflagAnysBitsSet(pReNative, off, (a_fBits)); \
1281	do {
1282
1283	/** Emits code for IEM_MC_IF_EFL_ANY_BITS_SET. */
1284	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfEflagAnysBitsSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitsInEfl)
1285	{
1286	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1287
1288	/* Get the eflags. */
1289	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1290	kIemNativeGstRegUse_ReadOnly);
1291
1292	/* Test and jump. */
1293	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, fBitsInEfl, pEntry->idxLabelElse);
1294
1295	/* Free but don't flush the EFlags register. */
1296	iemNativeRegFreeTmp(pReNative, idxEflReg);
1297
1298	/* Make a copy of the core state now as we start the if-block. */
1299	iemNativeCondStartIfBlock(pReNative, off);
1300
1301	return off;
1302	}
1303
1304
1305	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) \
1306	off = iemNativeEmitIfEflagNoBitsSet(pReNative, off, (a_fBits)); \
1307	do {
1308
1309	/** Emits code for IEM_MC_IF_EFL_NO_BITS_SET. */
1310	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfEflagNoBitsSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitsInEfl)
1311	{
1312	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1313
1314	/* Get the eflags. */
1315	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1316	kIemNativeGstRegUse_ReadOnly);
1317
1318	/* Test and jump. */
1319	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxEflReg, fBitsInEfl, pEntry->idxLabelElse);
1320
1321	/* Free but don't flush the EFlags register. */
1322	iemNativeRegFreeTmp(pReNative, idxEflReg);
1323
1324	/* Make a copy of the core state now as we start the if-block. */
1325	iemNativeCondStartIfBlock(pReNative, off);
1326
1327	return off;
1328	}
1329
1330
1331	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) \
1332	off = iemNativeEmitIfEflagsBitSet(pReNative, off, (a_fBit)); \
1333	do {
1334
1335	/** Emits code for IEM_MC_IF_EFL_BIT_SET. */
1336	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfEflagsBitSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitInEfl)
1337	{
1338	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1339
1340	/* Get the eflags. */
1341	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1342	kIemNativeGstRegUse_ReadOnly);
1343
1344	unsigned const iBitNo = ASMBitFirstSetU32(fBitInEfl) - 1;
1345	Assert(RT_BIT_32(iBitNo) == fBitInEfl);
1346
1347	/* Test and jump. */
1348	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, iBitNo, pEntry->idxLabelElse);
1349
1350	/* Free but don't flush the EFlags register. */
1351	iemNativeRegFreeTmp(pReNative, idxEflReg);
1352
1353	/* Make a copy of the core state now as we start the if-block. */
1354	iemNativeCondStartIfBlock(pReNative, off);
1355
1356	return off;
1357	}
1358
1359
1360	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) \
1361	off = iemNativeEmitIfEflagsBitNotSet(pReNative, off, (a_fBit)); \
1362	do {
1363
1364	/** Emits code for IEM_MC_IF_EFL_BIT_NOT_SET. */
1365	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfEflagsBitNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitInEfl)
1366	{
1367	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1368
1369	/* Get the eflags. */
1370	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1371	kIemNativeGstRegUse_ReadOnly);
1372
1373	unsigned const iBitNo = ASMBitFirstSetU32(fBitInEfl) - 1;
1374	Assert(RT_BIT_32(iBitNo) == fBitInEfl);
1375
1376	/* Test and jump. */
1377	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxEflReg, iBitNo, pEntry->idxLabelElse);
1378
1379	/* Free but don't flush the EFlags register. */
1380	iemNativeRegFreeTmp(pReNative, idxEflReg);
1381
1382	/* Make a copy of the core state now as we start the if-block. */
1383	iemNativeCondStartIfBlock(pReNative, off);
1384
1385	return off;
1386	}
1387
1388
1389	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
1390	off = iemNativeEmitIfEflagsTwoBitsEqual(pReNative, off, a_fBit1, a_fBit2, false /fInverted/); \
1391	do {
1392
1393	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
1394	off = iemNativeEmitIfEflagsTwoBitsEqual(pReNative, off, a_fBit1, a_fBit2, true /fInverted/); \
1395	do {
1396
1397	/** Emits code for IEM_MC_IF_EFL_BITS_EQ and IEM_MC_IF_EFL_BITS_NE. */
1398	DECL_INLINE_THROW(uint32_t)
1399	iemNativeEmitIfEflagsTwoBitsEqual(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1400	uint32_t fBit1InEfl, uint32_t fBit2InEfl, bool fInverted)
1401	{
1402	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1403
1404	/* Get the eflags. */
1405	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1406	kIemNativeGstRegUse_ReadOnly);
1407
1408	unsigned const iBitNo1 = ASMBitFirstSetU32(fBit1InEfl) - 1;
1409	Assert(RT_BIT_32(iBitNo1) == fBit1InEfl);
1410
1411	unsigned const iBitNo2 = ASMBitFirstSetU32(fBit2InEfl) - 1;
1412	Assert(RT_BIT_32(iBitNo2) == fBit2InEfl);
1413	Assert(iBitNo1 != iBitNo2);
1414
1415	#ifdef RT_ARCH_AMD64
1416	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBit1InEfl);
1417
1418	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxTmpReg, idxEflReg);
1419	if (iBitNo1 > iBitNo2)
1420	off = iemNativeEmitShiftGpr32Right(pReNative, off, idxTmpReg, iBitNo1 - iBitNo2);
1421	else
1422	off = iemNativeEmitShiftGpr32Left(pReNative, off, idxTmpReg, iBitNo2 - iBitNo1);
1423	off = iemNativeEmitXorGpr32ByGpr32(pReNative, off, idxTmpReg, idxEflReg);
1424
1425	#elif defined(RT_ARCH_ARM64)
1426	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
1427	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1428
1429	/* and tmpreg, eflreg, #1<<iBitNo1 */
1430	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxTmpReg, idxEflReg, 0 /uImm7SizeLen -> 32/, 32 - iBitNo1, false /f64Bit/);
1431
1432	/* eeyore tmpreg, eflreg, tmpreg, LSL/LSR, #abs(iBitNo2 - iBitNo1) */
1433	if (iBitNo1 > iBitNo2)
1434	pu32CodeBuf[off++] = Armv8A64MkInstrEor(idxTmpReg, idxEflReg, idxTmpReg, false /64bit/,
1435	iBitNo1 - iBitNo2, kArmv8A64InstrShift_Lsr);
1436	else
1437	pu32CodeBuf[off++] = Armv8A64MkInstrEor(idxTmpReg, idxEflReg, idxTmpReg, false /64bit/,
1438	iBitNo2 - iBitNo1, kArmv8A64InstrShift_Lsl);
1439
1440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1441
1442	#else
1443	# error "Port me"
1444	#endif
1445
1446	/* Test (bit #2 is set in tmpreg if not-equal) and jump. */
1447	off = iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, idxTmpReg, iBitNo2,
1448	pEntry->idxLabelElse, !fInverted /fJmpIfSet/);
1449
1450	/* Free but don't flush the EFlags and tmp registers. */
1451	iemNativeRegFreeTmp(pReNative, idxTmpReg);
1452	iemNativeRegFreeTmp(pReNative, idxEflReg);
1453
1454	/* Make a copy of the core state now as we start the if-block. */
1455	iemNativeCondStartIfBlock(pReNative, off);
1456
1457	return off;
1458	}
1459
1460
1461	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
1462	off = iemNativeEmitIfEflagsBitNotSetAndTwoBitsEqual(pReNative, off, a_fBit, a_fBit1, a_fBit2, false /fInverted/); \
1463	do {
1464
1465	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
1466	off = iemNativeEmitIfEflagsBitNotSetAndTwoBitsEqual(pReNative, off, a_fBit, a_fBit1, a_fBit2, true /fInverted/); \
1467	do {
1468
1469	/** Emits code for IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ and
1470	* IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE. */
1471	DECL_INLINE_THROW(uint32_t)
1472	iemNativeEmitIfEflagsBitNotSetAndTwoBitsEqual(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitInEfl,
1473	uint32_t fBit1InEfl, uint32_t fBit2InEfl, bool fInverted)
1474	{
1475	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1476
1477	/* We need an if-block label for the non-inverted variant. */
1478	uint32_t const idxLabelIf = fInverted ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_If, UINT32_MAX,
1479	pReNative->paLabels[pEntry->idxLabelElse].uData) : UINT32_MAX;
1480
1481	/* Get the eflags. */
1482	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1483	kIemNativeGstRegUse_ReadOnly);
1484
1485	/* Translate the flag masks to bit numbers. */
1486	unsigned const iBitNo = ASMBitFirstSetU32(fBitInEfl) - 1;
1487	Assert(RT_BIT_32(iBitNo) == fBitInEfl);
1488
1489	unsigned const iBitNo1 = ASMBitFirstSetU32(fBit1InEfl) - 1;
1490	Assert(RT_BIT_32(iBitNo1) == fBit1InEfl);
1491	Assert(iBitNo1 != iBitNo);
1492
1493	unsigned const iBitNo2 = ASMBitFirstSetU32(fBit2InEfl) - 1;
1494	Assert(RT_BIT_32(iBitNo2) == fBit2InEfl);
1495	Assert(iBitNo2 != iBitNo);
1496	Assert(iBitNo2 != iBitNo1);
1497
1498	#ifdef RT_ARCH_AMD64
1499	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBit1InEfl); /* This must come before we jump anywhere! */
1500	#elif defined(RT_ARCH_ARM64)
1501	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
1502	#endif
1503
1504	/* Check for the lone bit first. */
1505	if (!fInverted)
1506	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxEflReg, iBitNo, pEntry->idxLabelElse);
1507	else
1508	off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxEflReg, iBitNo, idxLabelIf);
1509
1510	/* Then extract and compare the other two bits. */
1511	#ifdef RT_ARCH_AMD64
1512	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxTmpReg, idxEflReg);
1513	if (iBitNo1 > iBitNo2)
1514	off = iemNativeEmitShiftGpr32Right(pReNative, off, idxTmpReg, iBitNo1 - iBitNo2);
1515	else
1516	off = iemNativeEmitShiftGpr32Left(pReNative, off, idxTmpReg, iBitNo2 - iBitNo1);
1517	off = iemNativeEmitXorGpr32ByGpr32(pReNative, off, idxTmpReg, idxEflReg);
1518
1519	#elif defined(RT_ARCH_ARM64)
1520	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1521
1522	/* and tmpreg, eflreg, #1<<iBitNo1 */
1523	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxTmpReg, idxEflReg, 0 /uImm7SizeLen -> 32/, 32 - iBitNo1, false /f64Bit/);
1524
1525	/* eeyore tmpreg, eflreg, tmpreg, LSL/LSR, #abs(iBitNo2 - iBitNo1) */
1526	if (iBitNo1 > iBitNo2)
1527	pu32CodeBuf[off++] = Armv8A64MkInstrEor(idxTmpReg, idxEflReg, idxTmpReg, false /64bit/,
1528	iBitNo1 - iBitNo2, kArmv8A64InstrShift_Lsr);
1529	else
1530	pu32CodeBuf[off++] = Armv8A64MkInstrEor(idxTmpReg, idxEflReg, idxTmpReg, false /64bit/,
1531	iBitNo2 - iBitNo1, kArmv8A64InstrShift_Lsl);
1532
1533	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1534
1535	#else
1536	# error "Port me"
1537	#endif
1538
1539	/* Test (bit #2 is set in tmpreg if not-equal) and jump. */
1540	off = iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, idxTmpReg, iBitNo2,
1541	pEntry->idxLabelElse, !fInverted /fJmpIfSet/);
1542
1543	/* Free but don't flush the EFlags and tmp registers. */
1544	iemNativeRegFreeTmp(pReNative, idxTmpReg);
1545	iemNativeRegFreeTmp(pReNative, idxEflReg);
1546
1547	/* Make a copy of the core state now as we start the if-block. */
1548	iemNativeCondStartIfBlock(pReNative, off, idxLabelIf);
1549
1550	return off;
1551	}
1552
1553
1554	#define IEM_MC_IF_CX_IS_NZ() \
1555	off = iemNativeEmitIfCxIsNotZero(pReNative, off); \
1556	do {
1557
1558	/** Emits code for IEM_MC_IF_CX_IS_NZ. */
1559	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfCxIsNotZero(PIEMRECOMPILERSTATE pReNative, uint32_t off)
1560	{
1561	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1562
1563	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1564	kIemNativeGstRegUse_ReadOnly);
1565	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxGstRcxReg, UINT16_MAX, pEntry->idxLabelElse);
1566	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1567
1568	iemNativeCondStartIfBlock(pReNative, off);
1569	return off;
1570	}
1571
1572
1573	#define IEM_MC_IF_ECX_IS_NZ() \
1574	off = iemNativeEmitIfRcxEcxIsNotZero(pReNative, off, false /f64Bit/); \
1575	do {
1576
1577	#define IEM_MC_IF_RCX_IS_NZ() \
1578	off = iemNativeEmitIfRcxEcxIsNotZero(pReNative, off, true /f64Bit/); \
1579	do {
1580
1581	/** Emits code for IEM_MC_IF_ECX_IS_NZ and IEM_MC_IF_RCX_IS_NZ. */
1582	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfRcxEcxIsNotZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, bool f64Bit)
1583	{
1584	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1585
1586	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1587	kIemNativeGstRegUse_ReadOnly);
1588	off = iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, idxGstRcxReg, f64Bit, pEntry->idxLabelElse);
1589	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1590
1591	iemNativeCondStartIfBlock(pReNative, off);
1592	return off;
1593	}
1594
1595
1596	#define IEM_MC_IF_CX_IS_NOT_ONE() \
1597	off = iemNativeEmitIfCxIsNotOne(pReNative, off); \
1598	do {
1599
1600	/** Emits code for IEM_MC_IF_CX_IS_NOT_ONE. */
1601	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfCxIsNotOne(PIEMRECOMPILERSTATE pReNative, uint32_t off)
1602	{
1603	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1604
1605	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1606	kIemNativeGstRegUse_ReadOnly);
1607	#ifdef RT_ARCH_AMD64
1608	off = iemNativeEmitTestIfGpr16EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1609	#else
1610	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
1611	off = iemNativeEmitTestIfGpr16EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse, idxTmpReg);
1612	iemNativeRegFreeTmp(pReNative, idxTmpReg);
1613	#endif
1614	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1615
1616	iemNativeCondStartIfBlock(pReNative, off);
1617	return off;
1618	}
1619
1620
1621	#define IEM_MC_IF_ECX_IS_NOT_ONE() \
1622	off = iemNativeEmitIfRcxEcxIsNotOne(pReNative, off, false /f64Bit/); \
1623	do {
1624
1625	#define IEM_MC_IF_RCX_IS_NOT_ONE() \
1626	off = iemNativeEmitIfRcxEcxIsNotOne(pReNative, off, true /f64Bit/); \
1627	do {
1628
1629	/** Emits code for IEM_MC_IF_ECX_IS_NOT_ONE and IEM_MC_IF_RCX_IS_NOT_ONE. */
1630	DECL_INLINE_THROW(uint32_t) iemNativeEmitIfRcxEcxIsNotOne(PIEMRECOMPILERSTATE pReNative, uint32_t off, bool f64Bit)
1631	{
1632	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1633
1634	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1635	kIemNativeGstRegUse_ReadOnly);
1636	if (f64Bit)
1637	off = iemNativeEmitTestIfGprEqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1638	else
1639	off = iemNativeEmitTestIfGpr32EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1640	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1641
1642	iemNativeCondStartIfBlock(pReNative, off);
1643	return off;
1644	}
1645
1646
1647	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
1648	off = iemNativeEmitIfCxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, true /fCheckIfSet/); \
1649	do {
1650
1651	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
1652	off = iemNativeEmitIfCxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, false /fCheckIfSet/); \
1653	do {
1654
1655	/** Emits code for IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET and
1656	* IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET. */
1657	DECL_INLINE_THROW(uint32_t)
1658	iemNativeEmitIfCxIsNotOneAndTestEflagsBit(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitInEfl, bool fCheckIfSet)
1659	{
1660	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1661
1662	/* We have to load both RCX and EFLAGS before we can start branching,
1663	otherwise we'll end up in the else-block with an inconsistent
1664	register allocator state.
1665	Doing EFLAGS first as it's more likely to be loaded, right? */
1666	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1667	kIemNativeGstRegUse_ReadOnly);
1668	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1669	kIemNativeGstRegUse_ReadOnly);
1670
1671	/** @todo we could reduce this to a single branch instruction by spending a
1672	* temporary register and some setnz stuff. Not sure if loops are
1673	* worth it. */
1674	/* Check CX. */
1675	#ifdef RT_ARCH_AMD64
1676	off = iemNativeEmitTestIfGpr16EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1677	#else
1678	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
1679	off = iemNativeEmitTestIfGpr16EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse, idxTmpReg);
1680	iemNativeRegFreeTmp(pReNative, idxTmpReg);
1681	#endif
1682
1683	/* Check the EFlags bit. */
1684	unsigned const iBitNo = ASMBitFirstSetU32(fBitInEfl) - 1;
1685	Assert(RT_BIT_32(iBitNo) == fBitInEfl);
1686	off = iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, idxEflReg, iBitNo, pEntry->idxLabelElse,
1687	!fCheckIfSet /fJmpIfSet/);
1688
1689	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1690	iemNativeRegFreeTmp(pReNative, idxEflReg);
1691
1692	iemNativeCondStartIfBlock(pReNative, off);
1693	return off;
1694	}
1695
1696
1697	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
1698	off = iemNativeEmitIfRcxEcxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, true /fCheckIfSet/, false /f64Bit/); \
1699	do {
1700
1701	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
1702	off = iemNativeEmitIfRcxEcxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, false /fCheckIfSet/, false /f64Bit/); \
1703	do {
1704
1705	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
1706	off = iemNativeEmitIfRcxEcxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, true /fCheckIfSet/, true /f64Bit/); \
1707	do {
1708
1709	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
1710	off = iemNativeEmitIfRcxEcxIsNotOneAndTestEflagsBit(pReNative, off, a_fBit, false /fCheckIfSet/, true /f64Bit/); \
1711	do {
1712
1713	/** Emits code for IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET,
1714	* IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET,
1715	* IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET and
1716	* IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET. */
1717	DECL_INLINE_THROW(uint32_t)
1718	iemNativeEmitIfRcxEcxIsNotOneAndTestEflagsBit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1719	uint32_t fBitInEfl, bool fCheckIfSet, bool f64Bit)
1720	{
1721	PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative, &off);
1722
1723	/* We have to load both RCX and EFLAGS before we can start branching,
1724	otherwise we'll end up in the else-block with an inconsistent
1725	register allocator state.
1726	Doing EFLAGS first as it's more likely to be loaded, right? */
1727	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
1728	kIemNativeGstRegUse_ReadOnly);
1729	uint8_t const idxGstRcxReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xCX),
1730	kIemNativeGstRegUse_ReadOnly);
1731
1732	/** @todo we could reduce this to a single branch instruction by spending a
1733	* temporary register and some setnz stuff. Not sure if loops are
1734	* worth it. */
1735	/* Check RCX/ECX. */
1736	if (f64Bit)
1737	off = iemNativeEmitTestIfGprEqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1738	else
1739	off = iemNativeEmitTestIfGpr32EqualsImmAndJmpToLabel(pReNative, off, idxGstRcxReg, 1, pEntry->idxLabelElse);
1740
1741	/* Check the EFlags bit. */
1742	unsigned const iBitNo = ASMBitFirstSetU32(fBitInEfl) - 1;
1743	Assert(RT_BIT_32(iBitNo) == fBitInEfl);
1744	off = iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, idxEflReg, iBitNo, pEntry->idxLabelElse,
1745	!fCheckIfSet /fJmpIfSet/);
1746
1747	iemNativeRegFreeTmp(pReNative, idxGstRcxReg);
1748	iemNativeRegFreeTmp(pReNative, idxEflReg);
1749
1750	iemNativeCondStartIfBlock(pReNative, off);
1751	return off;
1752	}
1753
1754
1755
1756	/*********************************************************************************************************************************
1757	* Emitters for IEM_MC_ARG_XXX, IEM_MC_LOCAL, IEM_MC_LOCAL_CONST, ++ *
1758	*********************************************************************************************************************************/
1759
1760	#define IEM_MC_NOREF(a_Name) \
1761	RT_NOREF_PV(a_Name)
1762
1763	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) \
1764	uint8_t const a_Name = iemNativeArgAlloc(pReNative, (a_iArg), sizeof(a_Type))
1765
1766	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) \
1767	uint8_t const a_Name = iemNativeArgAllocConst(pReNative, (a_iArg), sizeof(a_Type), (a_Value))
1768
1769	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) \
1770	uint8_t const a_Name = iemNativeArgAllocLocalRef(pReNative, (a_iArg), (a_Local))
1771
1772	#define IEM_MC_LOCAL(a_Type, a_Name) \
1773	uint8_t const a_Name = iemNativeVarAlloc(pReNative, sizeof(a_Type))
1774
1775	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) \
1776	uint8_t const a_Name = iemNativeVarAllocConst(pReNative, sizeof(a_Type), (a_Value))
1777
1778
1779	/**
1780	* Sets the host register for @a idxVarRc to @a idxReg.
1781	*
1782	* The register must not be allocated. Any guest register shadowing will be
1783	* implictly dropped by this call.
1784	*
1785	* The variable must not have any register associated with it (causes
1786	* VERR_IEM_VAR_IPE_10 to be raised). Conversion to a stack variable is
1787	* implied.
1788	*
1789	* @returns idxReg
1790	* @param pReNative The recompiler state.
1791	* @param idxVar The variable.
1792	* @param idxReg The host register (typically IEMNATIVE_CALL_RET_GREG).
1793	* @param off For recording in debug info.
1794	*
1795	* @throws VERR_IEM_VAR_IPE_10, VERR_IEM_VAR_IPE_11
1796	*/
1797	DECL_INLINE_THROW(uint8_t) iemNativeVarRegisterSet(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint8_t idxReg, uint32_t off)
1798	{
1799	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1800	PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
1801	Assert(!pVar->fRegAcquired);
1802	Assert(idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs));
1803	AssertStmt(pVar->idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_10));
1804	AssertStmt(!(pReNative->Core.bmHstRegs & RT_BIT_32(idxReg)), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_11));
1805
1806	iemNativeRegClearGstRegShadowing(pReNative, idxReg, off);
1807	iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
1808
1809	iemNativeVarSetKindToStack(pReNative, idxVar);
1810	pVar->idxReg = idxReg;
1811
1812	return idxReg;
1813	}
1814
1815
1816	/**
1817	* A convenient helper function.
1818	*/
1819	DECL_INLINE_THROW(uint8_t) iemNativeVarRegisterSetAndAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1820	uint8_t idxReg, uint32_t *poff)
1821	{
1822	idxReg = iemNativeVarRegisterSet(pReNative, idxVar, idxReg, *poff);
1823	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = true;
1824	return idxReg;
1825	}
1826
1827
1828	/**
1829	* This is called by IEM_MC_END() to clean up all variables.
1830	*/
1831	DECL_FORCE_INLINE(void) iemNativeVarFreeAll(PIEMRECOMPILERSTATE pReNative)
1832	{
1833	uint32_t const bmVars = pReNative->Core.bmVars;
1834	if (bmVars != 0)
1835	iemNativeVarFreeAllSlow(pReNative, bmVars);
1836	Assert(pReNative->Core.u64ArgVars == UINT64_MAX);
1837	Assert(pReNative->Core.bmStack == 0);
1838	}
1839
1840
1841	#define IEM_MC_FREE_LOCAL(a_Name) iemNativeVarFreeLocal(pReNative, a_Name)
1842
1843	/**
1844	* This is called by IEM_MC_FREE_LOCAL.
1845	*/
1846	DECLINLINE(void) iemNativeVarFreeLocal(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1847	{
1848	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1849	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].uArgNo == UINT8_MAX);
1850	iemNativeVarFreeOneWorker(pReNative, IEMNATIVE_VAR_IDX_UNPACK(idxVar));
1851	}
1852
1853
1854	#define IEM_MC_FREE_ARG(a_Name) iemNativeVarFreeArg(pReNative, a_Name)
1855
1856	/**
1857	* This is called by IEM_MC_FREE_ARG.
1858	*/
1859	DECLINLINE(void) iemNativeVarFreeArg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1860	{
1861	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1862	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].uArgNo < RT_ELEMENTS(pReNative->Core.aidxArgVars));
1863	iemNativeVarFreeOneWorker(pReNative, IEMNATIVE_VAR_IDX_UNPACK(idxVar));
1864	}
1865
1866
1867	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) off = iemNativeVarAssignToSmaller(pReNative, off, a_VarDst, a_VarSrcEol)
1868
1869	/**
1870	* This is called by IEM_MC_ASSIGN_TO_SMALLER.
1871	*/
1872	DECL_INLINE_THROW(uint32_t)
1873	iemNativeVarAssignToSmaller(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarDst, uint8_t idxVarSrc)
1874	{
1875	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarDst);
1876	PIEMNATIVEVAR const pVarDst = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarDst)];
1877	AssertStmt(pVarDst->enmKind == kIemNativeVarKind_Invalid, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
1878	Assert( pVarDst->cbVar == sizeof(uint16_t)
1879	\|\| pVarDst->cbVar == sizeof(uint32_t));
1880
1881	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarSrc);
1882	PIEMNATIVEVAR const pVarSrc = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarSrc)];
1883	AssertStmt( pVarSrc->enmKind == kIemNativeVarKind_Stack
1884	\|\| pVarSrc->enmKind == kIemNativeVarKind_Immediate,
1885	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
1886
1887	Assert(pVarDst->cbVar < pVarSrc->cbVar);
1888
1889	/*
1890	* Special case for immediates.
1891	*/
1892	if (pVarSrc->enmKind == kIemNativeVarKind_Immediate)
1893	{
1894	switch (pVarDst->cbVar)
1895	{
1896	case sizeof(uint16_t):
1897	iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint16_t)pVarSrc->u.uValue);
1898	break;
1899	case sizeof(uint32_t):
1900	iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint32_t)pVarSrc->u.uValue);
1901	break;
1902	default: AssertFailed(); break;
1903	}
1904	}
1905	else
1906	{
1907	/*
1908	* The generic solution for now.
1909	*/
1910	/** @todo optimize this by having the python script make sure the source
1911	* variable passed to IEM_MC_ASSIGN_TO_SMALLER is not used after the
1912	* statement. Then we could just transfer the register assignments. */
1913	uint8_t const idxRegDst = iemNativeVarRegisterAcquire(pReNative, idxVarDst, &off);
1914	uint8_t const idxRegSrc = iemNativeVarRegisterAcquire(pReNative, idxVarSrc, &off);
1915	switch (pVarDst->cbVar)
1916	{
1917	case sizeof(uint16_t):
1918	off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxRegDst, idxRegSrc);
1919	break;
1920	case sizeof(uint32_t):
1921	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegDst, idxRegSrc);
1922	break;
1923	default: AssertFailed(); break;
1924	}
1925	iemNativeVarRegisterRelease(pReNative, idxVarSrc);
1926	iemNativeVarRegisterRelease(pReNative, idxVarDst);
1927	}
1928	return off;
1929	}
1930
1931
1932
1933	/*********************************************************************************************************************************
1934	* Emitters for IEM_MC_CALL_CIMPL_XXX *
1935	*********************************************************************************************************************************/
1936
1937	/** Common emit function for IEM_MC_CALL_CIMPL_XXXX. */
1938	DECL_INLINE_THROW(uint32_t)
1939	iemNativeEmitCallCImplCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
1940	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cArgs)
1941
1942	{
1943	/*
1944	* Do all the call setup and cleanup.
1945	*/
1946	off = iemNativeEmitCallCommon(pReNative, off, cArgs + IEM_CIMPL_HIDDEN_ARGS, IEM_CIMPL_HIDDEN_ARGS);
1947
1948	/*
1949	* Load the two or three hidden arguments.
1950	*/
1951	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1952	off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict */
1953	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1954	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, cbInstr);
1955	#else
1956	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1957	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, cbInstr);
1958	#endif
1959
1960	/*
1961	* Make the call and check the return code.
1962	*
1963	* Shadow PC copies are always flushed here, other stuff depends on flags.
1964	* Segment and general purpose registers are explictily flushed via the
1965	* IEM_MC_HINT_FLUSH_GUEST_SHADOW_GREG and IEM_MC_HINT_FLUSH_GUEST_SHADOW_SREG
1966	* macros.
1967	*/
1968	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)pfnCImpl);
1969	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1970	off = iemNativeEmitLoadGprByBpU32(pReNative, off, X86_GREG_xAX, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict (see above) */
1971	#endif
1972	fGstShwFlush = iemNativeCImplFlagsToGuestShadowFlushMask(pReNative->fCImpl, fGstShwFlush \| RT_BIT_64(kIemNativeGstReg_Pc));
1973	if (!(pReNative->fMc & IEM_MC_F_WITHOUT_FLAGS)) /** @todo We don't emit with-flags/without-flags variations for CIMPL calls. */
1974	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1975	iemNativeRegFlushGuestShadows(pReNative, fGstShwFlush);
1976
1977	return iemNativeEmitCheckCallRetAndPassUp(pReNative, off, idxInstr);
1978	}
1979
1980
1981	#define IEM_MC_CALL_CIMPL_1_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
1982	off = iemNativeEmitCallCImpl1(pReNative, off, a_cbInstr, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, a0)
1983
1984	/** Emits code for IEM_MC_CALL_CIMPL_1. */
1985	DECL_INLINE_THROW(uint32_t)
1986	iemNativeEmitCallCImpl1(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr, uint64_t fGstShwFlush,
1987	uintptr_t pfnCImpl, uint8_t idxArg0)
1988	{
1989	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_CIMPL_HIDDEN_ARGS);
1990	return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, fGstShwFlush, pfnCImpl, 1);
1991	}
1992
1993
1994	#define IEM_MC_CALL_CIMPL_2_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
1995	off = iemNativeEmitCallCImpl2(pReNative, off, a_cbInstr, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, a0, a1)
1996
1997	/** Emits code for IEM_MC_CALL_CIMPL_2. */
1998	DECL_INLINE_THROW(uint32_t)
1999	iemNativeEmitCallCImpl2(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr, uint64_t fGstShwFlush,
2000	uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1)
2001	{
2002	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_CIMPL_HIDDEN_ARGS);
2003	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1 + IEM_CIMPL_HIDDEN_ARGS);
2004	return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, fGstShwFlush, pfnCImpl, 2);
2005	}
2006
2007
2008	#define IEM_MC_CALL_CIMPL_3_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2009	off = iemNativeEmitCallCImpl3(pReNative, off, a_cbInstr, pCallEntry->idxInstr, a_fGstShwFlush, \
2010	(uintptr_t)a_pfnCImpl, a0, a1, a2)
2011
2012	/** Emits code for IEM_MC_CALL_CIMPL_3. */
2013	DECL_INLINE_THROW(uint32_t)
2014	iemNativeEmitCallCImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr, uint64_t fGstShwFlush,
2015	uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
2016	{
2017	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_CIMPL_HIDDEN_ARGS);
2018	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1 + IEM_CIMPL_HIDDEN_ARGS);
2019	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg2, 2 + IEM_CIMPL_HIDDEN_ARGS);
2020	return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, fGstShwFlush, pfnCImpl, 3);
2021	}
2022
2023
2024	#define IEM_MC_CALL_CIMPL_4_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2025	off = iemNativeEmitCallCImpl4(pReNative, off, a_cbInstr, pCallEntry->idxInstr, a_fGstShwFlush, \
2026	(uintptr_t)a_pfnCImpl, a0, a1, a2, a3)
2027
2028	/** Emits code for IEM_MC_CALL_CIMPL_4. */
2029	DECL_INLINE_THROW(uint32_t)
2030	iemNativeEmitCallCImpl4(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr, uint64_t fGstShwFlush,
2031	uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3)
2032	{
2033	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_CIMPL_HIDDEN_ARGS);
2034	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1 + IEM_CIMPL_HIDDEN_ARGS);
2035	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg2, 2 + IEM_CIMPL_HIDDEN_ARGS);
2036	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg3, 3 + IEM_CIMPL_HIDDEN_ARGS);
2037	return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, fGstShwFlush, pfnCImpl, 4);
2038	}
2039
2040
2041	#define IEM_MC_CALL_CIMPL_5_THREADED(a_cbInstr, a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2042	off = iemNativeEmitCallCImpl5(pReNative, off, a_cbInstr, pCallEntry->idxInstr, a_fGstShwFlush, \
2043	(uintptr_t)a_pfnCImpl, a0, a1, a2, a3, a4)
2044
2045	/** Emits code for IEM_MC_CALL_CIMPL_4. */
2046	DECL_INLINE_THROW(uint32_t)
2047	iemNativeEmitCallCImpl5(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr, uint64_t fGstShwFlush,
2048	uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3, uint8_t idxArg4)
2049	{
2050	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_CIMPL_HIDDEN_ARGS);
2051	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1 + IEM_CIMPL_HIDDEN_ARGS);
2052	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg2, 2 + IEM_CIMPL_HIDDEN_ARGS);
2053	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg3, 3 + IEM_CIMPL_HIDDEN_ARGS);
2054	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg4, 4 + IEM_CIMPL_HIDDEN_ARGS);
2055	return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, fGstShwFlush, pfnCImpl, 5);
2056	}
2057
2058
2059	/** Recompiler debugging: Flush guest register shadow copies. */
2060	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) iemNativeRegFlushGuestShadows(pReNative, g_fGstShwFlush)
2061
2062
2063
2064	/*********************************************************************************************************************************
2065	* Emitters for IEM_MC_CALL_VOID_AIMPL_XXX and IEM_MC_CALL_AIMPL_XXX *
2066	*********************************************************************************************************************************/
2067
2068	/**
2069	* Common worker for IEM_MC_CALL_VOID_AIMPL_XXX and IEM_MC_CALL_AIMPL_XXX.
2070	*/
2071	DECL_INLINE_THROW(uint32_t)
2072	iemNativeEmitCallAImplCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc,
2073	uintptr_t pfnAImpl, uint8_t cArgs)
2074	{
2075	if (idxVarRc != UINT8_MAX)
2076	{
2077	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRc);
2078	PIEMNATIVEVAR const pVarRc = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarRc)];
2079	AssertStmt(pVarRc->uArgNo == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_8));
2080	AssertStmt(pVarRc->cbVar <= sizeof(uint64_t), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_9));
2081	}
2082
2083	/*
2084	* Do all the call setup and cleanup.
2085	*/
2086	off = iemNativeEmitCallCommon(pReNative, off, cArgs, 0 /cHiddenArgs/);
2087
2088	/*
2089	* Make the call and update the return code variable if we've got one.
2090	*/
2091	off = iemNativeEmitCallImm(pReNative, off, pfnAImpl);
2092	if (idxVarRc != UINT8_MAX)
2093	{
2094	off = iemNativeEmitBrk(pReNative, off, 0x4222); /** @todo test IEM_MC_CALL_AIMPL_3 and IEM_MC_CALL_AIMPL_4 return codes. */
2095	iemNativeVarRegisterSet(pReNative, idxVarRc, IEMNATIVE_CALL_RET_GREG, off);
2096	}
2097
2098	return off;
2099	}
2100
2101
2102
2103	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) \
2104	off = iemNativeEmitCallAImpl0(pReNative, off, UINT8_MAX /idxVarRc/, (uintptr_t)(a_pfn))
2105
2106	#define IEM_MC_CALL_AIMPL_0(a_rc, a_pfn) \
2107	off = iemNativeEmitCallAImpl0(pReNative, off, a_rc, (uintptr_t)(a_pfn))
2108
2109	/** Emits code for IEM_MC_CALL_VOID_AIMPL_0 and IEM_MC_CALL_AIMPL_0. */
2110	DECL_INLINE_THROW(uint32_t)
2111	iemNativeEmitCallAImpl0(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc, uintptr_t pfnAImpl)
2112	{
2113	return iemNativeEmitCallAImplCommon(pReNative, off, idxVarRc, pfnAImpl, 0);
2114	}
2115
2116
2117	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) \
2118	off = iemNativeEmitCallAImpl1(pReNative, off, UINT8_MAX /idxVarRc/, (uintptr_t)(a_pfn), a0)
2119
2120	#define IEM_MC_CALL_AIMPL_1(a_rc, a_pfn, a0) \
2121	off = iemNativeEmitCallAImpl1(pReNative, off, a_rc, (uintptr_t)(a_pfn), a0)
2122
2123	/** Emits code for IEM_MC_CALL_VOID_AIMPL_1 and IEM_MC_CALL_AIMPL_1. */
2124	DECL_INLINE_THROW(uint32_t)
2125	iemNativeEmitCallAImpl1(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc, uintptr_t pfnAImpl, uint8_t idxArg0)
2126	{
2127	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0);
2128	return iemNativeEmitCallAImplCommon(pReNative, off, idxVarRc, pfnAImpl, 1);
2129	}
2130
2131
2132	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) \
2133	off = iemNativeEmitCallAImpl2(pReNative, off, UINT8_MAX /idxVarRc/, (uintptr_t)(a_pfn), a0, a1)
2134
2135	#define IEM_MC_CALL_AIMPL_2(a_rc, a_pfn, a0, a1) \
2136	off = iemNativeEmitCallAImpl2(pReNative, off, a_rc, (uintptr_t)(a_pfn), a0, a1)
2137
2138	/** Emits code for IEM_MC_CALL_VOID_AIMPL_2 and IEM_MC_CALL_AIMPL_2. */
2139	DECL_INLINE_THROW(uint32_t)
2140	iemNativeEmitCallAImpl2(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc,
2141	uintptr_t pfnAImpl, uint8_t idxArg0, uint8_t idxArg1)
2142	{
2143	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0);
2144	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1);
2145	return iemNativeEmitCallAImplCommon(pReNative, off, idxVarRc, pfnAImpl, 2);
2146	}
2147
2148
2149	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) \
2150	off = iemNativeEmitCallAImpl3(pReNative, off, UINT8_MAX /idxVarRc/, (uintptr_t)(a_pfn), a0, a1, a2)
2151
2152	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) \
2153	off = iemNativeEmitCallAImpl3(pReNative, off, a_rc, (uintptr_t)(a_pfn), a0, a1, a2)
2154
2155	/** Emits code for IEM_MC_CALL_VOID_AIMPL_3 and IEM_MC_CALL_AIMPL_3. */
2156	DECL_INLINE_THROW(uint32_t)
2157	iemNativeEmitCallAImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc,
2158	uintptr_t pfnAImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
2159	{
2160	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0);
2161	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1);
2162	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg2, 2);
2163	return iemNativeEmitCallAImplCommon(pReNative, off, idxVarRc, pfnAImpl, 3);
2164	}
2165
2166
2167	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) \
2168	off = iemNativeEmitCallAImpl4(pReNative, off, UINT8_MAX /idxVarRc/, (uintptr_t)(a_pfn), a0, a1, a2, a3)
2169
2170	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) \
2171	off = iemNativeEmitCallAImpl4(pReNative, off, a_rc, (uintptr_t)(a_pfn), a0, a1, a2, a3)
2172
2173	/** Emits code for IEM_MC_CALL_VOID_AIMPL_4 and IEM_MC_CALL_AIMPL_4. */
2174	DECL_INLINE_THROW(uint32_t)
2175	iemNativeEmitCallAImpl4(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRc,
2176	uintptr_t pfnAImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3)
2177	{
2178	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0);
2179	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg1, 1);
2180	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg2, 2);
2181	IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg3, 3);
2182	return iemNativeEmitCallAImplCommon(pReNative, off, idxVarRc, pfnAImpl, 4);
2183	}
2184
2185
2186
2187	/*********************************************************************************************************************************
2188	* Emitters for general purpose register fetches (IEM_MC_FETCH_GREG_XXX). *
2189	*********************************************************************************************************************************/
2190
2191	#define IEM_MC_FETCH_GREG_U8_THREADED(a_u8Dst, a_iGRegEx) \
2192	off = iemNativeEmitFetchGregU8(pReNative, off, a_u8Dst, a_iGRegEx, sizeof(uint8_t) /cbZeroExtended/)
2193
2194	#define IEM_MC_FETCH_GREG_U8_ZX_U16_THREADED(a_u16Dst, a_iGRegEx) \
2195	off = iemNativeEmitFetchGregU8(pReNative, off, a_u16Dst, a_iGRegEx, sizeof(uint16_t) /cbZeroExtended/)
2196
2197	#define IEM_MC_FETCH_GREG_U8_ZX_U32_THREADED(a_u32Dst, a_iGRegEx) \
2198	off = iemNativeEmitFetchGregU8(pReNative, off, a_u32Dst, a_iGRegEx, sizeof(uint32_t) /cbZeroExtended/)
2199
2200	#define IEM_MC_FETCH_GREG_U8_ZX_U64_THREADED(a_u64Dst, a_iGRegEx) \
2201	off = iemNativeEmitFetchGregU8(pReNative, off, a_u64Dst, a_iGRegEx, sizeof(uint64_t) /cbZeroExtended/)
2202
2203
2204	/** Emits code for IEM_MC_FETCH_GREG_U8_THREADED and
2205	* IEM_MC_FETCH_GREG_U8_ZX_U16/32/64_THREADED. */
2206	DECL_INLINE_THROW(uint32_t)
2207	iemNativeEmitFetchGregU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGRegEx, int8_t cbZeroExtended)
2208	{
2209	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2210	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
2211	Assert(iGRegEx < 20);
2212
2213	/* Same discussion as in iemNativeEmitFetchGregU16 */
2214	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGRegEx & 15),
2215	kIemNativeGstRegUse_ReadOnly);
2216
2217	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2218	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2219
2220	/* The value is zero-extended to the full 64-bit host register width. */
2221	if (iGRegEx < 16)
2222	off = iemNativeEmitLoadGprFromGpr8(pReNative, off, idxVarReg, idxGstFullReg);
2223	else
2224	off = iemNativeEmitLoadGprFromGpr8Hi(pReNative, off, idxVarReg, idxGstFullReg);
2225
2226	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2227	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2228	return off;
2229	}
2230
2231
2232	#define IEM_MC_FETCH_GREG_U8_SX_U16_THREADED(a_u16Dst, a_iGRegEx) \
2233	off = iemNativeEmitFetchGregU8Sx(pReNative, off, a_u16Dst, a_iGRegEx, sizeof(uint16_t))
2234
2235	#define IEM_MC_FETCH_GREG_U8_SX_U32_THREADED(a_u32Dst, a_iGRegEx) \
2236	off = iemNativeEmitFetchGregU8Sx(pReNative, off, a_u32Dst, a_iGRegEx, sizeof(uint32_t))
2237
2238	#define IEM_MC_FETCH_GREG_U8_SX_U64_THREADED(a_u64Dst, a_iGRegEx) \
2239	off = iemNativeEmitFetchGregU8Sx(pReNative, off, a_u64Dst, a_iGRegEx, sizeof(uint64_t))
2240
2241	/** Emits code for IEM_MC_FETCH_GREG_U8_SX_U16/32/64_THREADED. */
2242	DECL_INLINE_THROW(uint32_t)
2243	iemNativeEmitFetchGregU8Sx(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGRegEx, uint8_t cbSignExtended)
2244	{
2245	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2246	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbSignExtended);
2247	Assert(iGRegEx < 20);
2248
2249	/* Same discussion as in iemNativeEmitFetchGregU16 */
2250	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGRegEx & 15),
2251	kIemNativeGstRegUse_ReadOnly);
2252
2253	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2254	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2255
2256	if (iGRegEx < 16)
2257	{
2258	switch (cbSignExtended)
2259	{
2260	case sizeof(uint16_t):
2261	off = iemNativeEmitLoadGpr16SignExtendedFromGpr8(pReNative, off, idxVarReg, idxGstFullReg);
2262	break;
2263	case sizeof(uint32_t):
2264	off = iemNativeEmitLoadGpr32SignExtendedFromGpr8(pReNative, off, idxVarReg, idxGstFullReg);
2265	break;
2266	case sizeof(uint64_t):
2267	off = iemNativeEmitLoadGprSignExtendedFromGpr8(pReNative, off, idxVarReg, idxGstFullReg);
2268	break;
2269	default: AssertFailed(); break;
2270	}
2271	}
2272	else
2273	{
2274	off = iemNativeEmitLoadGprFromGpr8Hi(pReNative, off, idxVarReg, idxGstFullReg);
2275	switch (cbSignExtended)
2276	{
2277	case sizeof(uint16_t):
2278	off = iemNativeEmitLoadGpr16SignExtendedFromGpr8(pReNative, off, idxVarReg, idxVarReg);
2279	break;
2280	case sizeof(uint32_t):
2281	off = iemNativeEmitLoadGpr32SignExtendedFromGpr8(pReNative, off, idxVarReg, idxVarReg);
2282	break;
2283	case sizeof(uint64_t):
2284	off = iemNativeEmitLoadGprSignExtendedFromGpr8(pReNative, off, idxVarReg, idxVarReg);
2285	break;
2286	default: AssertFailed(); break;
2287	}
2288	}
2289
2290	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2291	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2292	return off;
2293	}
2294
2295
2296
2297	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) \
2298	off = iemNativeEmitFetchGregU16(pReNative, off, a_u16Dst, a_iGReg, sizeof(uint16_t))
2299
2300	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u16Dst, a_iGReg) \
2301	off = iemNativeEmitFetchGregU16(pReNative, off, a_u16Dst, a_iGReg, sizeof(uint32_t))
2302
2303	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u16Dst, a_iGReg) \
2304	off = iemNativeEmitFetchGregU16(pReNative, off, a_u16Dst, a_iGReg, sizeof(uint64_t))
2305
2306	/** Emits code for IEM_MC_FETCH_GREG_U16 and IEM_MC_FETCH_GREG_U16_ZX_U32/64. */
2307	DECL_INLINE_THROW(uint32_t)
2308	iemNativeEmitFetchGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg, uint8_t cbZeroExtended)
2309	{
2310	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2311	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
2312	Assert(iGReg < 16);
2313
2314	/*
2315	* We can either just load the low 16-bit of the GPR into a host register
2316	* for the variable, or we can do so via a shadow copy host register. The
2317	* latter will avoid having to reload it if it's being stored later, but
2318	* will waste a host register if it isn't touched again. Since we don't
2319	* know what going to happen, we choose the latter for now.
2320	*/
2321	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2322	kIemNativeGstRegUse_ReadOnly);
2323
2324	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2325	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2326	off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxVarReg, idxGstFullReg);
2327	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2328
2329	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2330	return off;
2331	}
2332
2333
2334	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u16Dst, a_iGReg) \
2335	off = iemNativeEmitFetchGregU16Sx(pReNative, off, a_u16Dst, a_iGReg, sizeof(uint32_t))
2336
2337	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u16Dst, a_iGReg) \
2338	off = iemNativeEmitFetchGregU16Sx(pReNative, off, a_u16Dst, a_iGReg, sizeof(uint64_t))
2339
2340	/** Emits code for IEM_MC_FETCH_GREG_U16_SX_U32/64. */
2341	DECL_INLINE_THROW(uint32_t)
2342	iemNativeEmitFetchGregU16Sx(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg, uint8_t cbSignExtended)
2343	{
2344	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2345	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbSignExtended);
2346	Assert(iGReg < 16);
2347
2348	/*
2349	* We can either just load the low 16-bit of the GPR into a host register
2350	* for the variable, or we can do so via a shadow copy host register. The
2351	* latter will avoid having to reload it if it's being stored later, but
2352	* will waste a host register if it isn't touched again. Since we don't
2353	* know what going to happen, we choose the latter for now.
2354	*/
2355	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2356	kIemNativeGstRegUse_ReadOnly);
2357
2358	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2359	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2360	if (cbSignExtended == sizeof(uint32_t))
2361	off = iemNativeEmitLoadGpr32SignExtendedFromGpr16(pReNative, off, idxVarReg, idxGstFullReg);
2362	else
2363	{
2364	Assert(cbSignExtended == sizeof(uint64_t));
2365	off = iemNativeEmitLoadGprSignExtendedFromGpr16(pReNative, off, idxVarReg, idxGstFullReg);
2366	}
2367	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2368
2369	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2370	return off;
2371	}
2372
2373
2374	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) \
2375	off = iemNativeEmitFetchGregU32(pReNative, off, a_u32Dst, a_iGReg, sizeof(uint32_t))
2376
2377	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u32Dst, a_iGReg) \
2378	off = iemNativeEmitFetchGregU32(pReNative, off, a_u32Dst, a_iGReg, sizeof(uint64_t))
2379
2380	/** Emits code for IEM_MC_FETCH_GREG_U32. */
2381	DECL_INLINE_THROW(uint32_t)
2382	iemNativeEmitFetchGregU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg, uint8_t cbZeroExtended)
2383	{
2384	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2385	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
2386	Assert(iGReg < 16);
2387
2388	/*
2389	* We can either just load the low 16-bit of the GPR into a host register
2390	* for the variable, or we can do so via a shadow copy host register. The
2391	* latter will avoid having to reload it if it's being stored later, but
2392	* will waste a host register if it isn't touched again. Since we don't
2393	* know what going to happen, we choose the latter for now.
2394	*/
2395	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2396	kIemNativeGstRegUse_ReadOnly);
2397
2398	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2399	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2400	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxVarReg, idxGstFullReg);
2401	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2402
2403	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2404	return off;
2405	}
2406
2407
2408	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u32Dst, a_iGReg) \
2409	off = iemNativeEmitFetchGregU32SxU64(pReNative, off, a_u32Dst, a_iGReg)
2410
2411	/** Emits code for IEM_MC_FETCH_GREG_U32. */
2412	DECL_INLINE_THROW(uint32_t)
2413	iemNativeEmitFetchGregU32SxU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg)
2414	{
2415	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2416	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
2417	Assert(iGReg < 16);
2418
2419	/*
2420	* We can either just load the low 32-bit of the GPR into a host register
2421	* for the variable, or we can do so via a shadow copy host register. The
2422	* latter will avoid having to reload it if it's being stored later, but
2423	* will waste a host register if it isn't touched again. Since we don't
2424	* know what going to happen, we choose the latter for now.
2425	*/
2426	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2427	kIemNativeGstRegUse_ReadOnly);
2428
2429	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2430	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2431	off = iemNativeEmitLoadGprSignExtendedFromGpr32(pReNative, off, idxVarReg, idxGstFullReg);
2432	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2433
2434	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2435	return off;
2436	}
2437
2438
2439	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) \
2440	off = iemNativeEmitFetchGregU64(pReNative, off, a_u64Dst, a_iGReg)
2441
2442	#define IEM_MC_FETCH_GREG_U64_ZX_U64(a_u64Dst, a_iGReg) \
2443	off = iemNativeEmitFetchGregU64(pReNative, off, a_u64Dst, a_iGReg)
2444
2445	/** Emits code for IEM_MC_FETCH_GREG_U64 (and the
2446	* IEM_MC_FETCH_GREG_U64_ZX_U64 alias). */
2447	DECL_INLINE_THROW(uint32_t)
2448	iemNativeEmitFetchGregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg)
2449	{
2450	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
2451	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
2452	Assert(iGReg < 16);
2453
2454	uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2455	kIemNativeGstRegUse_ReadOnly);
2456
2457	iemNativeVarSetKindToStack(pReNative, idxDstVar);
2458	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
2459	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxVarReg, idxGstFullReg);
2460	/** @todo name the register a shadow one already? */
2461	iemNativeVarRegisterRelease(pReNative, idxDstVar);
2462
2463	iemNativeRegFreeTmp(pReNative, idxGstFullReg);
2464	return off;
2465	}
2466
2467
2468
2469	/*********************************************************************************************************************************
2470	* Emitters for general purpose register stores (IEM_MC_STORE_GREG_XXX). *
2471	*********************************************************************************************************************************/
2472
2473	#define IEM_MC_STORE_GREG_U8_CONST_THREADED(a_iGRegEx, a_u8Value) \
2474	off = iemNativeEmitStoreGregU8Const(pReNative, off, a_iGRegEx, a_u8Value)
2475
2476	/** Emits code for IEM_MC_STORE_GREG_U8_CONST_THREADED. */
2477	DECL_INLINE_THROW(uint32_t)
2478	iemNativeEmitStoreGregU8Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t u8Value)
2479	{
2480	Assert(iGRegEx < 20);
2481	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGRegEx & 15),
2482	kIemNativeGstRegUse_ForUpdate);
2483	#ifdef RT_ARCH_AMD64
2484	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 12);
2485
2486	/* To the lowest byte of the register: mov r8, imm8 */
2487	if (iGRegEx < 16)
2488	{
2489	if (idxGstTmpReg >= 8)
2490	pbCodeBuf[off++] = X86_OP_REX_B;
2491	else if (idxGstTmpReg >= 4)
2492	pbCodeBuf[off++] = X86_OP_REX;
2493	pbCodeBuf[off++] = 0xb0 + (idxGstTmpReg & 7);
2494	pbCodeBuf[off++] = u8Value;
2495	}
2496	/* Otherwise it's to ah, ch, dh or bh: use mov r8, imm8 if we can, otherwise, we rotate. */
2497	else if (idxGstTmpReg < 4)
2498	{
2499	pbCodeBuf[off++] = 0xb4 + idxGstTmpReg;
2500	pbCodeBuf[off++] = u8Value;
2501	}
2502	else
2503	{
2504	/* ror reg64, 8 */
2505	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg < 8 ? 0 : X86_OP_REX_B);
2506	pbCodeBuf[off++] = 0xc1;
2507	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, idxGstTmpReg & 7);
2508	pbCodeBuf[off++] = 8;
2509
2510	/* mov reg8, imm8 */
2511	if (idxGstTmpReg >= 8)
2512	pbCodeBuf[off++] = X86_OP_REX_B;
2513	else if (idxGstTmpReg >= 4)
2514	pbCodeBuf[off++] = X86_OP_REX;
2515	pbCodeBuf[off++] = 0xb0 + (idxGstTmpReg & 7);
2516	pbCodeBuf[off++] = u8Value;
2517
2518	/* rol reg64, 8 */
2519	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg < 8 ? 0 : X86_OP_REX_B);
2520	pbCodeBuf[off++] = 0xc1;
2521	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2522	pbCodeBuf[off++] = 8;
2523	}
2524
2525	#elif defined(RT_ARCH_ARM64)
2526	uint8_t const idxImmReg = iemNativeRegAllocTmpImm(pReNative, &off, u8Value);
2527	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2528	if (iGRegEx < 16)
2529	/* bfi w1, w2, 0, 8 - moves bits 7:0 from idxImmReg to idxGstTmpReg bits 7:0. */
2530	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxImmReg, 0, 8);
2531	else
2532	/* bfi w1, w2, 8, 8 - moves bits 7:0 from idxImmReg to idxGstTmpReg bits 15:8. */
2533	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxImmReg, 8, 8);
2534	iemNativeRegFreeTmp(pReNative, idxImmReg);
2535
2536	#else
2537	# error "Port me!"
2538	#endif
2539
2540	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2541
2542	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGRegEx & 15]));
2543
2544	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2545	return off;
2546	}
2547
2548
2549	#define IEM_MC_STORE_GREG_U8_THREADED(a_iGRegEx, a_u8Value) \
2550	off = iemNativeEmitStoreGregU8(pReNative, off, a_iGRegEx, a_u8Value)
2551
2552	/** Emits code for IEM_MC_STORE_GREG_U8_THREADED. */
2553	DECL_INLINE_THROW(uint32_t)
2554	iemNativeEmitStoreGregU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t idxValueVar)
2555	{
2556	Assert(iGRegEx < 20);
2557	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxValueVar);
2558
2559	/*
2560	* If it's a constant value (unlikely) we treat this as a
2561	* IEM_MC_STORE_GREG_U8_CONST statement.
2562	*/
2563	PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
2564	if (pValueVar->enmKind == kIemNativeVarKind_Stack)
2565	{ /* likely */ }
2566	else
2567	{
2568	AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
2569	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
2570	return iemNativeEmitStoreGregU8Const(pReNative, off, iGRegEx, (uint8_t)pValueVar->u.uValue);
2571	}
2572
2573	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGRegEx & 15),
2574	kIemNativeGstRegUse_ForUpdate);
2575	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxValueVar, &off, true /fInitialized/);
2576
2577	#ifdef RT_ARCH_AMD64
2578	/* To the lowest byte of the register: mov reg8, reg8(r/m) */
2579	if (iGRegEx < 16)
2580	{
2581	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2582	if (idxGstTmpReg >= 8 \|\| idxVarReg >= 8)
2583	pbCodeBuf[off++] = (idxGstTmpReg >= 8 ? X86_OP_REX_R : 0) \| (idxVarReg >= 8 ? X86_OP_REX_B : 0);
2584	else if (idxGstTmpReg >= 4 \|\| idxVarReg >= 4)
2585	pbCodeBuf[off++] = X86_OP_REX;
2586	pbCodeBuf[off++] = 0x8a;
2587	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg & 7, idxVarReg & 7);
2588	}
2589	/* Otherwise it's to ah, ch, dh or bh from al, cl, dl or bl: use mov r8, r8 if we can, otherwise, we rotate. */
2590	else if (idxGstTmpReg < 4 && idxVarReg < 4)
2591	{
2592	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2+1);
2593	pbCodeBuf[off++] = 0x8a;
2594	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg + 4, idxVarReg);
2595	}
2596	else
2597	{
2598	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 15);
2599
2600	/* ror reg64, 8 */
2601	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg < 8 ? 0 : X86_OP_REX_B);
2602	pbCodeBuf[off++] = 0xc1;
2603	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, idxGstTmpReg & 7);
2604	pbCodeBuf[off++] = 8;
2605
2606	/* mov reg8, reg8(r/m) */
2607	if (idxGstTmpReg >= 8 \|\| idxVarReg >= 8)
2608	pbCodeBuf[off++] = (idxGstTmpReg >= 8 ? X86_OP_REX_R : 0) \| (idxVarReg >= 8 ? X86_OP_REX_B : 0);
2609	else if (idxGstTmpReg >= 4 \|\| idxVarReg >= 4)
2610	pbCodeBuf[off++] = X86_OP_REX;
2611	pbCodeBuf[off++] = 0x8a;
2612	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg & 7, idxVarReg & 7);
2613
2614	/* rol reg64, 8 */
2615	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg < 8 ? 0 : X86_OP_REX_B);
2616	pbCodeBuf[off++] = 0xc1;
2617	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2618	pbCodeBuf[off++] = 8;
2619	}
2620
2621	#elif defined(RT_ARCH_ARM64)
2622	/* bfi w1, w2, 0, 8 - moves bits 7:0 from idxVarReg to idxGstTmpReg bits 7:0.
2623	or
2624	bfi w1, w2, 8, 8 - moves bits 7:0 from idxVarReg to idxGstTmpReg bits 15:8. */
2625	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2626	if (iGRegEx < 16)
2627	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxVarReg, 0, 8);
2628	else
2629	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxVarReg, 8, 8);
2630
2631	#else
2632	# error "Port me!"
2633	#endif
2634	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2635
2636	iemNativeVarRegisterRelease(pReNative, idxValueVar);
2637
2638	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGRegEx & 15]));
2639	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2640	return off;
2641	}
2642
2643
2644
2645	#define IEM_MC_STORE_GREG_U16_CONST(a_iGReg, a_u16Const) \
2646	off = iemNativeEmitStoreGregU16Const(pReNative, off, a_iGReg, a_u16Const)
2647
2648	/** Emits code for IEM_MC_STORE_GREG_U16. */
2649	DECL_INLINE_THROW(uint32_t)
2650	iemNativeEmitStoreGregU16Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint16_t uValue)
2651	{
2652	Assert(iGReg < 16);
2653	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2654	kIemNativeGstRegUse_ForUpdate);
2655	#ifdef RT_ARCH_AMD64
2656	/* mov reg16, imm16 */
2657	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2658	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2659	if (idxGstTmpReg >= 8)
2660	pbCodeBuf[off++] = X86_OP_REX_B;
2661	pbCodeBuf[off++] = 0xb8 + (idxGstTmpReg & 7);
2662	pbCodeBuf[off++] = RT_BYTE1(uValue);
2663	pbCodeBuf[off++] = RT_BYTE2(uValue);
2664
2665	#elif defined(RT_ARCH_ARM64)
2666	/* movk xdst, #uValue, lsl #0 */
2667	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2668	pu32CodeBuf[off++] = Armv8A64MkInstrMovK(idxGstTmpReg, uValue);
2669
2670	#else
2671	# error "Port me!"
2672	#endif
2673
2674	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2675
2676	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2677	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2678	return off;
2679	}
2680
2681
2682	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) \
2683	off = iemNativeEmitStoreGregU16(pReNative, off, a_iGReg, a_u16Value)
2684
2685	/** Emits code for IEM_MC_STORE_GREG_U16. */
2686	DECL_INLINE_THROW(uint32_t)
2687	iemNativeEmitStoreGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t idxValueVar)
2688	{
2689	Assert(iGReg < 16);
2690	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxValueVar);
2691
2692	/*
2693	* If it's a constant value (unlikely) we treat this as a
2694	* IEM_MC_STORE_GREG_U16_CONST statement.
2695	*/
2696	PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
2697	if (pValueVar->enmKind == kIemNativeVarKind_Stack)
2698	{ /* likely */ }
2699	else
2700	{
2701	AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
2702	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
2703	return iemNativeEmitStoreGregU16Const(pReNative, off, iGReg, (uint16_t)pValueVar->u.uValue);
2704	}
2705
2706	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2707	kIemNativeGstRegUse_ForUpdate);
2708
2709	#ifdef RT_ARCH_AMD64
2710	/* mov reg16, reg16 or [mem16] */
2711	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 12);
2712	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2713	if (pValueVar->idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
2714	{
2715	if (idxGstTmpReg >= 8 \|\| pValueVar->idxReg >= 8)
2716	pbCodeBuf[off++] = (idxGstTmpReg >= 8 ? X86_OP_REX_R : 0)
2717	\| (pValueVar->idxReg >= 8 ? X86_OP_REX_B : 0);
2718	pbCodeBuf[off++] = 0x8b;
2719	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg & 7, pValueVar->idxReg & 7);
2720	}
2721	else
2722	{
2723	uint8_t const idxStackSlot = pValueVar->idxStackSlot;
2724	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
2725	if (idxGstTmpReg >= 8)
2726	pbCodeBuf[off++] = X86_OP_REX_R;
2727	pbCodeBuf[off++] = 0x8b;
2728	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, idxGstTmpReg, iemNativeStackCalcBpDisp(idxStackSlot), pReNative);
2729	}
2730
2731	#elif defined(RT_ARCH_ARM64)
2732	/* bfi w1, w2, 0, 16 - moves bits 15:0 from idxVarReg to idxGstTmpReg bits 15:0. */
2733	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxValueVar, &off, true /fInitialized/);
2734	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2735	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxVarReg, 0, 16);
2736	iemNativeVarRegisterRelease(pReNative, idxValueVar);
2737
2738	#else
2739	# error "Port me!"
2740	#endif
2741
2742	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2743
2744	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2745	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2746	return off;
2747	}
2748
2749
2750	#define IEM_MC_STORE_GREG_U32_CONST(a_iGReg, a_u32Const) \
2751	off = iemNativeEmitStoreGregU32Const(pReNative, off, a_iGReg, a_u32Const)
2752
2753	/** Emits code for IEM_MC_STORE_GREG_U32_CONST. */
2754	DECL_INLINE_THROW(uint32_t)
2755	iemNativeEmitStoreGregU32Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint32_t uValue)
2756	{
2757	Assert(iGReg < 16);
2758	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2759	kIemNativeGstRegUse_ForFullWrite);
2760	off = iemNativeEmitLoadGprImm64(pReNative, off, idxGstTmpReg, uValue);
2761	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2762	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2763	return off;
2764	}
2765
2766
2767	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) \
2768	off = iemNativeEmitStoreGregU32(pReNative, off, a_iGReg, a_u32Value)
2769
2770	/** Emits code for IEM_MC_STORE_GREG_U32. */
2771	DECL_INLINE_THROW(uint32_t)
2772	iemNativeEmitStoreGregU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t idxValueVar)
2773	{
2774	Assert(iGReg < 16);
2775	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxValueVar);
2776
2777	/*
2778	* If it's a constant value (unlikely) we treat this as a
2779	* IEM_MC_STORE_GREG_U32_CONST statement.
2780	*/
2781	PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
2782	if (pValueVar->enmKind == kIemNativeVarKind_Stack)
2783	{ /* likely */ }
2784	else
2785	{
2786	AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
2787	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
2788	return iemNativeEmitStoreGregU32Const(pReNative, off, iGReg, (uint32_t)pValueVar->u.uValue);
2789	}
2790
2791	/*
2792	* For the rest we allocate a guest register for the variable and writes
2793	* it to the CPUMCTX structure.
2794	*/
2795	uint8_t const idxVarReg = iemNativeVarRegisterAcquireForGuestReg(pReNative, idxValueVar, IEMNATIVEGSTREG_GPR(iGReg), &off);
2796	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxVarReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2797	#ifdef VBOX_STRICT
2798	off = iemNativeEmitTop32BitsClearCheck(pReNative, off, idxVarReg);
2799	#endif
2800	iemNativeVarRegisterRelease(pReNative, idxValueVar);
2801	return off;
2802	}
2803
2804
2805	#define IEM_MC_STORE_GREG_U64_CONST(a_iGReg, a_u64Const) \
2806	off = iemNativeEmitStoreGregU64Const(pReNative, off, a_iGReg, a_u64Const)
2807
2808	/** Emits code for IEM_MC_STORE_GREG_U64_CONST. */
2809	DECL_INLINE_THROW(uint32_t)
2810	iemNativeEmitStoreGregU64Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint64_t uValue)
2811	{
2812	Assert(iGReg < 16);
2813	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2814	kIemNativeGstRegUse_ForFullWrite);
2815	off = iemNativeEmitLoadGprImm64(pReNative, off, idxGstTmpReg, uValue);
2816	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2817	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2818	return off;
2819	}
2820
2821
2822	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) \
2823	off = iemNativeEmitStoreGregU64(pReNative, off, a_iGReg, a_u64Value)
2824
2825	/** Emits code for IEM_MC_STORE_GREG_U64. */
2826	DECL_INLINE_THROW(uint32_t)
2827	iemNativeEmitStoreGregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t idxValueVar)
2828	{
2829	Assert(iGReg < 16);
2830	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxValueVar);
2831
2832	/*
2833	* If it's a constant value (unlikely) we treat this as a
2834	* IEM_MC_STORE_GREG_U64_CONST statement.
2835	*/
2836	PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
2837	if (pValueVar->enmKind == kIemNativeVarKind_Stack)
2838	{ /* likely */ }
2839	else
2840	{
2841	AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
2842	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
2843	return iemNativeEmitStoreGregU64Const(pReNative, off, iGReg, pValueVar->u.uValue);
2844	}
2845
2846	/*
2847	* For the rest we allocate a guest register for the variable and writes
2848	* it to the CPUMCTX structure.
2849	*/
2850	uint8_t const idxVarReg = iemNativeVarRegisterAcquireForGuestReg(pReNative, idxValueVar, IEMNATIVEGSTREG_GPR(iGReg), &off);
2851	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxVarReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2852	iemNativeVarRegisterRelease(pReNative, idxValueVar);
2853	return off;
2854	}
2855
2856
2857	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) \
2858	off = iemNativeEmitClearHighGregU64(pReNative, off, a_iGReg)
2859
2860	/** Emits code for IEM_MC_CLEAR_HIGH_GREG_U64. */
2861	DECL_INLINE_THROW(uint32_t)
2862	iemNativeEmitClearHighGregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg)
2863	{
2864	Assert(iGReg < 16);
2865	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2866	kIemNativeGstRegUse_ForUpdate);
2867	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxGstTmpReg, idxGstTmpReg);
2868	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2869	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2870	return off;
2871	}
2872
2873
2874	/*********************************************************************************************************************************
2875	* General purpose register manipulation (add, sub). *
2876	*********************************************************************************************************************************/
2877
2878	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u8SubtrahendConst) \
2879	off = iemNativeEmitAddGregU16(pReNative, off, a_iGReg, a_u8SubtrahendConst)
2880
2881	/** Emits code for IEM_MC_ADD_GREG_U16. */
2882	DECL_INLINE_THROW(uint32_t)
2883	iemNativeEmitAddGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uAddend)
2884	{
2885	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2886	kIemNativeGstRegUse_ForUpdate);
2887
2888	#ifdef RT_ARCH_AMD64
2889	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2890	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2891	if (idxGstTmpReg >= 8)
2892	pbCodeBuf[off++] = X86_OP_REX_B;
2893	if (uAddend == 1)
2894	{
2895	pbCodeBuf[off++] = 0xff; /* inc */
2896	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2897	}
2898	else
2899	{
2900	pbCodeBuf[off++] = 0x81;
2901	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2902	pbCodeBuf[off++] = uAddend;
2903	pbCodeBuf[off++] = 0;
2904	}
2905
2906	#else
2907	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
2908	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2909
2910	/* sub tmp, gstgrp, uAddend */
2911	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxTmpReg, idxGstTmpReg, uAddend, false /f64Bit/);
2912
2913	/* bfi w1, w2, 0, 16 - moves bits 15:0 from tmpreg2 to tmpreg. */
2914	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxTmpReg, 0, 16);
2915
2916	iemNativeRegFreeTmp(pReNative, idxTmpReg);
2917	#endif
2918
2919	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2920
2921	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2922
2923	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2924	return off;
2925	}
2926
2927
2928	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u8Const) \
2929	off = iemNativeEmitAddGregU32U64(pReNative, off, a_iGReg, a_u8Const, false /f64Bit/)
2930
2931	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u8Const) \
2932	off = iemNativeEmitAddGregU32U64(pReNative, off, a_iGReg, a_u8Const, true /f64Bit/)
2933
2934	/** Emits code for IEM_MC_ADD_GREG_U32 and IEM_MC_ADD_GREG_U64. */
2935	DECL_INLINE_THROW(uint32_t)
2936	iemNativeEmitAddGregU32U64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uAddend, bool f64Bit)
2937	{
2938	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2939	kIemNativeGstRegUse_ForUpdate);
2940
2941	#ifdef RT_ARCH_AMD64
2942	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2943	if (f64Bit)
2944	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg >= 8 ? X86_OP_REX_B : 0);
2945	else if (idxGstTmpReg >= 8)
2946	pbCodeBuf[off++] = X86_OP_REX_B;
2947	if (uAddend == 1)
2948	{
2949	pbCodeBuf[off++] = 0xff; /* inc */
2950	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2951	}
2952	else if (uAddend < 128)
2953	{
2954	pbCodeBuf[off++] = 0x83; /* add */
2955	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2956	pbCodeBuf[off++] = RT_BYTE1(uAddend);
2957	}
2958	else
2959	{
2960	pbCodeBuf[off++] = 0x81; /* add */
2961	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxGstTmpReg & 7);
2962	pbCodeBuf[off++] = RT_BYTE1(uAddend);
2963	pbCodeBuf[off++] = 0;
2964	pbCodeBuf[off++] = 0;
2965	pbCodeBuf[off++] = 0;
2966	}
2967
2968	#else
2969	/* sub tmp, gstgrp, uAddend */
2970	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2971	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxGstTmpReg, idxGstTmpReg, uAddend, f64Bit);
2972
2973	#endif
2974
2975	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2976
2977	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
2978
2979	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
2980	return off;
2981	}
2982
2983
2984
2985	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8SubtrahendConst) \
2986	off = iemNativeEmitSubGregU16(pReNative, off, a_iGReg, a_u8SubtrahendConst)
2987
2988	/** Emits code for IEM_MC_SUB_GREG_U16. */
2989	DECL_INLINE_THROW(uint32_t)
2990	iemNativeEmitSubGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend)
2991	{
2992	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
2993	kIemNativeGstRegUse_ForUpdate);
2994
2995	#ifdef RT_ARCH_AMD64
2996	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2997	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2998	if (idxGstTmpReg >= 8)
2999	pbCodeBuf[off++] = X86_OP_REX_B;
3000	if (uSubtrahend == 1)
3001	{
3002	pbCodeBuf[off++] = 0xff; /* dec */
3003	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, idxGstTmpReg & 7);
3004	}
3005	else
3006	{
3007	pbCodeBuf[off++] = 0x81;
3008	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, idxGstTmpReg & 7);
3009	pbCodeBuf[off++] = uSubtrahend;
3010	pbCodeBuf[off++] = 0;
3011	}
3012
3013	#else
3014	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
3015	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3016
3017	/* sub tmp, gstgrp, uSubtrahend */
3018	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, idxTmpReg, idxGstTmpReg, uSubtrahend, false /f64Bit/);
3019
3020	/* bfi w1, w2, 0, 16 - moves bits 15:0 from tmpreg2 to tmpreg. */
3021	pu32CodeBuf[off++] = Armv8A64MkInstrBfi(idxGstTmpReg, idxTmpReg, 0, 16);
3022
3023	iemNativeRegFreeTmp(pReNative, idxTmpReg);
3024	#endif
3025
3026	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3027
3028	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
3029
3030	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
3031	return off;
3032	}
3033
3034
3035	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
3036	off = iemNativeEmitSubGregU32U64(pReNative, off, a_iGReg, a_u8Const, false /f64Bit/)
3037
3038	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) \
3039	off = iemNativeEmitSubGregU32U64(pReNative, off, a_iGReg, a_u8Const, true /f64Bit/)
3040
3041	/** Emits code for IEM_MC_SUB_GREG_U32 and IEM_MC_SUB_GREG_U64. */
3042	DECL_INLINE_THROW(uint32_t)
3043	iemNativeEmitSubGregU32U64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend, bool f64Bit)
3044	{
3045	uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(iGReg),
3046	kIemNativeGstRegUse_ForUpdate);
3047
3048	#ifdef RT_ARCH_AMD64
3049	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3050	if (f64Bit)
3051	pbCodeBuf[off++] = X86_OP_REX_W \| (idxGstTmpReg >= 8 ? X86_OP_REX_B : 0);
3052	else if (idxGstTmpReg >= 8)
3053	pbCodeBuf[off++] = X86_OP_REX_B;
3054	if (uSubtrahend == 1)
3055	{
3056	pbCodeBuf[off++] = 0xff; /* dec */
3057	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, idxGstTmpReg & 7);
3058	}
3059	else if (uSubtrahend < 128)
3060	{
3061	pbCodeBuf[off++] = 0x83; /* sub */
3062	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, idxGstTmpReg & 7);
3063	pbCodeBuf[off++] = RT_BYTE1(uSubtrahend);
3064	}
3065	else
3066	{
3067	pbCodeBuf[off++] = 0x81; /* sub */
3068	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, idxGstTmpReg & 7);
3069	pbCodeBuf[off++] = RT_BYTE1(uSubtrahend);
3070	pbCodeBuf[off++] = 0;
3071	pbCodeBuf[off++] = 0;
3072	pbCodeBuf[off++] = 0;
3073	}
3074
3075	#else
3076	/* sub tmp, gstgrp, uSubtrahend */
3077	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3078	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, idxGstTmpReg, idxGstTmpReg, uSubtrahend, f64Bit);
3079
3080	#endif
3081
3082	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3083
3084	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxGstTmpReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[iGReg]));
3085
3086	iemNativeRegFreeTmp(pReNative, idxGstTmpReg);
3087	return off;
3088	}
3089
3090
3091	/*********************************************************************************************************************************
3092	* Local variable manipulation (add, sub, and, or). *
3093	*********************************************************************************************************************************/
3094
3095	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) \
3096	off = iemNativeEmitAndLocal(pReNative, off, a_u8Local, a_u8Mask, sizeof(uint8_t))
3097
3098	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) \
3099	off = iemNativeEmitAndLocal(pReNative, off, a_u16Local, a_u16Mask, sizeof(uint16_t))
3100
3101	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) \
3102	off = iemNativeEmitAndLocal(pReNative, off, a_u32Local, a_u32Mask, sizeof(uint32_t))
3103
3104	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) \
3105	off = iemNativeEmitAndLocal(pReNative, off, a_u64Local, a_u64Mask, sizeof(uint64_t))
3106
3107	/** Emits code for AND'ing a local and a constant value. */
3108	DECL_INLINE_THROW(uint32_t)
3109	iemNativeEmitAndLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint64_t uMask, uint8_t cbMask)
3110	{
3111	#ifdef VBOX_STRICT
3112	switch (cbMask)
3113	{
3114	case sizeof(uint8_t): Assert((uint8_t)uMask == uMask); break;
3115	case sizeof(uint16_t): Assert((uint16_t)uMask == uMask); break;
3116	case sizeof(uint32_t): Assert((uint32_t)uMask == uMask); break;
3117	case sizeof(uint64_t): break;
3118	default: AssertFailedBreak();
3119	}
3120	#endif
3121
3122	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /fInitialized/);
3123	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbMask);
3124
3125	if (cbMask <= sizeof(uint32_t))
3126	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxVarReg, uMask);
3127	else
3128	off = iemNativeEmitAndGprByImm(pReNative, off, idxVarReg, uMask);
3129
3130	iemNativeVarRegisterRelease(pReNative, idxVar);
3131	return off;
3132	}
3133
3134
3135	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) \
3136	off = iemNativeEmitOrLocal(pReNative, off, a_u8Local, a_u8Mask, sizeof(uint8_t))
3137
3138	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) \
3139	off = iemNativeEmitOrLocal(pReNative, off, a_u16Local, a_u16Mask, sizeof(uint16_t))
3140
3141	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) \
3142	off = iemNativeEmitOrLocal(pReNative, off, a_u32Local, a_u32Mask, sizeof(uint32_t))
3143
3144	#define IEM_MC_OR_LOCAL_U64(a_u64Local, a_u64Mask) \
3145	off = iemNativeEmitOrLocal(pReNative, off, a_u64Local, a_u64Mask, sizeof(uint64_t))
3146
3147	/** Emits code for OR'ing a local and a constant value. */
3148	DECL_INLINE_THROW(uint32_t)
3149	iemNativeEmitOrLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint64_t uMask, uint8_t cbMask)
3150	{
3151	#ifdef VBOX_STRICT
3152	switch (cbMask)
3153	{
3154	case sizeof(uint8_t): Assert((uint8_t)uMask == uMask); break;
3155	case sizeof(uint16_t): Assert((uint16_t)uMask == uMask); break;
3156	case sizeof(uint32_t): Assert((uint32_t)uMask == uMask); break;
3157	case sizeof(uint64_t): break;
3158	default: AssertFailedBreak();
3159	}
3160	#endif
3161
3162	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /fInitialized/);
3163	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbMask);
3164
3165	if (cbMask <= sizeof(uint32_t))
3166	off = iemNativeEmitOrGpr32ByImm(pReNative, off, idxVarReg, uMask);
3167	else
3168	off = iemNativeEmitOrGprByImm(pReNative, off, idxVarReg, uMask);
3169
3170	iemNativeVarRegisterRelease(pReNative, idxVar);
3171	return off;
3172	}
3173
3174
3175	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) \
3176	off = iemNativeEmitBswapLocal(pReNative, off, a_u16Local, sizeof(uint16_t))
3177
3178	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) \
3179	off = iemNativeEmitBswapLocal(pReNative, off, a_u32Local, sizeof(uint32_t))
3180
3181	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) \
3182	off = iemNativeEmitBswapLocal(pReNative, off, a_u64Local, sizeof(uint64_t))
3183
3184	/** Emits code for reversing the byte order in a local value. */
3185	DECL_INLINE_THROW(uint32_t)
3186	iemNativeEmitBswapLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint8_t cbLocal)
3187	{
3188	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /fInitialized/);
3189	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbLocal);
3190
3191	switch (cbLocal)
3192	{
3193	case sizeof(uint16_t): off = iemNativeEmitBswapGpr16(pReNative, off, idxVarReg); break;
3194	case sizeof(uint32_t): off = iemNativeEmitBswapGpr32(pReNative, off, idxVarReg); break;
3195	case sizeof(uint64_t): off = iemNativeEmitBswapGpr(pReNative, off, idxVarReg); break;
3196	default: AssertFailedBreak();
3197	}
3198
3199	iemNativeVarRegisterRelease(pReNative, idxVar);
3200	return off;
3201	}
3202
3203
3204
3205	/*********************************************************************************************************************************
3206	* EFLAGS *
3207	*********************************************************************************************************************************/
3208
3209	#if !defined(VBOX_WITH_STATISTICS) \|\| !defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS)
3210	# define IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput) ((void)0)
3211	#else
3212	# define IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput) \
3213	iemNativeEFlagsOptimizationStats(pReNative, a_fEflInput, a_fEflOutput)
3214
3215	DECLINLINE(void) iemNativeEFlagsOptimizationStats(PIEMRECOMPILERSTATE pReNative, uint32_t fEflInput, uint32_t fEflOutput)
3216	{
3217	if (fEflOutput)
3218	{
3219	PVMCPUCC const pVCpu = pReNative->pVCpu;
3220	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
3221	IEMLIVENESSBIT const LivenessBit0 = pReNative->paLivenessEntries[pReNative->idxCurCall].Bit0;
3222	IEMLIVENESSBIT const LivenessBit1 = pReNative->paLivenessEntries[pReNative->idxCurCall].Bit1;
3223	AssertCompile(IEMLIVENESS_STATE_CLOBBERED == 0);
3224	# define CHECK_FLAG_AND_UPDATE_STATS(a_fEfl, a_fLivenessMember, a_CoreStatName) \
3225	if (fEflOutput & (a_fEfl)) \
3226	{ \
3227	if (LivenessBit0.a_fLivenessMember \| LivenessBit1.a_fLivenessMember) \
3228	STAM_COUNTER_INC(&pVCpu->iem.s.a_CoreStatName ## Required); \
3229	else \
3230	STAM_COUNTER_INC(&pVCpu->iem.s.a_CoreStatName ## Skippable); \
3231	} else do { } while (0)
3232	# else
3233	PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall];
3234	IEMLIVENESSBIT const LivenessClobbered =
3235	{
3236	pLivenessEntry->aBits[IEMLIVENESS_BIT_WRITE].bm64
3237	& ~( pLivenessEntry->aBits[IEMLIVENESS_BIT_READ].bm64
3238	\| pLivenessEntry->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64
3239	\| pLivenessEntry->aBits[IEMLIVENESS_BIT_OTHER].bm64)
3240	};
3241	IEMLIVENESSBIT const LivenessDelayable =
3242	{
3243	pLivenessEntry->aBits[IEMLIVENESS_BIT_WRITE].bm64
3244	& pLivenessEntry->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64
3245	& ~( pLivenessEntry->aBits[IEMLIVENESS_BIT_READ].bm64
3246	\| pLivenessEntry->aBits[IEMLIVENESS_BIT_OTHER].bm64)
3247	};
3248	# define CHECK_FLAG_AND_UPDATE_STATS(a_fEfl, a_fLivenessMember, a_CoreStatName) \
3249	if (fEflOutput & (a_fEfl)) \
3250	{ \
3251	if (LivenessClobbered.a_fLivenessMember) \
3252	STAM_COUNTER_INC(&pVCpu->iem.s.a_CoreStatName ## Skippable); \
3253	else if (LivenessDelayable.a_fLivenessMember) \
3254	STAM_COUNTER_INC(&pVCpu->iem.s.a_CoreStatName ## Delayable); \
3255	else \
3256	STAM_COUNTER_INC(&pVCpu->iem.s.a_CoreStatName ## Required); \
3257	} else do { } while (0)
3258	# endif
3259	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_CF, fEflCf, StatNativeLivenessEflCf);
3260	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_PF, fEflPf, StatNativeLivenessEflPf);
3261	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_AF, fEflAf, StatNativeLivenessEflAf);
3262	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_ZF, fEflZf, StatNativeLivenessEflZf);
3263	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_SF, fEflSf, StatNativeLivenessEflSf);
3264	CHECK_FLAG_AND_UPDATE_STATS(X86_EFL_OF, fEflOf, StatNativeLivenessEflOf);
3265	//CHECK_FLAG_AND_UPDATE_STATS(~X86_EFL_STATUS_BITS, fEflOther, StatNativeLivenessEflOther);
3266	# undef CHECK_FLAG_AND_UPDATE_STATS
3267	}
3268	RT_NOREF(fEflInput);
3269	}
3270	#endif /* VBOX_WITH_STATISTICS */
3271
3272	#undef IEM_MC_FETCH_EFLAGS /* should not be used */
3273	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) \
3274	off = iemNativeEmitFetchEFlags(pReNative, off, a_EFlags, a_fEflInput, a_fEflOutput)
3275
3276	/** Handles IEM_MC_FETCH_EFLAGS_EX. */
3277	DECL_INLINE_THROW(uint32_t)
3278	iemNativeEmitFetchEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarEFlags,
3279	uint32_t fEflInput, uint32_t fEflOutput)
3280	{
3281	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarEFlags);
3282	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarEFlags, sizeof(uint32_t));
3283	RT_NOREF(fEflInput, fEflOutput);
3284
3285	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
3286	# ifdef VBOX_STRICT
3287	if ( pReNative->idxCurCall != 0
3288	&& (fEflInput != 0 \|\| fEflOutput != 0) /* for NOT these are both zero for now. */)
3289	{
3290	PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall - 1];
3291	uint32_t const fBoth = fEflInput \| fEflOutput;
3292	# define ASSERT_ONE_EFL(a_fElfConst, a_idxField) \
3293	AssertMsg( !(fBoth & (a_fElfConst)) \
3294	\|\| (!(fEflInput & (a_fElfConst)) \
3295	? IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, a_idxField)) \
3296	: !(fEflOutput & (a_fElfConst)) \
3297	? IEMLIVENESS_STATE_IS_INPUT_EXPECTED( iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, a_idxField)) \
3298	: IEMLIVENESS_STATE_IS_MODIFY_EXPECTED( iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, a_idxField)) ), \
3299	("%s - %u\n", #a_fElfConst, iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, a_idxField)))
3300	ASSERT_ONE_EFL(~(uint32_t)X86_EFL_STATUS_BITS, IEMLIVENESSBIT_IDX_EFL_OTHER);
3301	ASSERT_ONE_EFL(X86_EFL_CF, IEMLIVENESSBIT_IDX_EFL_CF);
3302	ASSERT_ONE_EFL(X86_EFL_PF, IEMLIVENESSBIT_IDX_EFL_PF);
3303	ASSERT_ONE_EFL(X86_EFL_AF, IEMLIVENESSBIT_IDX_EFL_AF);
3304	ASSERT_ONE_EFL(X86_EFL_ZF, IEMLIVENESSBIT_IDX_EFL_ZF);
3305	ASSERT_ONE_EFL(X86_EFL_SF, IEMLIVENESSBIT_IDX_EFL_SF);
3306	ASSERT_ONE_EFL(X86_EFL_OF, IEMLIVENESSBIT_IDX_EFL_OF);
3307	# undef ASSERT_ONE_EFL
3308	}
3309	# endif
3310	#endif
3311
3312	/** @todo this is suboptimial. EFLAGS is probably shadowed and we should use
3313	* the existing shadow copy. */
3314	uint8_t const idxReg = iemNativeVarRegisterAcquire(pReNative, idxVarEFlags, &off, false /fInitialized/);
3315	iemNativeRegClearAndMarkAsGstRegShadow(pReNative, idxReg, kIemNativeGstReg_EFlags, off);
3316	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.eflags));
3317	iemNativeVarRegisterRelease(pReNative, idxVarEFlags);
3318	return off;
3319	}
3320
3321
3322
3323	/** @todo emit strict build assertions for IEM_MC_COMMIT_EFLAGS_EX when we
3324	* start using it with custom native code emission (inlining assembly
3325	* instruction helpers). */
3326	#undef IEM_MC_COMMIT_EFLAGS /* should not be used */
3327	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) \
3328	IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput); \
3329	off = iemNativeEmitCommitEFlags(pReNative, off, a_EFlags, a_fEflOutput)
3330
3331	/** Handles IEM_MC_COMMIT_EFLAGS_EX. */
3332	DECL_INLINE_THROW(uint32_t)
3333	iemNativeEmitCommitEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarEFlags, uint32_t fEflOutput)
3334	{
3335	RT_NOREF(fEflOutput);
3336	uint8_t const idxReg = iemNativeVarRegisterAcquire(pReNative, idxVarEFlags, &off, true /fInitialized/);
3337	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarEFlags, sizeof(uint32_t));
3338
3339	#ifdef VBOX_STRICT
3340	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, idxReg, X86_EFL_RA1_MASK);
3341	uint32_t offFixup = off;
3342	off = iemNativeEmitJnzToFixed(pReNative, off, off);
3343	off = iemNativeEmitBrk(pReNative, off, UINT32_C(0x2001));
3344	iemNativeFixupFixedJump(pReNative, offFixup, off);
3345
3346	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, idxReg, X86_EFL_RAZ_MASK & CPUMX86EFLAGS_HW_MASK_32);
3347	offFixup = off;
3348	off = iemNativeEmitJzToFixed(pReNative, off, off);
3349	off = iemNativeEmitBrk(pReNative, off, UINT32_C(0x2002));
3350	iemNativeFixupFixedJump(pReNative, offFixup, off);
3351
3352	/** @todo validate that only bits in the fElfOutput mask changed. */
3353	#endif
3354
3355	iemNativeRegClearAndMarkAsGstRegShadow(pReNative, idxReg, kIemNativeGstReg_EFlags, off);
3356	off = iemNativeEmitStoreGprToVCpuU32(pReNative, off, idxReg, RT_UOFFSETOF_DYN(VMCPUCC, cpum.GstCtx.eflags));
3357	iemNativeVarRegisterRelease(pReNative, idxVarEFlags);
3358	return off;
3359	}
3360
3361
3362
3363	/*********************************************************************************************************************************
3364	* Emitters for segment register fetches (IEM_MC_FETCH_SREG_XXX).
3365	*********************************************************************************************************************************/
3366
3367	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) \
3368	off = iemNativeEmitFetchSReg(pReNative, off, a_u16Dst, a_iSReg, sizeof(uint16_t))
3369
3370	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) \
3371	off = iemNativeEmitFetchSReg(pReNative, off, a_u32Dst, a_iSReg, sizeof(uint32_t))
3372
3373	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) \
3374	off = iemNativeEmitFetchSReg(pReNative, off, a_u64Dst, a_iSReg, sizeof(uint64_t))
3375
3376
3377	/** Emits code for IEM_MC_FETCH_SREG_U16, IEM_MC_FETCH_SREG_ZX_U32 and
3378	* IEM_MC_FETCH_SREG_ZX_U64. */
3379	DECL_INLINE_THROW(uint32_t)
3380	iemNativeEmitFetchSReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iSReg, int8_t cbVar)
3381	{
3382	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
3383	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbVar); RT_NOREF(cbVar);
3384	Assert(iSReg < X86_SREG_COUNT);
3385
3386	/*
3387	* For now, we will not create a shadow copy of a selector. The rational
3388	* is that since we do not recompile the popping and loading of segment
3389	* registers and that the the IEM_MC_FETCH_SREG_U* MCs are only used for
3390	* pushing and moving to registers, there is only a small chance that the
3391	* shadow copy will be accessed again before the register is reloaded. One
3392	* scenario would be nested called in 16-bit code, but I doubt it's worth
3393	* the extra register pressure atm.
3394	*
3395	* What we really need first, though, is to combine iemNativeRegAllocTmpForGuestReg
3396	* and iemNativeVarRegisterAcquire for a load scenario. We only got the
3397	* store scencario covered at present (r160730).
3398	*/
3399	iemNativeVarSetKindToStack(pReNative, idxDstVar);
3400	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
3401	off = iemNativeEmitLoadGprFromVCpuU16(pReNative, off, idxVarReg, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aSRegs[iSReg].Sel));
3402	iemNativeVarRegisterRelease(pReNative, idxDstVar);
3403	return off;
3404	}
3405
3406
3407
3408	/*********************************************************************************************************************************
3409	* Register references. *
3410	*********************************************************************************************************************************/
3411
3412	#define IEM_MC_REF_GREG_U8_THREADED(a_pu8Dst, a_iGRegEx) \
3413	off = iemNativeEmitRefGregU8(pReNative, off, a_pu8Dst, a_iGRegEx, false /fConst/)
3414
3415	#define IEM_MC_REF_GREG_U8_CONST_THREADED(a_pu8Dst, a_iGRegEx) \
3416	off = iemNativeEmitRefGregU8(pReNative, off, a_pu8Dst, a_iGRegEx, true /fConst/)
3417
3418	/** Handles IEM_MC_REF_GREG_U8[_CONST]. */
3419	DECL_INLINE_THROW(uint32_t)
3420	iemNativeEmitRefGregU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint8_t iGRegEx, bool fConst)
3421	{
3422	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRef);
3423	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
3424	Assert(iGRegEx < 20);
3425
3426	if (iGRegEx < 16)
3427	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_Gpr, iGRegEx & 15);
3428	else
3429	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_GprHighByte, iGRegEx & 15);
3430
3431	/* If we've delayed writing back the register value, flush it now. */
3432	off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_Gpr, iGRegEx & 15);
3433
3434	/* If it's not a const reference we need to flush the shadow copy of the register now. */
3435	if (!fConst)
3436	iemNativeRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTREG_GPR(iGRegEx & 15)));
3437
3438	return off;
3439	}
3440
3441	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) \
3442	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu16Dst, a_iGReg, false /fConst/)
3443
3444	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) \
3445	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu16Dst, a_iGReg, true /fConst/)
3446
3447	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) \
3448	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu32Dst, a_iGReg, false /fConst/)
3449
3450	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) \
3451	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu32Dst, a_iGReg, true /fConst/)
3452
3453	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) \
3454	off = iemNativeEmitRefGregUxx(pReNative, off, a_pi32Dst, a_iGReg, false /fConst/)
3455
3456	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) \
3457	off = iemNativeEmitRefGregUxx(pReNative, off, a_pi32Dst, a_iGReg, true /fConst/)
3458
3459	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) \
3460	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu64Dst, a_iGReg, false /fConst/)
3461
3462	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) \
3463	off = iemNativeEmitRefGregUxx(pReNative, off, a_pu64Dst, a_iGReg, true /fConst/)
3464
3465	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) \
3466	off = iemNativeEmitRefGregUxx(pReNative, off, a_pi64Dst, a_iGReg, false /fConst/)
3467
3468	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) \
3469	off = iemNativeEmitRefGregUxx(pReNative, off, a_pi64Dst, a_iGReg, true /fConst/)
3470
3471	/** Handles IEM_MC_REF_GREG_Uxx[_CONST] and IEM_MC_REF_GREG_Ixx[_CONST]. */
3472	DECL_INLINE_THROW(uint32_t)
3473	iemNativeEmitRefGregUxx(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint8_t iGReg, bool fConst)
3474	{
3475	Assert(iGReg < 16);
3476	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_Gpr, iGReg);
3477	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
3478
3479	/* If we've delayed writing back the register value, flush it now. */
3480	off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_Gpr, iGReg);
3481
3482	/* If it's not a const reference we need to flush the shadow copy of the register now. */
3483	if (!fConst)
3484	iemNativeRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTREG_GPR(iGReg)));
3485
3486	return off;
3487	}
3488
3489
3490	#undef IEM_MC_REF_EFLAGS /* should not be used. */
3491	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) \
3492	IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput); \
3493	off = iemNativeEmitRefEFlags(pReNative, off, a_pEFlags)
3494
3495	/** Handles IEM_MC_REF_EFLAGS. */
3496	DECL_INLINE_THROW(uint32_t)
3497	iemNativeEmitRefEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef)
3498	{
3499	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_EFlags, 0);
3500	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
3501
3502	/* If we've delayed writing back the register value, flush it now. */
3503	off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_EFlags, 0);
3504
3505	/* If there is a shadow copy of guest EFLAGS, flush it now. */
3506	iemNativeRegFlushGuestShadows(pReNative, RT_BIT_64(kIemNativeGstReg_EFlags));
3507
3508	return off;
3509	}
3510
3511
3512	/** @todo Emit code for IEM_MC_ASSERT_EFLAGS in strict builds? Once we emit
3513	* different code from threaded recompiler, maybe it would be helpful. For now
3514	* we assume the threaded recompiler catches any incorrect EFLAGS delcarations. */
3515	#define IEM_MC_ASSERT_EFLAGS(a_fEflInput, a_fEflOutput) ((void)0)
3516
3517
3518	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
3519	off = iemNativeEmitRefXregXxx(pReNative, off, a_pu128Dst, a_iXReg, false /fConst/)
3520
3521	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
3522	off = iemNativeEmitRefXregXxx(pReNative, off, a_pu128Dst, a_iXReg, true /fConst/)
3523
3524	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
3525	off = iemNativeEmitRefXregXxx(pReNative, off, a_pXmmDst, a_iXReg, true /fConst/)
3526
3527	/** Handles IEM_MC_REF_XREG_xxx[_CONST]. */
3528	DECL_INLINE_THROW(uint32_t)
3529	iemNativeEmitRefXregXxx(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint8_t iXReg, bool fConst)
3530	{
3531	Assert(iXReg < 16);
3532	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_XReg, iXReg);
3533	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
3534
3535	/* If we've delayed writing back the register value, flush it now. */
3536	off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_XReg, iXReg);
3537
3538	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
3539	/* If it's not a const reference we need to flush the shadow copy of the register now. */
3540	if (!fConst)
3541	iemNativeSimdRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTSIMDREG_SIMD(iXReg)));
3542	#else
3543	RT_NOREF(fConst);
3544	#endif
3545
3546	return off;
3547	}
3548
3549
3550	#define IEM_MC_REF_MXCSR(a_pfMxcsr) \
3551	off = iemNativeEmitRefMxcsr(pReNative, off, a_pfMxcsr)
3552
3553	/** Handles IEM_MC_REF_MXCSR. */
3554	DECL_INLINE_THROW(uint32_t)
3555	iemNativeEmitRefMxcsr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef)
3556	{
3557	iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_MxCsr, 0);
3558	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
3559
3560	/* If we've delayed writing back the register value, flush it now. */
3561	off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_MxCsr, 0);
3562
3563	/* If there is a shadow copy of guest MXCSR, flush it now. */
3564	iemNativeRegFlushGuestShadows(pReNative, RT_BIT_64(kIemNativeGstReg_MxCsr));
3565
3566	return off;
3567	}
3568
3569
3570
3571	/*********************************************************************************************************************************
3572	* Effective Address Calculation *
3573	*********************************************************************************************************************************/
3574	#define IEM_MC_CALC_RM_EFF_ADDR_THREADED_16(a_GCPtrEff, a_bRm, a_u16Disp) \
3575	off = iemNativeEmitCalcRmEffAddrThreadedAddr16(pReNative, off, a_bRm, a_u16Disp, a_GCPtrEff)
3576
3577	/** Emit code for IEM_MC_CALC_RM_EFF_ADDR_THREADED_16.
3578	* @sa iemOpHlpCalcRmEffAddrThreadedAddr16 */
3579	DECL_INLINE_THROW(uint32_t)
3580	iemNativeEmitCalcRmEffAddrThreadedAddr16(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3581	uint8_t bRm, uint16_t u16Disp, uint8_t idxVarRet)
3582	{
3583	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRet);
3584
3585	/*
3586	* Handle the disp16 form with no registers first.
3587	*
3588	* Convert to an immediate value, as that'll delay the register allocation
3589	* and assignment till the memory access / call / whatever and we can use
3590	* a more appropriate register (or none at all).
3591	*/
3592	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
3593	{
3594	iemNativeVarSetKindToConst(pReNative, idxVarRet, u16Disp);
3595	return off;
3596	}
3597
3598	/* Determin the displacment. */
3599	uint16_t u16EffAddr;
3600	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
3601	{
3602	case 0: u16EffAddr = 0; break;
3603	case 1: u16EffAddr = (int16_t)(int8_t)u16Disp; break;
3604	case 2: u16EffAddr = u16Disp; break;
3605	default: AssertFailedStmt(u16EffAddr = 0);
3606	}
3607
3608	/* Determine the registers involved. */
3609	uint8_t idxGstRegBase;
3610	uint8_t idxGstRegIndex;
3611	switch (bRm & X86_MODRM_RM_MASK)
3612	{
3613	case 0:
3614	idxGstRegBase = X86_GREG_xBX;
3615	idxGstRegIndex = X86_GREG_xSI;
3616	break;
3617	case 1:
3618	idxGstRegBase = X86_GREG_xBX;
3619	idxGstRegIndex = X86_GREG_xDI;
3620	break;
3621	case 2:
3622	idxGstRegBase = X86_GREG_xBP;
3623	idxGstRegIndex = X86_GREG_xSI;
3624	break;
3625	case 3:
3626	idxGstRegBase = X86_GREG_xBP;
3627	idxGstRegIndex = X86_GREG_xDI;
3628	break;
3629	case 4:
3630	idxGstRegBase = X86_GREG_xSI;
3631	idxGstRegIndex = UINT8_MAX;
3632	break;
3633	case 5:
3634	idxGstRegBase = X86_GREG_xDI;
3635	idxGstRegIndex = UINT8_MAX;
3636	break;
3637	case 6:
3638	idxGstRegBase = X86_GREG_xBP;
3639	idxGstRegIndex = UINT8_MAX;
3640	break;
3641	#ifdef _MSC_VER /* lazy compiler, thinks idxGstRegBase and idxGstRegIndex may otherwise be used uninitialized. */
3642	default:
3643	#endif
3644	case 7:
3645	idxGstRegBase = X86_GREG_xBX;
3646	idxGstRegIndex = UINT8_MAX;
3647	break;
3648	}
3649
3650	/*
3651	* Now emit code that calculates: idxRegRet = (uint16_t)(u16EffAddr + idxGstRegBase [+ idxGstRegIndex])
3652	*/
3653	uint8_t const idxRegRet = iemNativeVarRegisterAcquire(pReNative, idxVarRet, &off);
3654	uint8_t const idxRegBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegBase),
3655	kIemNativeGstRegUse_ReadOnly);
3656	uint8_t const idxRegIndex = idxGstRegIndex != UINT8_MAX
3657	? iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegIndex),
3658	kIemNativeGstRegUse_ReadOnly)
3659	: UINT8_MAX;
3660	#ifdef RT_ARCH_AMD64
3661	if (idxRegIndex == UINT8_MAX)
3662	{
3663	if (u16EffAddr == 0)
3664	{
3665	/* movxz ret, base */
3666	off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxRegRet, idxRegBase);
3667	}
3668	else
3669	{
3670	/* lea ret32, [base64 + disp32] */
3671	Assert(idxRegBase != X86_GREG_xSP /SIB/);
3672	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
3673	if (idxRegRet >= 8 \|\| idxRegBase >= 8)
3674	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0) \| (idxRegBase >= 8 ? X86_OP_REX_B : 0);
3675	pbCodeBuf[off++] = 0x8d;
3676	if (idxRegBase != X86_GREG_x12 /SIB/)
3677	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxRegRet & 7, idxRegBase & 7);
3678	else
3679	{
3680	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxRegRet & 7, 4 /SIB/);
3681	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_x12 & 7, 4 /no index/, 0);
3682	}
3683	pbCodeBuf[off++] = RT_BYTE1(u16EffAddr);
3684	pbCodeBuf[off++] = RT_BYTE2(u16EffAddr);
3685	pbCodeBuf[off++] = 0;
3686	pbCodeBuf[off++] = 0;
3687	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3688
3689	off = iemNativeEmitClear16UpGpr(pReNative, off, idxRegRet);
3690	}
3691	}
3692	else
3693	{
3694	/* lea ret32, [index64 + base64 (+ disp32)] */
3695	Assert(idxRegIndex != X86_GREG_xSP /no-index/);
3696	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
3697	if (idxRegRet >= 8 \|\| idxRegBase >= 8 \|\| idxRegIndex >= 8)
3698	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
3699	\| (idxRegBase >= 8 ? X86_OP_REX_B : 0)
3700	\| (idxRegIndex >= 8 ? X86_OP_REX_X : 0);
3701	pbCodeBuf[off++] = 0x8d;
3702	uint8_t const bMod = u16EffAddr == 0 && (idxRegBase & 7) != X86_GREG_xBP ? X86_MOD_MEM0 : X86_MOD_MEM4;
3703	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, 4 /SIB/);
3704	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegBase & 7, idxRegIndex & 7, 0);
3705	if (bMod == X86_MOD_MEM4)
3706	{
3707	pbCodeBuf[off++] = RT_BYTE1(u16EffAddr);
3708	pbCodeBuf[off++] = RT_BYTE2(u16EffAddr);
3709	pbCodeBuf[off++] = 0;
3710	pbCodeBuf[off++] = 0;
3711	}
3712	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3713	off = iemNativeEmitClear16UpGpr(pReNative, off, idxRegRet);
3714	}
3715
3716	#elif defined(RT_ARCH_ARM64)
3717	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3718	if (u16EffAddr == 0)
3719	{
3720	if (idxRegIndex == UINT8_MAX)
3721	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(idxRegRet, idxRegBase);
3722	else
3723	{
3724	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegBase, idxRegIndex, false /f64Bit/);
3725	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(idxRegRet, idxRegRet);
3726	}
3727	}
3728	else
3729	{
3730	if ((int16_t)u16EffAddr < 4096 && (int16_t)u16EffAddr >= 0)
3731	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegRet, idxRegBase, u16EffAddr, false /f64Bit/);
3732	else if ((int16_t)u16EffAddr > -4096 && (int16_t)u16EffAddr < 0)
3733	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, idxRegRet, idxRegBase,
3734	(uint16_t)-(int16_t)u16EffAddr, false /f64Bit/);
3735	else
3736	{
3737	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegRet, u16EffAddr);
3738	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegBase, false /f64Bit/);
3739	}
3740	if (idxRegIndex != UINT8_MAX)
3741	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegIndex, false /f64Bit/);
3742	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(idxRegRet, idxRegRet);
3743	}
3744
3745	#else
3746	# error "port me"
3747	#endif
3748
3749	if (idxRegIndex != UINT8_MAX)
3750	iemNativeRegFreeTmp(pReNative, idxRegIndex);
3751	iemNativeRegFreeTmp(pReNative, idxRegBase);
3752	iemNativeVarRegisterRelease(pReNative, idxVarRet);
3753	return off;
3754	}
3755
3756
3757	#define IEM_MC_CALC_RM_EFF_ADDR_THREADED_32(a_GCPtrEff, a_bRm, a_uSibAndRspOffset, a_u32Disp) \
3758	off = iemNativeEmitCalcRmEffAddrThreadedAddr32(pReNative, off, a_bRm, a_uSibAndRspOffset, a_u32Disp, a_GCPtrEff)
3759
3760	/** Emit code for IEM_MC_CALC_RM_EFF_ADDR_THREADED_32.
3761	* @see iemOpHlpCalcRmEffAddrThreadedAddr32 */
3762	DECL_INLINE_THROW(uint32_t)
3763	iemNativeEmitCalcRmEffAddrThreadedAddr32(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3764	uint8_t bRm, uint32_t uSibAndRspOffset, uint32_t u32Disp, uint8_t idxVarRet)
3765	{
3766	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRet);
3767
3768	/*
3769	* Handle the disp32 form with no registers first.
3770	*
3771	* Convert to an immediate value, as that'll delay the register allocation
3772	* and assignment till the memory access / call / whatever and we can use
3773	* a more appropriate register (or none at all).
3774	*/
3775	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
3776	{
3777	iemNativeVarSetKindToConst(pReNative, idxVarRet, u32Disp);
3778	return off;
3779	}
3780
3781	/* Calculate the fixed displacement (more down in SIB.B=4 and SIB.B=5 on this). */
3782	uint32_t u32EffAddr = 0;
3783	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
3784	{
3785	case 0: break;
3786	case 1: u32EffAddr = (int8_t)u32Disp; break;
3787	case 2: u32EffAddr = u32Disp; break;
3788	default: AssertFailed();
3789	}
3790
3791	/* Get the register (or SIB) value. */
3792	uint8_t idxGstRegBase = UINT8_MAX;
3793	uint8_t idxGstRegIndex = UINT8_MAX;
3794	uint8_t cShiftIndex = 0;
3795	switch (bRm & X86_MODRM_RM_MASK)
3796	{
3797	case 0: idxGstRegBase = X86_GREG_xAX; break;
3798	case 1: idxGstRegBase = X86_GREG_xCX; break;
3799	case 2: idxGstRegBase = X86_GREG_xDX; break;
3800	case 3: idxGstRegBase = X86_GREG_xBX; break;
3801	case 4: /* SIB */
3802	{
3803	/* index /w scaling . */
3804	cShiftIndex = (uSibAndRspOffset >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
3805	switch ((uSibAndRspOffset >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
3806	{
3807	case 0: idxGstRegIndex = X86_GREG_xAX; break;
3808	case 1: idxGstRegIndex = X86_GREG_xCX; break;
3809	case 2: idxGstRegIndex = X86_GREG_xDX; break;
3810	case 3: idxGstRegIndex = X86_GREG_xBX; break;
3811	case 4: cShiftIndex = 0; /no index/ break;
3812	case 5: idxGstRegIndex = X86_GREG_xBP; break;
3813	case 6: idxGstRegIndex = X86_GREG_xSI; break;
3814	case 7: idxGstRegIndex = X86_GREG_xDI; break;
3815	}
3816
3817	/* base */
3818	switch (uSibAndRspOffset & X86_SIB_BASE_MASK)
3819	{
3820	case 0: idxGstRegBase = X86_GREG_xAX; break;
3821	case 1: idxGstRegBase = X86_GREG_xCX; break;
3822	case 2: idxGstRegBase = X86_GREG_xDX; break;
3823	case 3: idxGstRegBase = X86_GREG_xBX; break;
3824	case 4:
3825	idxGstRegBase = X86_GREG_xSP;
3826	u32EffAddr += uSibAndRspOffset >> 8;
3827	break;
3828	case 5:
3829	if ((bRm & X86_MODRM_MOD_MASK) != 0)
3830	idxGstRegBase = X86_GREG_xBP;
3831	else
3832	{
3833	Assert(u32EffAddr == 0);
3834	u32EffAddr = u32Disp;
3835	}
3836	break;
3837	case 6: idxGstRegBase = X86_GREG_xSI; break;
3838	case 7: idxGstRegBase = X86_GREG_xDI; break;
3839	}
3840	break;
3841	}
3842	case 5: idxGstRegBase = X86_GREG_xBP; break;
3843	case 6: idxGstRegBase = X86_GREG_xSI; break;
3844	case 7: idxGstRegBase = X86_GREG_xDI; break;
3845	}
3846
3847	/*
3848	* If no registers are involved (SIB.B=5, SIB.X=4) repeat what we did at
3849	* the start of the function.
3850	*/
3851	if (idxGstRegBase == UINT8_MAX && idxGstRegIndex == UINT8_MAX)
3852	{
3853	iemNativeVarSetKindToConst(pReNative, idxVarRet, u32EffAddr);
3854	return off;
3855	}
3856
3857	/*
3858	* Now emit code that calculates: idxRegRet = (uint32_t)(u32EffAddr [+ idxGstRegBase] [+ (idxGstRegIndex << cShiftIndex)])
3859	*/
3860	uint8_t const idxRegRet = iemNativeVarRegisterAcquire(pReNative, idxVarRet, &off);
3861	uint8_t idxRegBase = idxGstRegBase == UINT8_MAX ? UINT8_MAX
3862	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegBase),
3863	kIemNativeGstRegUse_ReadOnly);
3864	uint8_t idxRegIndex = idxGstRegIndex == UINT8_MAX ? UINT8_MAX
3865	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegIndex),
3866	kIemNativeGstRegUse_ReadOnly);
3867
3868	/* If base is not given and there is no shifting, swap the registers to avoid code duplication. */
3869	if (idxRegBase == UINT8_MAX && cShiftIndex == 0)
3870	{
3871	idxRegBase = idxRegIndex;
3872	idxRegIndex = UINT8_MAX;
3873	}
3874
3875	#ifdef RT_ARCH_AMD64
3876	if (idxRegIndex == UINT8_MAX)
3877	{
3878	if (u32EffAddr == 0)
3879	{
3880	/* mov ret, base */
3881	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRet, idxRegBase);
3882	}
3883	else
3884	{
3885	/* lea ret32, [base64 + disp32] */
3886	Assert(idxRegBase != X86_GREG_xSP /SIB/);
3887	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
3888	if (idxRegRet >= 8 \|\| idxRegBase >= 8)
3889	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0) \| (idxRegBase >= 8 ? X86_OP_REX_B : 0);
3890	pbCodeBuf[off++] = 0x8d;
3891	uint8_t const bMod = (int8_t)u32EffAddr == (int32_t)u32EffAddr ? X86_MOD_MEM1 : X86_MOD_MEM4;
3892	if (idxRegBase != X86_GREG_x12 /SIB/)
3893	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, idxRegBase & 7);
3894	else
3895	{
3896	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, 4 /SIB/);
3897	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_x12 & 7, 4 /no index/, 0);
3898	}
3899	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
3900	if (bMod == X86_MOD_MEM4)
3901	{
3902	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
3903	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
3904	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
3905	}
3906	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3907	}
3908	}
3909	else
3910	{
3911	Assert(idxRegIndex != X86_GREG_xSP /no-index/);
3912	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
3913	if (idxRegBase == UINT8_MAX)
3914	{
3915	/* lea ret32, [(index64 << cShiftIndex) + disp32] */
3916	if (idxRegRet >= 8 \|\| idxRegIndex >= 8)
3917	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
3918	\| (idxRegIndex >= 8 ? X86_OP_REX_X : 0);
3919	pbCodeBuf[off++] = 0x8d;
3920	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, idxRegRet & 7, 4 /SIB/);
3921	pbCodeBuf[off++] = X86_SIB_MAKE(5 /nobase/bp/, idxRegIndex & 7, cShiftIndex);
3922	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
3923	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
3924	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
3925	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
3926	}
3927	else
3928	{
3929	/* lea ret32, [(index64 << cShiftIndex) + base64 (+ disp32)] */
3930	if (idxRegRet >= 8 \|\| idxRegBase >= 8 \|\| idxRegIndex >= 8)
3931	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
3932	\| (idxRegBase >= 8 ? X86_OP_REX_B : 0)
3933	\| (idxRegIndex >= 8 ? X86_OP_REX_X : 0);
3934	pbCodeBuf[off++] = 0x8d;
3935	uint8_t const bMod = u32EffAddr == 0 && (idxRegBase & 7) != X86_GREG_xBP ? X86_MOD_MEM0
3936	: (int8_t)u32EffAddr == (int32_t)u32EffAddr ? X86_MOD_MEM1 : X86_MOD_MEM4;
3937	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, 4 /SIB/);
3938	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegBase & 7, idxRegIndex & 7, cShiftIndex);
3939	if (bMod != X86_MOD_MEM0)
3940	{
3941	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
3942	if (bMod == X86_MOD_MEM4)
3943	{
3944	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
3945	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
3946	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
3947	}
3948	}
3949	}
3950	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3951	}
3952
3953	#elif defined(RT_ARCH_ARM64)
3954	if (u32EffAddr == 0)
3955	{
3956	if (idxRegIndex == UINT8_MAX)
3957	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRet, idxRegBase);
3958	else if (idxRegBase == UINT8_MAX)
3959	{
3960	if (cShiftIndex == 0)
3961	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRet, idxRegIndex);
3962	else
3963	{
3964	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3965	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(idxRegRet, idxRegIndex, cShiftIndex, false /f64Bit/);
3966	}
3967	}
3968	else
3969	{
3970	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3971	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegBase, idxRegIndex,
3972	false /f64Bit/, false /fSetFlags/, cShiftIndex);
3973	}
3974	}
3975	else
3976	{
3977	if ((int32_t)u32EffAddr < 4096 && (int32_t)u32EffAddr >= 0 && idxRegBase != UINT8_MAX)
3978	{
3979	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3980	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegRet, idxRegBase, u32EffAddr, false /f64Bit/);
3981	}
3982	else if ((int32_t)u32EffAddr > -4096 && (int32_t)u32EffAddr < 0 && idxRegBase != UINT8_MAX)
3983	{
3984	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3985	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, idxRegRet, idxRegBase,
3986	(uint32_t)-(int32_t)u32EffAddr, false /f64Bit/);
3987	}
3988	else
3989	{
3990	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegRet, u32EffAddr);
3991	if (idxRegBase != UINT8_MAX)
3992	{
3993	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3994	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegBase, false /f64Bit/);
3995	}
3996	}
3997	if (idxRegIndex != UINT8_MAX)
3998	{
3999	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4000	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegIndex,
4001	false /f64Bit/, false /fSetFlags/, cShiftIndex);
4002	}
4003	}
4004
4005	#else
4006	# error "port me"
4007	#endif
4008
4009	if (idxRegIndex != UINT8_MAX)
4010	iemNativeRegFreeTmp(pReNative, idxRegIndex);
4011	if (idxRegBase != UINT8_MAX)
4012	iemNativeRegFreeTmp(pReNative, idxRegBase);
4013	iemNativeVarRegisterRelease(pReNative, idxVarRet);
4014	return off;
4015	}
4016
4017
4018	#define IEM_MC_CALC_RM_EFF_ADDR_THREADED_64(a_GCPtrEff, a_bRmEx, a_uSibAndRspOffset, a_u32Disp, a_cbImm) \
4019	off = iemNativeEmitCalcRmEffAddrThreadedAddr64(pReNative, off, a_bRmEx, a_uSibAndRspOffset, \
4020	a_u32Disp, a_cbImm, a_GCPtrEff, true /f64Bit/)
4021
4022	#define IEM_MC_CALC_RM_EFF_ADDR_THREADED_64_FSGS(a_GCPtrEff, a_bRmEx, a_uSibAndRspOffset, a_u32Disp, a_cbImm) \
4023	off = iemNativeEmitCalcRmEffAddrThreadedAddr64(pReNative, off, a_bRmEx, a_uSibAndRspOffset, \
4024	a_u32Disp, a_cbImm, a_GCPtrEff, true /f64Bit/)
4025
4026	#define IEM_MC_CALC_RM_EFF_ADDR_THREADED_64_ADDR32(a_GCPtrEff, a_bRmEx, a_uSibAndRspOffset, a_u32Disp, a_cbImm) \
4027	off = iemNativeEmitCalcRmEffAddrThreadedAddr64(pReNative, off, a_bRmEx, a_uSibAndRspOffset, \
4028	a_u32Disp, a_cbImm, a_GCPtrEff, false /f64Bit/)
4029
4030	/**
4031	* Emit code for IEM_MC_CALC_RM_EFF_ADDR_THREADED_64*.
4032	*
4033	* @returns New off.
4034	* @param pReNative .
4035	* @param off .
4036	* @param bRmEx The ModRM byte but with bit 3 set to REX.B and
4037	* bit 4 to REX.X. The two bits are part of the
4038	* REG sub-field, which isn't needed in this
4039	* function.
4040	* @param uSibAndRspOffset Two parts:
4041	* - The first 8 bits make up the SIB byte.
4042	* - The next 8 bits are the fixed RSP/ESP offset
4043	* in case of a pop [xSP].
4044	* @param u32Disp The displacement byte/word/dword, if any.
4045	* @param cbInstr The size of the fully decoded instruction. Used
4046	* for RIP relative addressing.
4047	* @param idxVarRet The result variable number.
4048	* @param f64Bit Whether to use a 64-bit or 32-bit address size
4049	* when calculating the address.
4050	*
4051	* @see iemOpHlpCalcRmEffAddrThreadedAddr64
4052	*/
4053	DECL_INLINE_THROW(uint32_t)
4054	iemNativeEmitCalcRmEffAddrThreadedAddr64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bRmEx, uint32_t uSibAndRspOffset,
4055	uint32_t u32Disp, uint8_t cbInstr, uint8_t idxVarRet, bool f64Bit)
4056	{
4057	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRet);
4058
4059	/*
4060	* Special case the rip + disp32 form first.
4061	*/
4062	if ((bRmEx & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
4063	{
4064	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
4065	/* Need to take the current PC offset into account for the displacement, no need to flush here
4066	* as the PC is only accessed readonly and there is no branching or calling helpers involved. */
4067	u32Disp += pReNative->Core.offPc;
4068	#endif
4069
4070	uint8_t const idxRegRet = iemNativeVarRegisterAcquire(pReNative, idxVarRet, &off);
4071	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
4072	kIemNativeGstRegUse_ReadOnly);
4073	#ifdef RT_ARCH_AMD64
4074	if (f64Bit)
4075	{
4076	int64_t const offFinalDisp = (int64_t)(int32_t)u32Disp + cbInstr;
4077	if ((int32_t)offFinalDisp == offFinalDisp)
4078	off = iemNativeEmitLoadGprFromGprWithAddendMaybeZero(pReNative, off, idxRegRet, idxRegPc, (int32_t)offFinalDisp);
4079	else
4080	{
4081	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegRet, idxRegPc, (int32_t)u32Disp);
4082	off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRet, cbInstr);
4083	}
4084	}
4085	else
4086	off = iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(pReNative, off, idxRegRet, idxRegPc, (int32_t)u32Disp + cbInstr);
4087
4088	#elif defined(RT_ARCH_ARM64)
4089	if (f64Bit)
4090	off = iemNativeEmitLoadGprFromGprWithAddendMaybeZero(pReNative, off, idxRegRet, idxRegPc,
4091	(int64_t)(int32_t)u32Disp + cbInstr);
4092	else
4093	off = iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(pReNative, off, idxRegRet, idxRegPc,
4094	(int32_t)u32Disp + cbInstr);
4095
4096	#else
4097	# error "Port me!"
4098	#endif
4099	iemNativeRegFreeTmp(pReNative, idxRegPc);
4100	iemNativeVarRegisterRelease(pReNative, idxVarRet);
4101	return off;
4102	}
4103
4104	/* Calculate the fixed displacement (more down in SIB.B=4 and SIB.B=5 on this). */
4105	int64_t i64EffAddr = 0;
4106	switch ((bRmEx >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
4107	{
4108	case 0: break;
4109	case 1: i64EffAddr = (int8_t)u32Disp; break;
4110	case 2: i64EffAddr = (int32_t)u32Disp; break;
4111	default: AssertFailed();
4112	}
4113
4114	/* Get the register (or SIB) value. */
4115	uint8_t idxGstRegBase = UINT8_MAX;
4116	uint8_t idxGstRegIndex = UINT8_MAX;
4117	uint8_t cShiftIndex = 0;
4118	if ((bRmEx & X86_MODRM_RM_MASK) != 4)
4119	idxGstRegBase = bRmEx & (X86_MODRM_RM_MASK \| 0x8); /* bRmEx[bit 3] = REX.B */
4120	else /* SIB: */
4121	{
4122	/* index /w scaling . */
4123	cShiftIndex = (uSibAndRspOffset >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
4124	idxGstRegIndex = ((uSibAndRspOffset >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
4125	\| ((bRmEx & 0x10) >> 1); /* bRmEx[bit 4] = REX.X */
4126	if (idxGstRegIndex == 4)
4127	{
4128	/* no index */
4129	cShiftIndex = 0;
4130	idxGstRegIndex = UINT8_MAX;
4131	}
4132
4133	/* base */
4134	idxGstRegBase = (uSibAndRspOffset & X86_SIB_BASE_MASK) \| (bRmEx & 0x8); /* bRmEx[bit 3] = REX.B */
4135	if (idxGstRegBase == 4)
4136	{
4137	/* pop [rsp] hack */
4138	i64EffAddr += uSibAndRspOffset >> 8; /* (this is why i64EffAddr must be 64-bit) */
4139	}
4140	else if ( (idxGstRegBase & X86_SIB_BASE_MASK) == 5
4141	&& (bRmEx & X86_MODRM_MOD_MASK) == 0)
4142	{
4143	/* mod=0 and base=5 -> disp32, no base reg. */
4144	Assert(i64EffAddr == 0);
4145	i64EffAddr = (int32_t)u32Disp;
4146	idxGstRegBase = UINT8_MAX;
4147	}
4148	}
4149
4150	/*
4151	* If no registers are involved (SIB.B=5, SIB.X=4) repeat what we did at
4152	* the start of the function.
4153	*/
4154	if (idxGstRegBase == UINT8_MAX && idxGstRegIndex == UINT8_MAX)
4155	{
4156	if (f64Bit)
4157	iemNativeVarSetKindToConst(pReNative, idxVarRet, (uint64_t)i64EffAddr);
4158	else
4159	iemNativeVarSetKindToConst(pReNative, idxVarRet, (uint32_t)i64EffAddr);
4160	return off;
4161	}
4162
4163	/*
4164	* Now emit code that calculates:
4165	* idxRegRet = (uint64_t)(i64EffAddr [+ idxGstRegBase] [+ (idxGstRegIndex << cShiftIndex)])
4166	* or if !f64Bit:
4167	* idxRegRet = (uint32_t)(i64EffAddr [+ idxGstRegBase] [+ (idxGstRegIndex << cShiftIndex)])
4168	*/
4169	uint8_t const idxRegRet = iemNativeVarRegisterAcquire(pReNative, idxVarRet, &off);
4170	uint8_t idxRegBase = idxGstRegBase == UINT8_MAX ? UINT8_MAX
4171	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegBase),
4172	kIemNativeGstRegUse_ReadOnly);
4173	uint8_t idxRegIndex = idxGstRegIndex == UINT8_MAX ? UINT8_MAX
4174	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGstRegIndex),
4175	kIemNativeGstRegUse_ReadOnly);
4176
4177	/* If base is not given and there is no shifting, swap the registers to avoid code duplication. */
4178	if (idxRegBase == UINT8_MAX && cShiftIndex == 0)
4179	{
4180	idxRegBase = idxRegIndex;
4181	idxRegIndex = UINT8_MAX;
4182	}
4183
4184	#ifdef RT_ARCH_AMD64
4185	uint8_t bFinalAdj;
4186	if (!f64Bit \|\| (int32_t)i64EffAddr == i64EffAddr)
4187	bFinalAdj = 0; /* likely */
4188	else
4189	{
4190	/* pop [rsp] with a problematic disp32 value. Split out the
4191	RSP offset and add it separately afterwards (bFinalAdj). */
4192	/** @todo testcase: pop [rsp] with problematic disp32 (mod4). */
4193	Assert(idxGstRegBase == X86_GREG_xSP);
4194	Assert(((bRmEx >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) == X86_MOD_MEM4);
4195	bFinalAdj = (uint8_t)(uSibAndRspOffset >> 8);
4196	Assert(bFinalAdj != 0);
4197	i64EffAddr -= bFinalAdj;
4198	Assert((int32_t)i64EffAddr == i64EffAddr);
4199	}
4200	uint32_t const u32EffAddr = (uint32_t)i64EffAddr;
4201	//pReNative->pInstrBuf[off++] = 0xcc;
4202
4203	if (idxRegIndex == UINT8_MAX)
4204	{
4205	if (u32EffAddr == 0)
4206	{
4207	/* mov ret, base */
4208	if (f64Bit)
4209	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegRet, idxRegBase);
4210	else
4211	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRet, idxRegBase);
4212	}
4213	else
4214	{
4215	/* lea ret, [base + disp32] */
4216	Assert(idxRegBase != X86_GREG_xSP /SIB/);
4217	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
4218	if (f64Bit \|\| idxRegRet >= 8 \|\| idxRegBase >= 8)
4219	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
4220	\| (idxRegBase >= 8 ? X86_OP_REX_B : 0)
4221	\| (f64Bit ? X86_OP_REX_W : 0);
4222	pbCodeBuf[off++] = 0x8d;
4223	uint8_t const bMod = (int8_t)u32EffAddr == (int32_t)u32EffAddr ? X86_MOD_MEM1 : X86_MOD_MEM4;
4224	if (idxRegBase != X86_GREG_x12 /SIB/)
4225	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, idxRegBase & 7);
4226	else
4227	{
4228	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, 4 /SIB/);
4229	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_x12 & 7, 4 /no index/, 0);
4230	}
4231	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
4232	if (bMod == X86_MOD_MEM4)
4233	{
4234	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
4235	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
4236	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
4237	}
4238	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4239	}
4240	}
4241	else
4242	{
4243	Assert(idxRegIndex != X86_GREG_xSP /no-index/);
4244	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
4245	if (idxRegBase == UINT8_MAX)
4246	{
4247	/* lea ret, [(index64 << cShiftIndex) + disp32] */
4248	if (f64Bit \|\| idxRegRet >= 8 \|\| idxRegIndex >= 8)
4249	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
4250	\| (idxRegIndex >= 8 ? X86_OP_REX_X : 0)
4251	\| (f64Bit ? X86_OP_REX_W : 0);
4252	pbCodeBuf[off++] = 0x8d;
4253	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, idxRegRet & 7, 4 /SIB/);
4254	pbCodeBuf[off++] = X86_SIB_MAKE(5 /nobase/bp/, idxRegIndex & 7, cShiftIndex);
4255	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
4256	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
4257	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
4258	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
4259	}
4260	else
4261	{
4262	/* lea ret, [(index64 << cShiftIndex) + base64 (+ disp32)] */
4263	if (f64Bit \|\| idxRegRet >= 8 \|\| idxRegBase >= 8 \|\| idxRegIndex >= 8)
4264	pbCodeBuf[off++] = (idxRegRet >= 8 ? X86_OP_REX_R : 0)
4265	\| (idxRegBase >= 8 ? X86_OP_REX_B : 0)
4266	\| (idxRegIndex >= 8 ? X86_OP_REX_X : 0)
4267	\| (f64Bit ? X86_OP_REX_W : 0);
4268	pbCodeBuf[off++] = 0x8d;
4269	uint8_t const bMod = u32EffAddr == 0 && (idxRegBase & 7) != X86_GREG_xBP ? X86_MOD_MEM0
4270	: (int8_t)u32EffAddr == (int32_t)u32EffAddr ? X86_MOD_MEM1 : X86_MOD_MEM4;
4271	pbCodeBuf[off++] = X86_MODRM_MAKE(bMod, idxRegRet & 7, 4 /SIB/);
4272	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegBase & 7, idxRegIndex & 7, cShiftIndex);
4273	if (bMod != X86_MOD_MEM0)
4274	{
4275	pbCodeBuf[off++] = RT_BYTE1(u32EffAddr);
4276	if (bMod == X86_MOD_MEM4)
4277	{
4278	pbCodeBuf[off++] = RT_BYTE2(u32EffAddr);
4279	pbCodeBuf[off++] = RT_BYTE3(u32EffAddr);
4280	pbCodeBuf[off++] = RT_BYTE4(u32EffAddr);
4281	}
4282	}
4283	}
4284	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4285	}
4286
4287	if (!bFinalAdj)
4288	{ /* likely */ }
4289	else
4290	{
4291	Assert(f64Bit);
4292	off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRet, bFinalAdj);
4293	}
4294
4295	#elif defined(RT_ARCH_ARM64)
4296	if (i64EffAddr == 0)
4297	{
4298	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4299	if (idxRegIndex == UINT8_MAX)
4300	pu32CodeBuf[off++] = Armv8A64MkInstrMov(idxRegRet, idxRegBase, f64Bit);
4301	else if (idxRegBase != UINT8_MAX)
4302	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegBase, idxRegIndex,
4303	f64Bit, false /fSetFlags/, cShiftIndex);
4304	else
4305	{
4306	Assert(cShiftIndex != 0); /* See base = index swap above when shift is 0 and we have no base reg. */
4307	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(idxRegRet, idxRegIndex, cShiftIndex, f64Bit);
4308	}
4309	}
4310	else
4311	{
4312	if (f64Bit)
4313	{ /* likely */ }
4314	else
4315	i64EffAddr = (int32_t)i64EffAddr;
4316
4317	if (i64EffAddr < 4096 && i64EffAddr >= 0 && idxRegBase != UINT8_MAX)
4318	{
4319	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4320	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegRet, idxRegBase, i64EffAddr, f64Bit);
4321	}
4322	else if (i64EffAddr > -4096 && i64EffAddr < 0 && idxRegBase != UINT8_MAX)
4323	{
4324	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4325	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, idxRegRet, idxRegBase, (uint32_t)-i64EffAddr, f64Bit);
4326	}
4327	else
4328	{
4329	if (f64Bit)
4330	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegRet, i64EffAddr);
4331	else
4332	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegRet, (uint32_t)i64EffAddr);
4333	if (idxRegBase != UINT8_MAX)
4334	{
4335	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4336	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegBase, f64Bit);
4337	}
4338	}
4339	if (idxRegIndex != UINT8_MAX)
4340	{
4341	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4342	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, idxRegRet, idxRegRet, idxRegIndex,
4343	f64Bit, false /fSetFlags/, cShiftIndex);
4344	}
4345	}
4346
4347	#else
4348	# error "port me"
4349	#endif
4350
4351	if (idxRegIndex != UINT8_MAX)
4352	iemNativeRegFreeTmp(pReNative, idxRegIndex);
4353	if (idxRegBase != UINT8_MAX)
4354	iemNativeRegFreeTmp(pReNative, idxRegBase);
4355	iemNativeVarRegisterRelease(pReNative, idxVarRet);
4356	return off;
4357	}
4358
4359
4360	/*********************************************************************************************************************************
4361	* Memory fetches and stores common *
4362	*********************************************************************************************************************************/
4363
4364	typedef enum IEMNATIVEMITMEMOP
4365	{
4366	kIemNativeEmitMemOp_Store = 0,
4367	kIemNativeEmitMemOp_Fetch,
4368	kIemNativeEmitMemOp_Fetch_Zx_U16,
4369	kIemNativeEmitMemOp_Fetch_Zx_U32,
4370	kIemNativeEmitMemOp_Fetch_Zx_U64,
4371	kIemNativeEmitMemOp_Fetch_Sx_U16,
4372	kIemNativeEmitMemOp_Fetch_Sx_U32,
4373	kIemNativeEmitMemOp_Fetch_Sx_U64
4374	} IEMNATIVEMITMEMOP;
4375
4376	/** Emits code for IEM_MC_FETCH_MEM_U8/16/32/64 and IEM_MC_STORE_MEM_U8/16/32/64,
4377	* and IEM_MC_FETCH_MEM_FLAT_U8/16/32/64 and IEM_MC_STORE_MEM_FLAT_U8/16/32/64
4378	* (with iSegReg = UINT8_MAX). */
4379	DECL_INLINE_THROW(uint32_t)
4380	iemNativeEmitMemFetchStoreDataCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarValue, uint8_t iSegReg,
4381	uint8_t idxVarGCPtrMem, uint8_t cbMem, uint8_t fAlignMask, IEMNATIVEMITMEMOP enmOp,
4382	uintptr_t pfnFunction, uint8_t idxInstr, uint8_t offDisp = 0)
4383	{
4384	/*
4385	* Assert sanity.
4386	*/
4387	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarValue);
4388	PIEMNATIVEVAR const pVarValue = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarValue)];
4389	Assert( enmOp != kIemNativeEmitMemOp_Store
4390	\|\| pVarValue->enmKind == kIemNativeVarKind_Immediate
4391	\|\| pVarValue->enmKind == kIemNativeVarKind_Stack);
4392	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarGCPtrMem);
4393	PIEMNATIVEVAR const pVarGCPtrMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarGCPtrMem)];
4394	AssertStmt( pVarGCPtrMem->enmKind == kIemNativeVarKind_Immediate
4395	\|\| pVarGCPtrMem->enmKind == kIemNativeVarKind_Stack,
4396	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
4397	Assert(iSegReg < 6 \|\| iSegReg == UINT8_MAX);
4398	Assert(cbMem == 1 \|\| cbMem == 2 \|\| cbMem == 4 \|\| cbMem == 8);
4399	AssertCompile(IEMNATIVE_CALL_ARG_GREG_COUNT >= 4);
4400	#ifdef VBOX_STRICT
4401	if (iSegReg == UINT8_MAX)
4402	{
4403	Assert( (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT
4404	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT
4405	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT);
4406	switch (cbMem)
4407	{
4408	case 1:
4409	Assert( pfnFunction
4410	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemFlatStoreDataU8
4411	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8
4412	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U16 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8
4413	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U32 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8
4414	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8
4415	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U16 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U16
4416	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U32 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U32
4417	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U64
4418	: UINT64_C(0xc000b000a0009000) ));
4419	break;
4420	case 2:
4421	Assert( pfnFunction
4422	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemFlatStoreDataU16
4423	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFlatFetchDataU16
4424	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U32 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU16
4425	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU16
4426	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U32 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU16_Sx_U32
4427	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU16_Sx_U64
4428	: UINT64_C(0xc000b000a0009000) ));
4429	break;
4430	case 4:
4431	Assert( pfnFunction
4432	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemFlatStoreDataU32
4433	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFlatFetchDataU32
4434	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU32
4435	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFlatFetchDataU32_Sx_U64
4436	: UINT64_C(0xc000b000a0009000) ));
4437	break;
4438	case 8:
4439	Assert( pfnFunction
4440	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemFlatStoreDataU64
4441	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFlatFetchDataU64
4442	: UINT64_C(0xc000b000a0009000) ));
4443	break;
4444	}
4445	}
4446	else
4447	{
4448	Assert(iSegReg < 6);
4449	switch (cbMem)
4450	{
4451	case 1:
4452	Assert( pfnFunction
4453	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemStoreDataU8
4454	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFetchDataU8
4455	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U16 ? (uintptr_t)iemNativeHlpMemFetchDataU8
4456	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U32 ? (uintptr_t)iemNativeHlpMemFetchDataU8
4457	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU8
4458	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U16 ? (uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U16
4459	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U32 ? (uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U32
4460	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U64
4461	: UINT64_C(0xc000b000a0009000) ));
4462	break;
4463	case 2:
4464	Assert( pfnFunction
4465	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemStoreDataU16
4466	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFetchDataU16
4467	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U32 ? (uintptr_t)iemNativeHlpMemFetchDataU16
4468	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU16
4469	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U32 ? (uintptr_t)iemNativeHlpMemFetchDataU16_Sx_U32
4470	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU16_Sx_U64
4471	: UINT64_C(0xc000b000a0009000) ));
4472	break;
4473	case 4:
4474	Assert( pfnFunction
4475	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemStoreDataU32
4476	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFetchDataU32
4477	: enmOp == kIemNativeEmitMemOp_Fetch_Zx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU32
4478	: enmOp == kIemNativeEmitMemOp_Fetch_Sx_U64 ? (uintptr_t)iemNativeHlpMemFetchDataU32_Sx_U64
4479	: UINT64_C(0xc000b000a0009000) ));
4480	break;
4481	case 8:
4482	Assert( pfnFunction
4483	== ( enmOp == kIemNativeEmitMemOp_Store ? (uintptr_t)iemNativeHlpMemStoreDataU64
4484	: enmOp == kIemNativeEmitMemOp_Fetch ? (uintptr_t)iemNativeHlpMemFetchDataU64
4485	: UINT64_C(0xc000b000a0009000) ));
4486	break;
4487	}
4488	}
4489	#endif
4490
4491	#ifdef VBOX_STRICT
4492	/*
4493	* Check that the fExec flags we've got make sense.
4494	*/
4495	off = iemNativeEmitExecFlagsCheck(pReNative, off, pReNative->fExec);
4496	#endif
4497
4498	/*
4499	* To keep things simple we have to commit any pending writes first as we
4500	* may end up making calls.
4501	*/
4502	/** @todo we could postpone this till we make the call and reload the
4503	* registers after returning from the call. Not sure if that's sensible or
4504	* not, though. */
4505	#ifndef IEMNATIVE_WITH_DELAYED_PC_UPDATING
4506	off = iemNativeRegFlushPendingWrites(pReNative, off);
4507	#else
4508	/* The program counter is treated differently for now. */
4509	off = iemNativeRegFlushPendingWrites(pReNative, off, RT_BIT_64(kIemNativeGstReg_Pc));
4510	#endif
4511
4512	#ifdef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4513	/*
4514	* Move/spill/flush stuff out of call-volatile registers.
4515	* This is the easy way out. We could contain this to the tlb-miss branch
4516	* by saving and restoring active stuff here.
4517	*/
4518	off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /* vacate all non-volatile regs */);
4519	#endif
4520
4521	/*
4522	* Define labels and allocate the result register (trying for the return
4523	* register if we can).
4524	*/
4525	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
4526	uint8_t const idxRegValueFetch = enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
4527	: !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
4528	? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarValue, IEMNATIVE_CALL_RET_GREG, &off)
4529	: iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off);
4530	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem, offDisp);
4531	uint8_t const idxRegValueStore = !TlbState.fSkip
4532	&& enmOp == kIemNativeEmitMemOp_Store
4533	&& pVarValue->enmKind != kIemNativeVarKind_Immediate
4534	? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off)
4535	: UINT8_MAX;
4536	uint32_t const idxRegMemResult = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
4537	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
4538	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
4539	: UINT32_MAX;
4540
4541	/*
4542	* Jump to the TLB lookup code.
4543	*/
4544	if (!TlbState.fSkip)
4545	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
4546
4547	/*
4548	* TlbMiss:
4549	*
4550	* Call helper to do the fetching.
4551	* We flush all guest register shadow copies here.
4552	*/
4553	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
4554
4555	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
4556	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
4557	#else
4558	RT_NOREF(idxInstr);
4559	#endif
4560
4561	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
4562	if (pReNative->Core.offPc)
4563	{
4564	/*
4565	* Update the program counter but restore it at the end of the TlbMiss branch.
4566	* This should allow delaying more program counter updates for the TlbLookup and hit paths
4567	* which are hopefully much more frequent, reducing the amount of memory accesses.
4568	*/
4569	/* Allocate a temporary PC register. */
4570	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
4571
4572	/* Perform the addition and store the result. */
4573	off = iemNativeEmitAddGprImm(pReNative, off, idxPcReg, pReNative->Core.offPc);
4574	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
4575
4576	/* Free and flush the PC register. */
4577	iemNativeRegFreeTmp(pReNative, idxPcReg);
4578	iemNativeRegFlushGuestShadowsByHostMask(pReNative, RT_BIT_32(idxPcReg));
4579	}
4580	#endif
4581
4582	#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4583	/* Save variables in volatile registers. */
4584	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
4585	\| (idxRegMemResult != UINT8_MAX ? RT_BIT_32(idxRegMemResult) : 0)
4586	\| (idxRegValueFetch != UINT8_MAX ? RT_BIT_32(idxRegValueFetch) : 0);
4587	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
4588	#endif
4589
4590	/* IEMNATIVE_CALL_ARG2/3_GREG = uValue (idxVarValue) - if store */
4591	uint32_t fVolGregMask = IEMNATIVE_CALL_VOLATILE_GREG_MASK;
4592	if (enmOp == kIemNativeEmitMemOp_Store)
4593	{
4594	uint8_t const idxRegArgValue = iSegReg == UINT8_MAX ? IEMNATIVE_CALL_ARG2_GREG : IEMNATIVE_CALL_ARG3_GREG;
4595	off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, idxRegArgValue, idxVarValue, 0 /cbAppend/,
4596	#ifdef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4597	IEMNATIVE_CALL_VOLATILE_GREG_MASK);
4598	#else
4599	IEMNATIVE_CALL_VOLATILE_GREG_MASK, true /fSpilledVarsInvolatileRegs/);
4600	fVolGregMask &= ~RT_BIT_32(idxRegArgValue);
4601	#endif
4602	}
4603
4604	/* IEMNATIVE_CALL_ARG1_GREG = GCPtrMem */
4605	off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxVarGCPtrMem, offDisp /cbAppend/,
4606	#ifdef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4607	fVolGregMask);
4608	#else
4609	fVolGregMask, true /fSpilledVarsInvolatileRegs/);
4610	#endif
4611
4612	if (iSegReg != UINT8_MAX)
4613	{
4614	/* IEMNATIVE_CALL_ARG2_GREG = iSegReg */
4615	AssertStmt(iSegReg < 6, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_BAD_SEG_REG_NO));
4616	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, iSegReg);
4617	}
4618
4619	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
4620	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
4621
4622	/* Done setting up parameters, make the call. */
4623	off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
4624
4625	/*
4626	* Put the result in the right register if this is a fetch.
4627	*/
4628	if (enmOp != kIemNativeEmitMemOp_Store)
4629	{
4630	Assert(idxRegValueFetch == pVarValue->idxReg);
4631	if (idxRegValueFetch != IEMNATIVE_CALL_RET_GREG)
4632	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegValueFetch, IEMNATIVE_CALL_RET_GREG);
4633	}
4634
4635	#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4636	/* Restore variables and guest shadow registers to volatile registers. */
4637	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
4638	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
4639	#endif
4640
4641	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
4642	if (pReNative->Core.offPc)
4643	{
4644	/*
4645	* Time to restore the program counter to its original value.
4646	*/
4647	/* Allocate a temporary PC register. */
4648	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
4649
4650	/* Restore the original value. */
4651	off = iemNativeEmitSubGprImm(pReNative, off, idxPcReg, pReNative->Core.offPc);
4652	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
4653
4654	/* Free and flush the PC register. */
4655	iemNativeRegFreeTmp(pReNative, idxPcReg);
4656	iemNativeRegFlushGuestShadowsByHostMask(pReNative, RT_BIT_32(idxPcReg));
4657	}
4658	#endif
4659
4660	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
4661	if (!TlbState.fSkip)
4662	{
4663	/* end of TlbMiss - Jump to the done label. */
4664	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
4665	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
4666
4667	/*
4668	* TlbLookup:
4669	*/
4670	off = iemNativeEmitTlbLookup<true>(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask,
4671	enmOp == kIemNativeEmitMemOp_Store ? IEM_ACCESS_TYPE_WRITE : IEM_ACCESS_TYPE_READ,
4672	idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult, offDisp);
4673
4674	/*
4675	* Emit code to do the actual storing / fetching.
4676	*/
4677	PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
4678	# ifdef VBOX_WITH_STATISTICS
4679	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
4680	enmOp == kIemNativeEmitMemOp_Store
4681	? RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForFetch)
4682	: RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStore));
4683	# endif
4684	switch (enmOp)
4685	{
4686	case kIemNativeEmitMemOp_Store:
4687	if (pVarValue->enmKind != kIemNativeVarKind_Immediate)
4688	{
4689	switch (cbMem)
4690	{
4691	case 1:
4692	off = iemNativeEmitStoreGpr8ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
4693	break;
4694	case 2:
4695	off = iemNativeEmitStoreGpr16ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
4696	break;
4697	case 4:
4698	off = iemNativeEmitStoreGpr32ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
4699	break;
4700	case 8:
4701	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
4702	break;
4703	default:
4704	AssertFailed();
4705	}
4706	}
4707	else
4708	{
4709	switch (cbMem)
4710	{
4711	case 1:
4712	off = iemNativeEmitStoreImm8ByGprEx(pCodeBuf, off, (uint8_t)pVarValue->u.uValue,
4713	idxRegMemResult, TlbState.idxReg1);
4714	break;
4715	case 2:
4716	off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off, (uint16_t)pVarValue->u.uValue,
4717	idxRegMemResult, TlbState.idxReg1);
4718	break;
4719	case 4:
4720	off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off, (uint32_t)pVarValue->u.uValue,
4721	idxRegMemResult, TlbState.idxReg1);
4722	break;
4723	case 8:
4724	off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pVarValue->u.uValue,
4725	idxRegMemResult, TlbState.idxReg1);
4726	break;
4727	default:
4728	AssertFailed();
4729	}
4730	}
4731	break;
4732
4733	case kIemNativeEmitMemOp_Fetch:
4734	case kIemNativeEmitMemOp_Fetch_Zx_U16:
4735	case kIemNativeEmitMemOp_Fetch_Zx_U32:
4736	case kIemNativeEmitMemOp_Fetch_Zx_U64:
4737	switch (cbMem)
4738	{
4739	case 1:
4740	off = iemNativeEmitLoadGprByGprU8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4741	break;
4742	case 2:
4743	off = iemNativeEmitLoadGprByGprU16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4744	break;
4745	case 4:
4746	off = iemNativeEmitLoadGprByGprU32Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4747	break;
4748	case 8:
4749	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4750	break;
4751	default:
4752	AssertFailed();
4753	}
4754	break;
4755
4756	case kIemNativeEmitMemOp_Fetch_Sx_U16:
4757	Assert(cbMem == 1);
4758	off = iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4759	break;
4760
4761	case kIemNativeEmitMemOp_Fetch_Sx_U32:
4762	Assert(cbMem == 1 \|\| cbMem == 2);
4763	if (cbMem == 1)
4764	off = iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4765	else
4766	off = iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4767	break;
4768
4769	case kIemNativeEmitMemOp_Fetch_Sx_U64:
4770	switch (cbMem)
4771	{
4772	case 1:
4773	off = iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4774	break;
4775	case 2:
4776	off = iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4777	break;
4778	case 4:
4779	off = iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
4780	break;
4781	default:
4782	AssertFailed();
4783	}
4784	break;
4785
4786	default:
4787	AssertFailed();
4788	}
4789
4790	iemNativeRegFreeTmp(pReNative, idxRegMemResult);
4791
4792	/*
4793	* TlbDone:
4794	*/
4795	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
4796
4797	TlbState.freeRegsAndReleaseVars(pReNative, idxVarGCPtrMem);
4798
4799	# ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
4800	/* Temp Hack: Flush all guest shadows in volatile registers in case of TLB miss. */
4801	iemNativeRegFlushGuestShadowsByHostMask(pReNative, IEMNATIVE_CALL_VOLATILE_GREG_MASK);
4802	# endif
4803	}
4804	#else
4805	RT_NOREF(fAlignMask, idxLabelTlbMiss);
4806	#endif
4807
4808	if (idxRegValueFetch != UINT8_MAX \|\| idxRegValueStore != UINT8_MAX)
4809	iemNativeVarRegisterRelease(pReNative, idxVarValue);
4810	return off;
4811	}
4812
4813
4814
4815	/*********************************************************************************************************************************
4816	* Memory fetches (IEM_MEM_FETCH_XXX). *
4817	*********************************************************************************************************************************/
4818
4819	/* 8-bit segmented: */
4820	#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
4821	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u8Dst, a_iSeg, a_GCPtrMem, \
4822	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch, \
4823	(uintptr_t)iemNativeHlpMemFetchDataU8, pCallEntry->idxInstr)
4824
4825	#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
4826	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, a_iSeg, a_GCPtrMem, \
4827	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U16, \
4828	(uintptr_t)iemNativeHlpMemFetchDataU8, pCallEntry->idxInstr)
4829
4830	#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
4831	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4832	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U32, \
4833	(uintptr_t)iemNativeHlpMemFetchDataU8, pCallEntry->idxInstr)
4834
4835	#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4836	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4837	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4838	(uintptr_t)iemNativeHlpMemFetchDataU8, pCallEntry->idxInstr)
4839
4840	#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
4841	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, a_iSeg, a_GCPtrMem, \
4842	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U16, \
4843	(uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U16, pCallEntry->idxInstr)
4844
4845	#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
4846	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4847	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U32, \
4848	(uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U32, pCallEntry->idxInstr)
4849
4850	#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4851	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4852	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U64, \
4853	(uintptr_t)iemNativeHlpMemFetchDataU8_Sx_U64, pCallEntry->idxInstr)
4854
4855	/* 16-bit segmented: */
4856	#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
4857	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, a_iSeg, a_GCPtrMem, \
4858	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch, \
4859	(uintptr_t)iemNativeHlpMemFetchDataU16, pCallEntry->idxInstr)
4860
4861	#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
4862	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, a_iSeg, a_GCPtrMem, \
4863	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch, \
4864	(uintptr_t)iemNativeHlpMemFetchDataU16, pCallEntry->idxInstr, a_offDisp)
4865
4866	#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
4867	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4868	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U32, \
4869	(uintptr_t)iemNativeHlpMemFetchDataU16, pCallEntry->idxInstr)
4870
4871	#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4872	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4873	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4874	(uintptr_t)iemNativeHlpMemFetchDataU16, pCallEntry->idxInstr)
4875
4876	#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
4877	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4878	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U32, \
4879	(uintptr_t)iemNativeHlpMemFetchDataU16_Sx_U32, pCallEntry->idxInstr)
4880
4881	#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4882	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4883	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U64, \
4884	(uintptr_t)iemNativeHlpMemFetchDataU16_Sx_U64, pCallEntry->idxInstr)
4885
4886
4887	/* 32-bit segmented: */
4888	#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
4889	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4890	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch, \
4891	(uintptr_t)iemNativeHlpMemFetchDataU32, pCallEntry->idxInstr)
4892
4893	#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
4894	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, a_iSeg, a_GCPtrMem, \
4895	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch, \
4896	(uintptr_t)iemNativeHlpMemFetchDataU32, pCallEntry->idxInstr, a_offDisp)
4897
4898	#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4899	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4900	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4901	(uintptr_t)iemNativeHlpMemFetchDataU32, pCallEntry->idxInstr)
4902
4903	#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4904	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4905	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U64, \
4906	(uintptr_t)iemNativeHlpMemFetchDataU32_Sx_U64, pCallEntry->idxInstr)
4907
4908
4909	/* 64-bit segmented: */
4910	#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
4911	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, a_iSeg, a_GCPtrMem, \
4912	sizeof(uint64_t), sizeof(uint64_t) - 1, kIemNativeEmitMemOp_Fetch, \
4913	(uintptr_t)iemNativeHlpMemFetchDataU64, pCallEntry->idxInstr)
4914
4915
4916
4917	/* 8-bit flat: */
4918	#define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
4919	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u8Dst, UINT8_MAX, a_GCPtrMem, \
4920	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch, \
4921	(uintptr_t)iemNativeHlpMemFlatFetchDataU8, pCallEntry->idxInstr)
4922
4923	#define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
4924	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, UINT8_MAX, a_GCPtrMem, \
4925	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U16, \
4926	(uintptr_t)iemNativeHlpMemFlatFetchDataU8, pCallEntry->idxInstr)
4927
4928	#define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
4929	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4930	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U32, \
4931	(uintptr_t)iemNativeHlpMemFlatFetchDataU8, pCallEntry->idxInstr)
4932
4933	#define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
4934	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
4935	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4936	(uintptr_t)iemNativeHlpMemFlatFetchDataU8, pCallEntry->idxInstr)
4937
4938	#define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
4939	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, UINT8_MAX, a_GCPtrMem, \
4940	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U16, \
4941	(uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U16, pCallEntry->idxInstr)
4942
4943	#define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
4944	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4945	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U32, \
4946	(uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U32, pCallEntry->idxInstr)
4947
4948	#define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
4949	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
4950	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Fetch_Sx_U64, \
4951	(uintptr_t)iemNativeHlpMemFlatFetchDataU8_Sx_U64, pCallEntry->idxInstr)
4952
4953
4954	/* 16-bit flat: */
4955	#define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
4956	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, UINT8_MAX, a_GCPtrMem, \
4957	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch, \
4958	(uintptr_t)iemNativeHlpMemFlatFetchDataU16, pCallEntry->idxInstr)
4959
4960	#define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
4961	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Dst, UINT8_MAX, a_GCPtrMem, \
4962	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch, \
4963	(uintptr_t)iemNativeHlpMemFlatFetchDataU16, pCallEntry->idxInstr, a_offDisp)
4964
4965	#define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
4966	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4967	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U32, \
4968	(uintptr_t)iemNativeHlpMemFlatFetchDataU16, pCallEntry->idxInstr)
4969
4970	#define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
4971	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
4972	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4973	(uintptr_t)iemNativeHlpMemFlatFetchDataU16, pCallEntry->idxInstr)
4974
4975	#define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
4976	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4977	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U32, \
4978	(uintptr_t)iemNativeHlpMemFlatFetchDataU16_Sx_U32, pCallEntry->idxInstr)
4979
4980	#define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
4981	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
4982	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U64, \
4983	(uintptr_t)iemNativeHlpMemFlatFetchDataU16_Sx_U64, pCallEntry->idxInstr)
4984
4985	/* 32-bit flat: */
4986	#define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
4987	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4988	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch, \
4989	(uintptr_t)iemNativeHlpMemFlatFetchDataU32, pCallEntry->idxInstr)
4990
4991	#define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
4992	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Dst, UINT8_MAX, a_GCPtrMem, \
4993	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch, \
4994	(uintptr_t)iemNativeHlpMemFlatFetchDataU32, pCallEntry->idxInstr, a_offDisp)
4995
4996	#define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
4997	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
4998	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch_Zx_U64, \
4999	(uintptr_t)iemNativeHlpMemFlatFetchDataU32, pCallEntry->idxInstr)
5000
5001	#define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
5002	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
5003	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Fetch_Sx_U64, \
5004	(uintptr_t)iemNativeHlpMemFlatFetchDataU32_Sx_U64, pCallEntry->idxInstr)
5005
5006	/* 64-bit flat: */
5007	#define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
5008	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Dst, UINT8_MAX, a_GCPtrMem, \
5009	sizeof(uint64_t), sizeof(uint64_t) - 1, kIemNativeEmitMemOp_Fetch, \
5010	(uintptr_t)iemNativeHlpMemFlatFetchDataU64, pCallEntry->idxInstr)
5011
5012
5013
5014	/*********************************************************************************************************************************
5015	* Memory stores (IEM_MEM_STORE_XXX). *
5016	*********************************************************************************************************************************/
5017
5018	#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
5019	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u8Value, a_iSeg, a_GCPtrMem, \
5020	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Store, \
5021	(uintptr_t)iemNativeHlpMemStoreDataU8, pCallEntry->idxInstr)
5022
5023	#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
5024	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Value, a_iSeg, a_GCPtrMem, \
5025	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Store, \
5026	(uintptr_t)iemNativeHlpMemStoreDataU16, pCallEntry->idxInstr)
5027
5028	#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
5029	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Value, a_iSeg, a_GCPtrMem, \
5030	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Store, \
5031	(uintptr_t)iemNativeHlpMemStoreDataU32, pCallEntry->idxInstr)
5032
5033	#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
5034	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Value, a_iSeg, a_GCPtrMem, \
5035	sizeof(uint64_t), sizeof(uint64_t) - 1, kIemNativeEmitMemOp_Store, \
5036	(uintptr_t)iemNativeHlpMemStoreDataU64, pCallEntry->idxInstr)
5037
5038
5039	#define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
5040	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u8Value, UINT8_MAX, a_GCPtrMem, \
5041	sizeof(uint8_t), 0 /fAlignMask/, kIemNativeEmitMemOp_Store, \
5042	(uintptr_t)iemNativeHlpMemFlatStoreDataU8, pCallEntry->idxInstr)
5043
5044	#define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
5045	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u16Value, UINT8_MAX, a_GCPtrMem, \
5046	sizeof(uint16_t), sizeof(uint16_t) - 1, kIemNativeEmitMemOp_Store, \
5047	(uintptr_t)iemNativeHlpMemFlatStoreDataU16, pCallEntry->idxInstr)
5048
5049	#define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
5050	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u32Value, UINT8_MAX, a_GCPtrMem, \
5051	sizeof(uint32_t), sizeof(uint32_t) - 1, kIemNativeEmitMemOp_Store, \
5052	(uintptr_t)iemNativeHlpMemFlatStoreDataU32, pCallEntry->idxInstr)
5053
5054	#define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
5055	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, a_u64Value, UINT8_MAX, a_GCPtrMem, \
5056	sizeof(uint64_t), sizeof(uint64_t) - 1, kIemNativeEmitMemOp_Store, \
5057	(uintptr_t)iemNativeHlpMemFlatStoreDataU64, pCallEntry->idxInstr)
5058
5059
5060	#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8ConstValue) \
5061	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u8ConstValue, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
5062	(uintptr_t)iemNativeHlpMemStoreDataU8, pCallEntry->idxInstr)
5063
5064	#define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16ConstValue) \
5065	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u16ConstValue, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
5066	(uintptr_t)iemNativeHlpMemStoreDataU16, pCallEntry->idxInstr)
5067
5068	#define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32ConstValue) \
5069	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u32ConstValue, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
5070	(uintptr_t)iemNativeHlpMemStoreDataU32, pCallEntry->idxInstr)
5071
5072	#define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64ConstValue) \
5073	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u64ConstValue, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
5074	(uintptr_t)iemNativeHlpMemStoreDataU64, pCallEntry->idxInstr)
5075
5076
5077	#define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8ConstValue) \
5078	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u8ConstValue, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
5079	(uintptr_t)iemNativeHlpMemFlatStoreDataU8, pCallEntry->idxInstr)
5080
5081	#define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16ConstValue) \
5082	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u16ConstValue, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
5083	(uintptr_t)iemNativeHlpMemFlatStoreDataU16, pCallEntry->idxInstr)
5084
5085	#define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32ConstValue) \
5086	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u32ConstValue, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
5087	(uintptr_t)iemNativeHlpMemFlatStoreDataU32, pCallEntry->idxInstr)
5088
5089	#define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64ConstValue) \
5090	off = iemNativeEmitMemStoreConstDataCommon(pReNative, off, a_u64ConstValue, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
5091	(uintptr_t)iemNativeHlpMemFlatStoreDataU64, pCallEntry->idxInstr)
5092
5093	/** Emits code for IEM_MC_STORE_MEM_U8/16/32/64_CONST and
5094	* IEM_MC_STORE_MEM_FLAT_U8/16/32/64_CONST (with iSegReg = UINT8_MAX). */
5095	DECL_INLINE_THROW(uint32_t)
5096	iemNativeEmitMemStoreConstDataCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t uValueConst, uint8_t iSegReg,
5097	uint8_t idxVarGCPtrMem, uint8_t cbMem, uintptr_t pfnFunction, uint8_t idxInstr)
5098	{
5099	/*
5100	* Create a temporary const variable and call iemNativeEmitMemFetchStoreDataCommon
5101	* to do the grunt work.
5102	*/
5103	uint8_t const idxVarConstValue = iemNativeVarAllocConst(pReNative, cbMem, uValueConst);
5104	off = iemNativeEmitMemFetchStoreDataCommon(pReNative, off, idxVarConstValue, iSegReg, idxVarGCPtrMem,
5105	cbMem, cbMem - 1, kIemNativeEmitMemOp_Store,
5106	pfnFunction, idxInstr);
5107	iemNativeVarFreeLocal(pReNative, idxVarConstValue);
5108	return off;
5109	}
5110
5111
5112
5113	/*********************************************************************************************************************************
5114	* Stack Accesses. *
5115	*********************************************************************************************************************************/
5116	/* RT_MAKE_U32_FROM_U8(cBitsVar, cBitsFlat, fSReg, 0) */
5117	#define IEM_MC_PUSH_U16(a_u16Value) \
5118	off = iemNativeEmitStackPush(pReNative, off, a_u16Value, RT_MAKE_U32_FROM_U8(16, 0, 0, 0), \
5119	(uintptr_t)iemNativeHlpStackStoreU16, pCallEntry->idxInstr)
5120	#define IEM_MC_PUSH_U32(a_u32Value) \
5121	off = iemNativeEmitStackPush(pReNative, off, a_u32Value, RT_MAKE_U32_FROM_U8(32, 0, 0, 0), \
5122	(uintptr_t)iemNativeHlpStackStoreU32, pCallEntry->idxInstr)
5123	#define IEM_MC_PUSH_U32_SREG(a_uSegVal) \
5124	off = iemNativeEmitStackPush(pReNative, off, a_uSegVal, RT_MAKE_U32_FROM_U8(32, 0, 1, 0), \
5125	(uintptr_t)iemNativeHlpStackStoreU32SReg, pCallEntry->idxInstr)
5126	#define IEM_MC_PUSH_U64(a_u64Value) \
5127	off = iemNativeEmitStackPush(pReNative, off, a_u64Value, RT_MAKE_U32_FROM_U8(64, 0, 0, 0), \
5128	(uintptr_t)iemNativeHlpStackStoreU64, pCallEntry->idxInstr)
5129
5130	#define IEM_MC_FLAT32_PUSH_U16(a_u16Value) \
5131	off = iemNativeEmitStackPush(pReNative, off, a_u16Value, RT_MAKE_U32_FROM_U8(16, 32, 0, 0), \
5132	(uintptr_t)iemNativeHlpStackFlatStoreU16, pCallEntry->idxInstr)
5133	#define IEM_MC_FLAT32_PUSH_U32(a_u32Value) \
5134	off = iemNativeEmitStackPush(pReNative, off, a_u32Value, RT_MAKE_U32_FROM_U8(32, 32, 0, 0), \
5135	(uintptr_t)iemNativeHlpStackFlatStoreU32, pCallEntry->idxInstr)
5136	#define IEM_MC_FLAT32_PUSH_U32_SREG(a_u32Value) \
5137	off = iemNativeEmitStackPush(pReNative, off, a_u32Value, RT_MAKE_U32_FROM_U8(32, 32, 1, 0), \
5138	(uintptr_t)iemNativeHlpStackFlatStoreU32SReg, pCallEntry->idxInstr)
5139
5140	#define IEM_MC_FLAT64_PUSH_U16(a_u16Value) \
5141	off = iemNativeEmitStackPush(pReNative, off, a_u16Value, RT_MAKE_U32_FROM_U8(16, 64, 0, 0), \
5142	(uintptr_t)iemNativeHlpStackFlatStoreU16, pCallEntry->idxInstr)
5143	#define IEM_MC_FLAT64_PUSH_U64(a_u64Value) \
5144	off = iemNativeEmitStackPush(pReNative, off, a_u64Value, RT_MAKE_U32_FROM_U8(64, 64, 0, 0), \
5145	(uintptr_t)iemNativeHlpStackFlatStoreU64, pCallEntry->idxInstr)
5146
5147
5148	DECL_FORCE_INLINE_THROW(uint32_t)
5149	iemNativeEmitStackPushUse16Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem)
5150	{
5151	/* Use16BitSp: */
5152	#ifdef RT_ARCH_AMD64
5153	off = iemNativeEmitSubGpr16ImmEx(pCodeBuf, off, idxRegRsp, cbMem); /* ASSUMES this does NOT modify bits [63:16]! */
5154	off = iemNativeEmitLoadGprFromGpr16Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
5155	#else
5156	/* sub regeff, regrsp, #cbMem */
5157	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegEffSp, idxRegRsp, cbMem, false /f64Bit/);
5158	/* and regeff, regeff, #0xffff */
5159	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
5160	pCodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegEffSp, idxRegEffSp, 15, 0, false /f64Bit/);
5161	/* bfi regrsp, regeff, #0, #16 - moves bits 15:0 from idxVarReg to idxGstTmpReg bits 15:0. */
5162	pCodeBuf[off++] = Armv8A64MkInstrBfi(idxRegRsp, idxRegEffSp, 0, 16, false /f64Bit/);
5163	#endif
5164	return off;
5165	}
5166
5167
5168	DECL_FORCE_INLINE(uint32_t)
5169	iemNativeEmitStackPushUse32Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem)
5170	{
5171	/* Use32BitSp: */
5172	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, idxRegRsp, cbMem);
5173	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
5174	return off;
5175	}
5176
5177
5178	/** IEM_MC[\|_FLAT32\|_FLAT64]_PUSH_U16/32/32_SREG/64 */
5179	DECL_INLINE_THROW(uint32_t)
5180	iemNativeEmitStackPush(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarValue,
5181	uint32_t cBitsVarAndFlat, uintptr_t pfnFunction, uint8_t idxInstr)
5182	{
5183	/*
5184	* Assert sanity.
5185	*/
5186	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarValue);
5187	PIEMNATIVEVAR const pVarValue = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarValue)];
5188	#ifdef VBOX_STRICT
5189	if (RT_BYTE2(cBitsVarAndFlat) != 0)
5190	{
5191	Assert( (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT
5192	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT
5193	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT);
5194	Assert( pfnFunction
5195	== ( cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatStoreU16
5196	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatStoreU32
5197	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 32, 1, 0) ? (uintptr_t)iemNativeHlpStackFlatStoreU32SReg
5198	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatStoreU16
5199	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatStoreU64
5200	: UINT64_C(0xc000b000a0009000) ));
5201	}
5202	else
5203	Assert( pfnFunction
5204	== ( cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackStoreU16
5205	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackStoreU32
5206	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 0, 1, 0) ? (uintptr_t)iemNativeHlpStackStoreU32SReg
5207	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackStoreU64
5208	: UINT64_C(0xc000b000a0009000) ));
5209	#endif
5210
5211	#ifdef VBOX_STRICT
5212	/*
5213	* Check that the fExec flags we've got make sense.
5214	*/
5215	off = iemNativeEmitExecFlagsCheck(pReNative, off, pReNative->fExec);
5216	#endif
5217
5218	/*
5219	* To keep things simple we have to commit any pending writes first as we
5220	* may end up making calls.
5221	*/
5222	/** @todo we could postpone this till we make the call and reload the
5223	* registers after returning from the call. Not sure if that's sensible or
5224	* not, though. */
5225	off = iemNativeRegFlushPendingWrites(pReNative, off);
5226
5227	/*
5228	* First we calculate the new RSP and the effective stack pointer value.
5229	* For 64-bit mode and flat 32-bit these two are the same.
5230	* (Code structure is very similar to that of PUSH)
5231	*/
5232	uint8_t const cbMem = RT_BYTE1(cBitsVarAndFlat) / 8;
5233	bool const fIsSegReg = RT_BYTE3(cBitsVarAndFlat) != 0;
5234	bool const fIsIntelSeg = fIsSegReg && IEM_IS_GUEST_CPU_INTEL(pReNative->pVCpu);
5235	uint8_t const cbMemAccess = !fIsIntelSeg \|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_16BIT
5236	? cbMem : sizeof(uint16_t);
5237	uint8_t const cBitsFlat = RT_BYTE2(cBitsVarAndFlat); RT_NOREF(cBitsFlat);
5238	uint8_t const idxRegRsp = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xSP),
5239	kIemNativeGstRegUse_ForUpdate, true /fNoVolatileRegs/);
5240	uint8_t const idxRegEffSp = cBitsFlat != 0 ? idxRegRsp : iemNativeRegAllocTmp(pReNative, &off);
5241	uint32_t offFixupJumpToUseOtherBitSp = UINT32_MAX;
5242	if (cBitsFlat != 0)
5243	{
5244	Assert(idxRegEffSp == idxRegRsp);
5245	Assert(cBitsFlat == 32 \|\| cBitsFlat == 64);
5246	Assert(IEM_F_MODE_X86_IS_FLAT(pReNative->fExec));
5247	if (cBitsFlat == 64)
5248	off = iemNativeEmitSubGprImm(pReNative, off, idxRegRsp, cbMem);
5249	else
5250	off = iemNativeEmitSubGpr32Imm(pReNative, off, idxRegRsp, cbMem);
5251	}
5252	else /** @todo We can skip the test if we're targeting pre-386 CPUs. */
5253	{
5254	Assert(idxRegEffSp != idxRegRsp);
5255	uint8_t const idxRegSsAttr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(X86_SREG_SS),
5256	kIemNativeGstRegUse_ReadOnly);
5257	#ifdef RT_ARCH_AMD64
5258	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
5259	#else
5260	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
5261	#endif
5262	off = iemNativeEmitTestAnyBitsInGpr32Ex(pCodeBuf, off, idxRegSsAttr, X86DESCATTR_D);
5263	iemNativeRegFreeTmp(pReNative, idxRegSsAttr);
5264	offFixupJumpToUseOtherBitSp = off;
5265	if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
5266	{
5267	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /8-bit suffices/, kIemNativeInstrCond_e); /* jump if zero */
5268	off = iemNativeEmitStackPushUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5269	}
5270	else
5271	{
5272	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /8-bit suffices/, kIemNativeInstrCond_ne); /* jump if not zero */
5273	off = iemNativeEmitStackPushUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5274	}
5275	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5276	}
5277	/* SpUpdateEnd: */
5278	uint32_t const offLabelSpUpdateEnd = off;
5279
5280	/*
5281	* Okay, now prepare for TLB lookup and jump to code (or the TlbMiss if
5282	* we're skipping lookup).
5283	*/
5284	uint8_t const iSegReg = cBitsFlat != 0 ? UINT8_MAX : X86_SREG_SS;
5285	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, idxRegEffSp, &off, iSegReg, cbMemAccess);
5286	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
5287	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
5288	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
5289	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
5290	: UINT32_MAX;
5291	uint8_t const idxRegValue = !TlbState.fSkip
5292	&& pVarValue->enmKind != kIemNativeVarKind_Immediate
5293	? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off, true /fInitialized/,
5294	IEMNATIVE_CALL_ARG2_GREG /idxRegPref/)
5295	: UINT8_MAX;
5296	uint8_t const idxRegMemResult = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
5297
5298
5299	if (!TlbState.fSkip)
5300	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
5301	else
5302	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbMiss); /** @todo short jump */
5303
5304	/*
5305	* Use16BitSp:
5306	*/
5307	if (cBitsFlat == 0)
5308	{
5309	#ifdef RT_ARCH_AMD64
5310	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
5311	#else
5312	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
5313	#endif
5314	iemNativeFixupFixedJump(pReNative, offFixupJumpToUseOtherBitSp, off);
5315	if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
5316	off = iemNativeEmitStackPushUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5317	else
5318	off = iemNativeEmitStackPushUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5319	off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, offLabelSpUpdateEnd);
5320	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5321	}
5322
5323	/*
5324	* TlbMiss:
5325	*
5326	* Call helper to do the pushing.
5327	*/
5328	iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
5329
5330	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
5331	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
5332	#else
5333	RT_NOREF(idxInstr);
5334	#endif
5335
5336	/* Save variables in volatile registers. */
5337	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
5338	\| (idxRegMemResult < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegMemResult) : 0)
5339	\| (idxRegEffSp != idxRegRsp ? RT_BIT_32(idxRegEffSp) : 0)
5340	\| (idxRegValue < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegValue) : 0);
5341	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
5342
5343	if ( idxRegValue == IEMNATIVE_CALL_ARG1_GREG
5344	&& idxRegEffSp == IEMNATIVE_CALL_ARG2_GREG)
5345	{
5346	/* Swap them using ARG0 as temp register: */
5347	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_CALL_ARG1_GREG);
5348	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_CALL_ARG2_GREG);
5349	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, IEMNATIVE_CALL_ARG0_GREG);
5350	}
5351	else if (idxRegEffSp != IEMNATIVE_CALL_ARG2_GREG)
5352	{
5353	/* IEMNATIVE_CALL_ARG2_GREG = idxVarValue (first!) */
5354	off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, idxVarValue,
5355	0 /offAddend/, IEMNATIVE_CALL_VOLATILE_GREG_MASK);
5356
5357	/* IEMNATIVE_CALL_ARG1_GREG = idxRegEffSp */
5358	if (idxRegEffSp != IEMNATIVE_CALL_ARG1_GREG)
5359	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxRegEffSp);
5360	}
5361	else
5362	{
5363	/* IEMNATIVE_CALL_ARG1_GREG = idxRegEffSp (first!) */
5364	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxRegEffSp);
5365
5366	/* IEMNATIVE_CALL_ARG2_GREG = idxVarValue */
5367	off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, idxVarValue, 0 /offAddend/,
5368	IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_CALL_ARG1_GREG);
5369	}
5370
5371	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
5372	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
5373
5374	/* Done setting up parameters, make the call. */
5375	off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
5376
5377	/* Restore variables and guest shadow registers to volatile registers. */
5378	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
5379	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
5380
5381	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
5382	if (!TlbState.fSkip)
5383	{
5384	/* end of TlbMiss - Jump to the done label. */
5385	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
5386	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
5387
5388	/*
5389	* TlbLookup:
5390	*/
5391	off = iemNativeEmitTlbLookup<true>(pReNative, off, &TlbState, iSegReg, cbMemAccess, cbMemAccess - 1,
5392	IEM_ACCESS_TYPE_WRITE, idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
5393
5394	/*
5395	* Emit code to do the actual storing / fetching.
5396	*/
5397	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
5398	# ifdef VBOX_WITH_STATISTICS
5399	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
5400	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStack));
5401	# endif
5402	if (idxRegValue != UINT8_MAX)
5403	{
5404	switch (cbMemAccess)
5405	{
5406	case 2:
5407	off = iemNativeEmitStoreGpr16ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
5408	break;
5409	case 4:
5410	if (!fIsIntelSeg)
5411	off = iemNativeEmitStoreGpr32ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
5412	else
5413	{
5414	/* intel real mode segment push. 10890XE adds the 2nd of half EFLAGS to a
5415	PUSH FS in real mode, so we have to try emulate that here.
5416	We borrow the now unused idxReg1 from the TLB lookup code here. */
5417	uint8_t idxRegEfl = iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(pReNative, &off,
5418	kIemNativeGstReg_EFlags);
5419	if (idxRegEfl != UINT8_MAX)
5420	{
5421	#ifdef ARCH_AMD64
5422	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, TlbState.idxReg1, idxRegEfl);
5423	off = iemNativeEmitAndGpr32ByImm(pReNative, off, TlbState.idxReg1,
5424	UINT32_C(0xffff0000) & ~X86_EFL_RAZ_MASK);
5425	#else
5426	off = iemNativeEmitGpr32EqGprAndImmEx(iemNativeInstrBufEnsure(pReNative, off, 3),
5427	off, TlbState.idxReg1, idxRegEfl,
5428	UINT32_C(0xffff0000) & ~X86_EFL_RAZ_MASK);
5429	#endif
5430	iemNativeRegFreeTmp(pReNative, idxRegEfl);
5431	}
5432	else
5433	{
5434	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, TlbState.idxReg1,
5435	RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.eflags));
5436	off = iemNativeEmitAndGpr32ByImm(pReNative, off, TlbState.idxReg1,
5437	UINT32_C(0xffff0000) & ~X86_EFL_RAZ_MASK);
5438	}
5439	/* ASSUMES the upper half of idxRegValue is ZERO. */
5440	off = iemNativeEmitOrGpr32ByGpr(pReNative, off, TlbState.idxReg1, idxRegValue);
5441	off = iemNativeEmitStoreGpr32ByGprEx(pCodeBuf, off, TlbState.idxReg1, idxRegMemResult);
5442	}
5443	break;
5444	case 8:
5445	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
5446	break;
5447	default:
5448	AssertFailed();
5449	}
5450	}
5451	else
5452	{
5453	switch (cbMemAccess)
5454	{
5455	case 2:
5456	off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off, (uint16_t)pVarValue->u.uValue,
5457	idxRegMemResult, TlbState.idxReg1);
5458	break;
5459	case 4:
5460	Assert(!fIsSegReg);
5461	off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off, (uint32_t)pVarValue->u.uValue,
5462	idxRegMemResult, TlbState.idxReg1);
5463	break;
5464	case 8:
5465	off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pVarValue->u.uValue, idxRegMemResult, TlbState.idxReg1);
5466	break;
5467	default:
5468	AssertFailed();
5469	}
5470	}
5471
5472	iemNativeRegFreeTmp(pReNative, idxRegMemResult);
5473	TlbState.freeRegsAndReleaseVars(pReNative);
5474
5475	/*
5476	* TlbDone:
5477	*
5478	* Commit the new RSP value.
5479	*/
5480	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
5481	}
5482	#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
5483
5484	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegRsp, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.rsp));
5485	iemNativeRegFreeTmp(pReNative, idxRegRsp);
5486	if (idxRegEffSp != idxRegRsp)
5487	iemNativeRegFreeTmp(pReNative, idxRegEffSp);
5488
5489	/* The value variable is implictly flushed. */
5490	if (idxRegValue != UINT8_MAX)
5491	iemNativeVarRegisterRelease(pReNative, idxVarValue);
5492	iemNativeVarFreeLocal(pReNative, idxVarValue);
5493
5494	return off;
5495	}
5496
5497
5498
5499	/* RT_MAKE_U32_FROM_U8(cBitsVar, cBitsFlat, 0, 0) */
5500	#define IEM_MC_POP_GREG_U16(a_iGReg) \
5501	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16, 0, 0, 0), \
5502	(uintptr_t)iemNativeHlpStackFetchU16, pCallEntry->idxInstr)
5503	#define IEM_MC_POP_GREG_U32(a_iGReg) \
5504	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(32, 0, 0, 0), \
5505	(uintptr_t)iemNativeHlpStackFetchU32, pCallEntry->idxInstr)
5506	#define IEM_MC_POP_GREG_U64(a_iGReg) \
5507	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(64, 0, 0, 0), \
5508	(uintptr_t)iemNativeHlpStackFetchU64, pCallEntry->idxInstr)
5509
5510	#define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) \
5511	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16, 32, 0, 0), \
5512	(uintptr_t)iemNativeHlpStackFlatFetchU16, pCallEntry->idxInstr)
5513	#define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) \
5514	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(32, 32, 0, 0), \
5515	(uintptr_t)iemNativeHlpStackFlatFetchU32, pCallEntry->idxInstr)
5516
5517	#define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) \
5518	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(16, 64, 0, 0), \
5519	(uintptr_t)iemNativeHlpStackFlatFetchU16, pCallEntry->idxInstr)
5520	#define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) \
5521	off = iemNativeEmitStackPopGReg(pReNative, off, a_iGReg, RT_MAKE_U32_FROM_U8(64, 64, 0, 0), \
5522	(uintptr_t)iemNativeHlpStackFlatFetchU64, pCallEntry->idxInstr)
5523
5524
5525	DECL_FORCE_INLINE_THROW(uint32_t)
5526	iemNativeEmitStackPopUse16Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem,
5527	uint8_t idxRegTmp)
5528	{
5529	/* Use16BitSp: */
5530	#ifdef RT_ARCH_AMD64
5531	off = iemNativeEmitLoadGprFromGpr16Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
5532	off = iemNativeEmitAddGpr16ImmEx(pCodeBuf, off, idxRegRsp, cbMem); /* ASSUMES this does NOT modify bits [63:16]! */
5533	RT_NOREF(idxRegTmp);
5534	#else
5535	/* ubfiz regeff, regrsp, #0, #16 - copies bits 15:0 from RSP to EffSp bits 15:0, zeroing bits 63:16. */
5536	pCodeBuf[off++] = Armv8A64MkInstrUbfiz(idxRegEffSp, idxRegRsp, 0, 16, false /f64Bit/);
5537	/* add tmp, regrsp, #cbMem */
5538	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegTmp, idxRegRsp, cbMem, false /f64Bit/);
5539	/* and tmp, tmp, #0xffff */
5540	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
5541	pCodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegTmp, idxRegTmp, 15, 0, false /f64Bit/);
5542	/* bfi regrsp, regeff, #0, #16 - moves bits 15:0 from tmp to RSP bits 15:0, keeping the other RSP bits as is. */
5543	pCodeBuf[off++] = Armv8A64MkInstrBfi(idxRegRsp, idxRegTmp, 0, 16, false /f64Bit/);
5544	#endif
5545	return off;
5546	}
5547
5548
5549	DECL_FORCE_INLINE(uint32_t)
5550	iemNativeEmitStackPopUse32Sp(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxRegRsp, uint8_t idxRegEffSp, uint8_t cbMem)
5551	{
5552	/* Use32BitSp: */
5553	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, idxRegEffSp, idxRegRsp);
5554	off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegRsp, cbMem);
5555	return off;
5556	}
5557
5558
5559	/** IEM_MC[\|_FLAT32\|_FLAT64]_POP_GREG_U16/32/64 */
5560	DECL_INLINE_THROW(uint32_t)
5561	iemNativeEmitStackPopGReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGReg,
5562	uint32_t cBitsVarAndFlat, uintptr_t pfnFunction, uint8_t idxInstr)
5563	{
5564	/*
5565	* Assert sanity.
5566	*/
5567	Assert(idxGReg < 16);
5568	#ifdef VBOX_STRICT
5569	if (RT_BYTE2(cBitsVarAndFlat) != 0)
5570	{
5571	Assert( (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT
5572	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT
5573	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT);
5574	Assert( pfnFunction
5575	== ( cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU16
5576	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 32, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU32
5577	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU16
5578	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 64, 0, 0) ? (uintptr_t)iemNativeHlpStackFlatFetchU64
5579	: UINT64_C(0xc000b000a0009000) ));
5580	}
5581	else
5582	Assert( pfnFunction
5583	== ( cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(16, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU16
5584	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(32, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU32
5585	: cBitsVarAndFlat == RT_MAKE_U32_FROM_U8(64, 0, 0, 0) ? (uintptr_t)iemNativeHlpStackFetchU64
5586	: UINT64_C(0xc000b000a0009000) ));
5587	#endif
5588
5589	#ifdef VBOX_STRICT
5590	/*
5591	* Check that the fExec flags we've got make sense.
5592	*/
5593	off = iemNativeEmitExecFlagsCheck(pReNative, off, pReNative->fExec);
5594	#endif
5595
5596	/*
5597	* To keep things simple we have to commit any pending writes first as we
5598	* may end up making calls.
5599	*/
5600	off = iemNativeRegFlushPendingWrites(pReNative, off);
5601
5602	/*
5603	* Determine the effective stack pointer, for non-FLAT modes we also update RSP.
5604	* For FLAT modes we'll do this in TlbDone as we'll be using the incoming RSP
5605	* directly as the effective stack pointer.
5606	* (Code structure is very similar to that of PUSH)
5607	*/
5608	uint8_t const cbMem = RT_BYTE1(cBitsVarAndFlat) / 8;
5609	uint8_t const cBitsFlat = RT_BYTE2(cBitsVarAndFlat); RT_NOREF(cBitsFlat);
5610	uint8_t const idxRegRsp = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xSP),
5611	kIemNativeGstRegUse_ForUpdate, true /fNoVolatileRegs/);
5612	uint8_t const idxRegEffSp = cBitsFlat != 0 ? idxRegRsp : iemNativeRegAllocTmp(pReNative, &off);
5613	/** @todo can do a better job picking the register here. For cbMem >= 4 this
5614	* will be the resulting register value. */
5615	uint8_t const idxRegMemResult = iemNativeRegAllocTmp(pReNative, &off); /* pointer then value; arm64 SP += 2/4 helper too. */
5616
5617	uint32_t offFixupJumpToUseOtherBitSp = UINT32_MAX;
5618	if (cBitsFlat != 0)
5619	{
5620	Assert(idxRegEffSp == idxRegRsp);
5621	Assert(cBitsFlat == 32 \|\| cBitsFlat == 64);
5622	Assert(IEM_F_MODE_X86_IS_FLAT(pReNative->fExec));
5623	}
5624	else /** @todo We can skip the test if we're targeting pre-386 CPUs. */
5625	{
5626	Assert(idxRegEffSp != idxRegRsp);
5627	uint8_t const idxRegSsAttr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(X86_SREG_SS),
5628	kIemNativeGstRegUse_ReadOnly);
5629	#ifdef RT_ARCH_AMD64
5630	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
5631	#else
5632	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
5633	#endif
5634	off = iemNativeEmitTestAnyBitsInGpr32Ex(pCodeBuf, off, idxRegSsAttr, X86DESCATTR_D);
5635	iemNativeRegFreeTmp(pReNative, idxRegSsAttr);
5636	offFixupJumpToUseOtherBitSp = off;
5637	if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
5638	{
5639	/** @todo can skip idxRegRsp updating when popping ESP. */
5640	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /8-bit suffices/, kIemNativeInstrCond_e); /* jump if zero */
5641	off = iemNativeEmitStackPopUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5642	}
5643	else
5644	{
5645	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off /8-bit suffices/, kIemNativeInstrCond_ne); /* jump if not zero */
5646	off = iemNativeEmitStackPopUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem, idxRegMemResult);
5647	}
5648	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5649	}
5650	/* SpUpdateEnd: */
5651	uint32_t const offLabelSpUpdateEnd = off;
5652
5653	/*
5654	* Okay, now prepare for TLB lookup and jump to code (or the TlbMiss if
5655	* we're skipping lookup).
5656	*/
5657	uint8_t const iSegReg = cBitsFlat != 0 ? UINT8_MAX : X86_SREG_SS;
5658	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, idxRegEffSp, &off, iSegReg, cbMem);
5659	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
5660	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
5661	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
5662	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
5663	: UINT32_MAX;
5664
5665	if (!TlbState.fSkip)
5666	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
5667	else
5668	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbMiss); /** @todo short jump */
5669
5670	/*
5671	* Use16BitSp:
5672	*/
5673	if (cBitsFlat == 0)
5674	{
5675	#ifdef RT_ARCH_AMD64
5676	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
5677	#else
5678	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
5679	#endif
5680	iemNativeFixupFixedJump(pReNative, offFixupJumpToUseOtherBitSp, off);
5681	if ((pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT)
5682	off = iemNativeEmitStackPopUse16Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem, idxRegMemResult);
5683	else
5684	off = iemNativeEmitStackPopUse32Sp(pCodeBuf, off, idxRegRsp, idxRegEffSp, cbMem);
5685	off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, offLabelSpUpdateEnd);
5686	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5687	}
5688
5689	/*
5690	* TlbMiss:
5691	*
5692	* Call helper to do the pushing.
5693	*/
5694	iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
5695
5696	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
5697	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
5698	#else
5699	RT_NOREF(idxInstr);
5700	#endif
5701
5702	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
5703	\| (idxRegMemResult < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegMemResult) : 0)
5704	\| (idxRegEffSp != idxRegRsp ? RT_BIT_32(idxRegEffSp) : 0);
5705	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
5706
5707
5708	/* IEMNATIVE_CALL_ARG1_GREG = EffSp/RSP */
5709	if (idxRegEffSp != IEMNATIVE_CALL_ARG1_GREG)
5710	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxRegEffSp);
5711
5712	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
5713	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
5714
5715	/* Done setting up parameters, make the call. */
5716	off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
5717
5718	/* Move the return register content to idxRegMemResult. */
5719	if (idxRegMemResult != IEMNATIVE_CALL_RET_GREG)
5720	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
5721
5722	/* Restore variables and guest shadow registers to volatile registers. */
5723	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
5724	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
5725
5726	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
5727	if (!TlbState.fSkip)
5728	{
5729	/* end of TlbMiss - Jump to the done label. */
5730	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
5731	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
5732
5733	/*
5734	* TlbLookup:
5735	*/
5736	off = iemNativeEmitTlbLookup<true>(pReNative, off, &TlbState, iSegReg, cbMem, cbMem - 1, IEM_ACCESS_TYPE_READ,
5737	idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
5738
5739	/*
5740	* Emit code to load the value (from idxRegMemResult into idxRegMemResult).
5741	*/
5742	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
5743	# ifdef VBOX_WITH_STATISTICS
5744	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
5745	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStack));
5746	# endif
5747	switch (cbMem)
5748	{
5749	case 2:
5750	off = iemNativeEmitLoadGprByGprU16Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
5751	break;
5752	case 4:
5753	off = iemNativeEmitLoadGprByGprU32Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
5754	break;
5755	case 8:
5756	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegMemResult, idxRegMemResult);
5757	break;
5758	default:
5759	AssertFailed();
5760	}
5761
5762	TlbState.freeRegsAndReleaseVars(pReNative);
5763
5764	/*
5765	* TlbDone:
5766	*
5767	* Set the new RSP value (FLAT accesses needs to calculate it first) and
5768	* commit the popped register value.
5769	*/
5770	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
5771	}
5772	#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
5773
5774	if (idxGReg != X86_GREG_xSP)
5775	{
5776	/* Set the register. */
5777	if (cbMem >= sizeof(uint32_t))
5778	{
5779	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
5780	AssertMsg( pReNative->idxCurCall == 0
5781	\|\| IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(iemNativeLivenessGetPrevStateByGstReg(pReNative, IEMNATIVEGSTREG_GPR(idxGReg))),
5782	("%s - %u\n", g_aGstShadowInfo[idxGReg].pszName, iemNativeLivenessGetPrevStateByGstReg(pReNative, IEMNATIVEGSTREG_GPR(idxGReg))));
5783	#endif
5784	iemNativeRegClearAndMarkAsGstRegShadow(pReNative, idxRegMemResult, IEMNATIVEGSTREG_GPR(idxGReg), off);
5785	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegMemResult,
5786	RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[idxGReg]));
5787	}
5788	else
5789	{
5790	Assert(cbMem == sizeof(uint16_t));
5791	uint8_t const idxRegDst = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(idxGReg),
5792	kIemNativeGstRegUse_ForUpdate);
5793	off = iemNativeEmitGprMergeInGpr16(pReNative, off, idxRegDst, idxRegMemResult);
5794	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegDst, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.aGRegs[idxGReg]));
5795	iemNativeRegFreeTmp(pReNative, idxRegDst);
5796	}
5797
5798	/* Complete RSP calculation for FLAT mode. */
5799	if (idxRegEffSp == idxRegRsp)
5800	{
5801	if (cBitsFlat == 64)
5802	off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRsp, sizeof(uint64_t));
5803	else
5804	off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxRegRsp, sizeof(uint32_t));
5805	}
5806	}
5807	else
5808	{
5809	/* We're popping RSP, ESP or SP. Only the is a bit extra work, of course. */
5810	if (cbMem == sizeof(uint64_t))
5811	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegRsp, idxRegMemResult);
5812	else if (cbMem == sizeof(uint32_t))
5813	off = iemNativeEmitLoadGprFromGpr32(pReNative, off, idxRegRsp, idxRegMemResult);
5814	else
5815	{
5816	if (idxRegEffSp == idxRegRsp)
5817	{
5818	if (cBitsFlat == 64)
5819	off = iemNativeEmitAddGprImm8(pReNative, off, idxRegRsp, sizeof(uint64_t));
5820	else
5821	off = iemNativeEmitAddGpr32Imm8(pReNative, off, idxRegRsp, sizeof(uint32_t));
5822	}
5823	off = iemNativeEmitGprMergeInGpr16(pReNative, off, idxRegRsp, idxRegMemResult);
5824	}
5825	}
5826	off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegRsp, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rsp));
5827
5828	iemNativeRegFreeTmp(pReNative, idxRegRsp);
5829	if (idxRegEffSp != idxRegRsp)
5830	iemNativeRegFreeTmp(pReNative, idxRegEffSp);
5831	iemNativeRegFreeTmp(pReNative, idxRegMemResult);
5832
5833	return off;
5834	}
5835
5836
5837
5838	/*********************************************************************************************************************************
5839	* Memory mapping (IEM_MEM_MAP_XXX, IEM_MEM_FLAT_MAP_XXX). *
5840	*********************************************************************************************************************************/
5841
5842	#define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5843	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
5844	IEM_ACCESS_DATA_ATOMIC, 0 /fAlignMask/, \
5845	(uintptr_t)iemNativeHlpMemMapDataU8Atomic, pCallEntry->idxInstr)
5846
5847	#define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5848	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
5849	IEM_ACCESS_DATA_RW, 0 /fAlignMask/, \
5850	(uintptr_t)iemNativeHlpMemMapDataU8Rw, pCallEntry->idxInstr)
5851
5852	#define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5853	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
5854	IEM_ACCESS_DATA_W, 0 /fAlignMask/, \
5855	(uintptr_t)iemNativeHlpMemMapDataU8Wo, pCallEntry->idxInstr) \
5856
5857	#define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5858	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
5859	IEM_ACCESS_DATA_R, 0 /fAlignMask/, \
5860	(uintptr_t)iemNativeHlpMemMapDataU8Ro, pCallEntry->idxInstr)
5861
5862
5863	#define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5864	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
5865	IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1 /fAlignMask/, \
5866	(uintptr_t)iemNativeHlpMemMapDataU16Atomic, pCallEntry->idxInstr)
5867
5868	#define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5869	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
5870	IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1 /fAlignMask/, \
5871	(uintptr_t)iemNativeHlpMemMapDataU16Rw, pCallEntry->idxInstr)
5872
5873	#define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5874	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
5875	IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /fAlignMask/, \
5876	(uintptr_t)iemNativeHlpMemMapDataU16Wo, pCallEntry->idxInstr) \
5877
5878	#define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5879	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
5880	IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1 /fAlignMask/, \
5881	(uintptr_t)iemNativeHlpMemMapDataU16Ro, pCallEntry->idxInstr)
5882
5883	#define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5884	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int16_t), \
5885	IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /fAlignMask/, \
5886	(uintptr_t)iemNativeHlpMemMapDataU16Wo, pCallEntry->idxInstr) \
5887
5888
5889	#define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5890	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
5891	IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1 /fAlignMask/, \
5892	(uintptr_t)iemNativeHlpMemMapDataU32Atomic, pCallEntry->idxInstr)
5893
5894	#define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5895	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
5896	IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1 /fAlignMask/, \
5897	(uintptr_t)iemNativeHlpMemMapDataU32Rw, pCallEntry->idxInstr)
5898
5899	#define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5900	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
5901	IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /fAlignMask/, \
5902	(uintptr_t)iemNativeHlpMemMapDataU32Wo, pCallEntry->idxInstr) \
5903
5904	#define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5905	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
5906	IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1 /fAlignMask/, \
5907	(uintptr_t)iemNativeHlpMemMapDataU32Ro, pCallEntry->idxInstr)
5908
5909	#define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5910	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int32_t), \
5911	IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /fAlignMask/, \
5912	(uintptr_t)iemNativeHlpMemMapDataU32Wo, pCallEntry->idxInstr) \
5913
5914
5915	#define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5916	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
5917	IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1 /fAlignMask/, \
5918	(uintptr_t)iemNativeHlpMemMapDataU64Atomic, pCallEntry->idxInstr)
5919
5920	#define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5921	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
5922	IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1 /fAlignMask/, \
5923	(uintptr_t)iemNativeHlpMemMapDataU64Rw, pCallEntry->idxInstr)
5924	#define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5925	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
5926	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
5927	(uintptr_t)iemNativeHlpMemMapDataU64Wo, pCallEntry->idxInstr) \
5928
5929	#define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5930	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
5931	IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1 /fAlignMask/, \
5932	(uintptr_t)iemNativeHlpMemMapDataU64Ro, pCallEntry->idxInstr)
5933
5934	#define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5935	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int64_t), \
5936	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
5937	(uintptr_t)iemNativeHlpMemMapDataU64Wo, pCallEntry->idxInstr) \
5938
5939
5940	#define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5941	off = iemNativeEmitMemMapCommon(pReNative, off, a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTFLOAT80U), \
5942	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
5943	(uintptr_t)iemNativeHlpMemMapDataR80Wo, pCallEntry->idxInstr) \
5944
5945	#define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5946	off = iemNativeEmitMemMapCommon(pReNative, off, a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTFLOAT80U), \
5947	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, /** @todo check BCD align */ \
5948	(uintptr_t)iemNativeHlpMemMapDataD80Wo, pCallEntry->idxInstr) \
5949
5950
5951	#define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5952	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
5953	IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1 /fAlignMask/, \
5954	(uintptr_t)iemNativeHlpMemMapDataU128Atomic, pCallEntry->idxInstr)
5955
5956	#define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5957	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
5958	IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1 /fAlignMask/, \
5959	(uintptr_t)iemNativeHlpMemMapDataU128Rw, pCallEntry->idxInstr)
5960
5961	#define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5962	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
5963	IEM_ACCESS_DATA_W, sizeof(RTUINT128U) - 1 /fAlignMask/, \
5964	(uintptr_t)iemNativeHlpMemMapDataU128Wo, pCallEntry->idxInstr) \
5965
5966	#define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
5967	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
5968	IEM_ACCESS_DATA_R, sizeof(RTUINT128U) - 1 /fAlignMask/, \
5969	(uintptr_t)iemNativeHlpMemMapDataU128Ro, pCallEntry->idxInstr)
5970
5971
5972
5973	#define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
5974	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
5975	IEM_ACCESS_DATA_ATOMIC, 0 /fAlignMask/, \
5976	(uintptr_t)iemNativeHlpMemFlatMapDataU8Atomic, pCallEntry->idxInstr)
5977
5978	#define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
5979	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
5980	IEM_ACCESS_DATA_RW, 0 /fAlignMask/, \
5981	(uintptr_t)iemNativeHlpMemFlatMapDataU8Rw, pCallEntry->idxInstr)
5982
5983	#define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
5984	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
5985	IEM_ACCESS_DATA_W, 0 /fAlignMask/, \
5986	(uintptr_t)iemNativeHlpMemFlatMapDataU8Wo, pCallEntry->idxInstr) \
5987
5988	#define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
5989	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
5990	IEM_ACCESS_DATA_R, 0 /fAlignMask/, \
5991	(uintptr_t)iemNativeHlpMemFlatMapDataU8Ro, pCallEntry->idxInstr)
5992
5993
5994	#define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
5995	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
5996	IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1 /fAlignMask/, \
5997	(uintptr_t)iemNativeHlpMemFlatMapDataU16Atomic, pCallEntry->idxInstr)
5998
5999	#define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
6000	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
6001	IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1 /fAlignMask/, \
6002	(uintptr_t)iemNativeHlpMemFlatMapDataU16Rw, pCallEntry->idxInstr)
6003
6004	#define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
6005	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
6006	IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /fAlignMask/, \
6007	(uintptr_t)iemNativeHlpMemFlatMapDataU16Wo, pCallEntry->idxInstr) \
6008
6009	#define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
6010	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
6011	IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1 /fAlignMask/, \
6012	(uintptr_t)iemNativeHlpMemFlatMapDataU16Ro, pCallEntry->idxInstr)
6013
6014	#define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
6015	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int16_t), \
6016	IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /fAlignMask/, \
6017	(uintptr_t)iemNativeHlpMemFlatMapDataU16Wo, pCallEntry->idxInstr) \
6018
6019
6020	#define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
6021	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
6022	IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1 /fAlignMask/, \
6023	(uintptr_t)iemNativeHlpMemFlatMapDataU32Atomic, pCallEntry->idxInstr)
6024
6025	#define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
6026	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
6027	IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1 /fAlignMask/, \
6028	(uintptr_t)iemNativeHlpMemFlatMapDataU32Rw, pCallEntry->idxInstr)
6029
6030	#define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
6031	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
6032	IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /fAlignMask/, \
6033	(uintptr_t)iemNativeHlpMemFlatMapDataU32Wo, pCallEntry->idxInstr) \
6034
6035	#define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
6036	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
6037	IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1 /fAlignMask/, \
6038	(uintptr_t)iemNativeHlpMemFlatMapDataU32Ro, pCallEntry->idxInstr)
6039
6040	#define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
6041	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int32_t), \
6042	IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /fAlignMask/, \
6043	(uintptr_t)iemNativeHlpMemFlatMapDataU32Wo, pCallEntry->idxInstr) \
6044
6045
6046	#define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
6047	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
6048	IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1 /fAlignMask/, \
6049	(uintptr_t)iemNativeHlpMemFlatMapDataU64Atomic, pCallEntry->idxInstr)
6050
6051	#define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
6052	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
6053	IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1 /fAlignMask/, \
6054	(uintptr_t)iemNativeHlpMemFlatMapDataU64Rw, pCallEntry->idxInstr)
6055
6056	#define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
6057	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
6058	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
6059	(uintptr_t)iemNativeHlpMemFlatMapDataU64Wo, pCallEntry->idxInstr) \
6060
6061	#define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
6062	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
6063	IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1 /fAlignMask/, \
6064	(uintptr_t)iemNativeHlpMemFlatMapDataU64Ro, pCallEntry->idxInstr)
6065
6066	#define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
6067	off = iemNativeEmitMemMapCommon(pReNative, off, a_pi64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int64_t), \
6068	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
6069	(uintptr_t)iemNativeHlpMemFlatMapDataU64Wo, pCallEntry->idxInstr) \
6070
6071
6072	#define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
6073	off = iemNativeEmitMemMapCommon(pReNative, off, a_pr80Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTFLOAT80U), \
6074	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, \
6075	(uintptr_t)iemNativeHlpMemFlatMapDataR80Wo, pCallEntry->idxInstr) \
6076
6077	#define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
6078	off = iemNativeEmitMemMapCommon(pReNative, off, a_pd80Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTFLOAT80U), \
6079	IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /fAlignMask/, /** @todo check BCD align */ \
6080	(uintptr_t)iemNativeHlpMemFlatMapDataD80Wo, pCallEntry->idxInstr) \
6081
6082
6083	#define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
6084	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
6085	IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1 /fAlignMask/, \
6086	(uintptr_t)iemNativeHlpMemFlatMapDataU128Atomic, pCallEntry->idxInstr)
6087
6088	#define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
6089	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
6090	IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1 /fAlignMask/, \
6091	(uintptr_t)iemNativeHlpMemFlatMapDataU128Rw, pCallEntry->idxInstr)
6092
6093	#define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
6094	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
6095	IEM_ACCESS_DATA_W, sizeof(RTUINT128U) - 1 /fAlignMask/, \
6096	(uintptr_t)iemNativeHlpMemFlatMapDataU128Wo, pCallEntry->idxInstr) \
6097
6098	#define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
6099	off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
6100	IEM_ACCESS_DATA_R, sizeof(RTUINT128U) - 1 /fAlignMask/, \
6101	(uintptr_t)iemNativeHlpMemFlatMapDataU128Ro, pCallEntry->idxInstr)
6102
6103
6104	DECL_INLINE_THROW(uint32_t)
6105	iemNativeEmitMemMapCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarMem, uint8_t idxVarUnmapInfo,
6106	uint8_t iSegReg, uint8_t idxVarGCPtrMem, uint8_t cbMem, uint32_t fAccess, uint8_t fAlignMask,
6107	uintptr_t pfnFunction, uint8_t idxInstr)
6108	{
6109	/*
6110	* Assert sanity.
6111	*/
6112	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarMem);
6113	PIEMNATIVEVAR const pVarMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarMem)];
6114	AssertStmt( pVarMem->enmKind == kIemNativeVarKind_Invalid
6115	&& pVarMem->cbVar == sizeof(void *),
6116	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6117
6118	PIEMNATIVEVAR const pVarUnmapInfo = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarUnmapInfo)];
6119	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarUnmapInfo);
6120	AssertStmt( pVarUnmapInfo->enmKind == kIemNativeVarKind_Invalid
6121	&& pVarUnmapInfo->cbVar == sizeof(uint8_t),
6122	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6123
6124	PIEMNATIVEVAR const pVarGCPtrMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarGCPtrMem)];
6125	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarGCPtrMem);
6126	AssertStmt( pVarGCPtrMem->enmKind == kIemNativeVarKind_Immediate
6127	\|\| pVarGCPtrMem->enmKind == kIemNativeVarKind_Stack,
6128	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6129
6130	Assert(iSegReg < 6 \|\| iSegReg == UINT8_MAX);
6131
6132	AssertCompile(IEMNATIVE_CALL_ARG_GREG_COUNT >= 4);
6133
6134	#ifdef VBOX_STRICT
6135	# define IEM_MAP_HLP_FN_NO_AT(a_fAccess, a_fnBase) \
6136	( ((a_fAccess) & (IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_ATOMIC)) == (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ) \
6137	? (uintptr_t)RT_CONCAT(a_fnBase,Rw) \
6138	: ((a_fAccess) & (IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_ATOMIC)) == IEM_ACCESS_TYPE_READ \
6139	? (uintptr_t)RT_CONCAT(a_fnBase,Ro) : (uintptr_t)RT_CONCAT(a_fnBase,Wo) )
6140	# define IEM_MAP_HLP_FN(a_fAccess, a_fnBase) \
6141	( ((a_fAccess) & (IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_ATOMIC)) == (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ \| IEM_ACCESS_ATOMIC) \
6142	? (uintptr_t)RT_CONCAT(a_fnBase,Atomic) \
6143	: IEM_MAP_HLP_FN_NO_AT(a_fAccess, a_fnBase) )
6144
6145	if (iSegReg == UINT8_MAX)
6146	{
6147	Assert( (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT
6148	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT
6149	\|\| (pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT);
6150	switch (cbMem)
6151	{
6152	case 1: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU8)); break;
6153	case 2: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU16)); break;
6154	case 4: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU32)); break;
6155	case 8: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU64)); break;
6156	case 10:
6157	Assert( pfnFunction == (uintptr_t)iemNativeHlpMemFlatMapDataR80Wo
6158	\|\| pfnFunction == (uintptr_t)iemNativeHlpMemFlatMapDataD80Wo);
6159	Assert((fAccess & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);
6160	break;
6161	case 16: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU128)); break;
6162	# if 0
6163	case 32: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemFlatMapDataU256)); break;
6164	case 64: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemFlatMapDataU512)); break;
6165	# endif
6166	default: AssertFailed(); break;
6167	}
6168	}
6169	else
6170	{
6171	Assert(iSegReg < 6);
6172	switch (cbMem)
6173	{
6174	case 1: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU8)); break;
6175	case 2: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU16)); break;
6176	case 4: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU32)); break;
6177	case 8: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU64)); break;
6178	case 10:
6179	Assert( pfnFunction == (uintptr_t)iemNativeHlpMemMapDataR80Wo
6180	\|\| pfnFunction == (uintptr_t)iemNativeHlpMemMapDataD80Wo);
6181	Assert((fAccess & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);
6182	break;
6183	case 16: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU128)); break;
6184	# if 0
6185	case 32: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemMapDataU256)); break;
6186	case 64: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemMapDataU512)); break;
6187	# endif
6188	default: AssertFailed(); break;
6189	}
6190	}
6191	# undef IEM_MAP_HLP_FN
6192	# undef IEM_MAP_HLP_FN_NO_AT
6193	#endif
6194
6195	#ifdef VBOX_STRICT
6196	/*
6197	* Check that the fExec flags we've got make sense.
6198	*/
6199	off = iemNativeEmitExecFlagsCheck(pReNative, off, pReNative->fExec);
6200	#endif
6201
6202	/*
6203	* To keep things simple we have to commit any pending writes first as we
6204	* may end up making calls.
6205	*/
6206	off = iemNativeRegFlushPendingWrites(pReNative, off);
6207
6208	#ifdef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
6209	/*
6210	* Move/spill/flush stuff out of call-volatile registers.
6211	* This is the easy way out. We could contain this to the tlb-miss branch
6212	* by saving and restoring active stuff here.
6213	*/
6214	/** @todo save+restore active registers and maybe guest shadows in tlb-miss. */
6215	off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /* vacate all non-volatile regs */);
6216	#endif
6217
6218	/* The bUnmapInfo variable will get a register in the tlb-hit code path,
6219	while the tlb-miss codepath will temporarily put it on the stack.
6220	Set the the type to stack here so we don't need to do it twice below. */
6221	iemNativeVarSetKindToStack(pReNative, idxVarUnmapInfo);
6222	uint8_t const idxRegUnmapInfo = iemNativeVarRegisterAcquire(pReNative, idxVarUnmapInfo, &off);
6223	/** @todo use a tmp register from TlbState, since they'll be free after tlb
6224	* lookup is done. */
6225
6226	/*
6227	* Define labels and allocate the result register (trying for the return
6228	* register if we can).
6229	*/
6230	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
6231	uint8_t const idxRegMemResult = !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
6232	? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarMem, IEMNATIVE_CALL_RET_GREG, &off)
6233	: iemNativeVarRegisterAcquire(pReNative, idxVarMem, &off);
6234	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem);
6235	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
6236	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
6237	: UINT32_MAX;
6238	//off=iemNativeEmitBrk(pReNative, off, 0);
6239	/*
6240	* Jump to the TLB lookup code.
6241	*/
6242	if (!TlbState.fSkip)
6243	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
6244
6245	/*
6246	* TlbMiss:
6247	*
6248	* Call helper to do the fetching.
6249	* We flush all guest register shadow copies here.
6250	*/
6251	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
6252
6253	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
6254	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
6255	#else
6256	RT_NOREF(idxInstr);
6257	#endif
6258
6259	#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
6260	/* Save variables in volatile registers. */
6261	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegMemResult) \| RT_BIT_32(idxRegUnmapInfo);
6262	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
6263	#endif
6264
6265	/* IEMNATIVE_CALL_ARG2_GREG = GCPtrMem - load first as it is from a variable. */
6266	off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, idxVarGCPtrMem, 0 /cbAppend/,
6267	#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
6268	IEMNATIVE_CALL_VOLATILE_GREG_MASK, true /fSpilledVarsInvolatileRegs/);
6269	#else
6270	IEMNATIVE_CALL_VOLATILE_GREG_MASK);
6271	#endif
6272
6273	/* IEMNATIVE_CALL_ARG3_GREG = iSegReg */
6274	if (iSegReg != UINT8_MAX)
6275	{
6276	AssertStmt(iSegReg < 6, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_BAD_SEG_REG_NO));
6277	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG3_GREG, iSegReg);
6278	}
6279
6280	/* IEMNATIVE_CALL_ARG1_GREG = &idxVarUnmapInfo; stackslot address, load any register with result after the call. */
6281	int32_t const offBpDispVarUnmapInfo = iemNativeStackCalcBpDisp(iemNativeVarGetStackSlot(pReNative, idxVarUnmapInfo));
6282	off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offBpDispVarUnmapInfo);
6283
6284	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
6285	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
6286
6287	/* Done setting up parameters, make the call. */
6288	off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
6289
6290	/*
6291	* Put the output in the right registers.
6292	*/
6293	Assert(idxRegMemResult == pVarMem->idxReg);
6294	if (idxRegMemResult != IEMNATIVE_CALL_RET_GREG)
6295	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
6296
6297	#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
6298	/* Restore variables and guest shadow registers to volatile registers. */
6299	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
6300	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
6301	#endif
6302
6303	Assert(pVarUnmapInfo->idxReg == idxRegUnmapInfo);
6304	off = iemNativeEmitLoadGprByBpU8(pReNative, off, idxRegUnmapInfo, offBpDispVarUnmapInfo);
6305
6306	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
6307	if (!TlbState.fSkip)
6308	{
6309	/* end of tlbsmiss - Jump to the done label. */
6310	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
6311	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
6312
6313	/*
6314	* TlbLookup:
6315	*/
6316	off = iemNativeEmitTlbLookup<true>(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask, fAccess,
6317	idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
6318	# ifdef VBOX_WITH_STATISTICS
6319	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
6320	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForMapped));
6321	# endif
6322
6323	/* [idxVarUnmapInfo] = 0; */
6324	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegUnmapInfo, 0);
6325
6326	/*
6327	* TlbDone:
6328	*/
6329	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
6330
6331	TlbState.freeRegsAndReleaseVars(pReNative, idxVarGCPtrMem);
6332
6333	# ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
6334	/* Temp Hack: Flush all guest shadows in volatile registers in case of TLB miss. */
6335	iemNativeRegFlushGuestShadowsByHostMask(pReNative, IEMNATIVE_CALL_VOLATILE_GREG_MASK);
6336	# endif
6337	}
6338	#else
6339	RT_NOREF(fAccess, fAlignMask, idxLabelTlbMiss);
6340	#endif
6341
6342	iemNativeVarRegisterRelease(pReNative, idxVarUnmapInfo);
6343	iemNativeVarRegisterRelease(pReNative, idxVarMem);
6344
6345	return off;
6346	}
6347
6348
6349	#define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) \
6350	off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_ATOMIC, \
6351	(uintptr_t)iemNativeHlpMemCommitAndUnmapAtomic, pCallEntry->idxInstr)
6352
6353	#define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) \
6354	off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_RW, \
6355	(uintptr_t)iemNativeHlpMemCommitAndUnmapRw, pCallEntry->idxInstr)
6356
6357	#define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) \
6358	off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_W, \
6359	(uintptr_t)iemNativeHlpMemCommitAndUnmapWo, pCallEntry->idxInstr)
6360
6361	#define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) \
6362	off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_R, \
6363	(uintptr_t)iemNativeHlpMemCommitAndUnmapRo, pCallEntry->idxInstr)
6364
6365	DECL_INLINE_THROW(uint32_t)
6366	iemNativeEmitMemCommitAndUnmap(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarUnmapInfo,
6367	uint32_t fAccess, uintptr_t pfnFunction, uint8_t idxInstr)
6368	{
6369	/*
6370	* Assert sanity.
6371	*/
6372	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarUnmapInfo);
6373	#if defined(VBOX_STRICT) \|\| defined(RT_ARCH_AMD64)
6374	PIEMNATIVEVAR const pVarUnmapInfo = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarUnmapInfo)];
6375	#endif
6376	Assert(pVarUnmapInfo->enmKind == kIemNativeVarKind_Stack);
6377	Assert( pVarUnmapInfo->idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs)
6378	\|\| pVarUnmapInfo->idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS); /* must be initialized */
6379	#ifdef VBOX_STRICT
6380	switch (fAccess & (IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_ATOMIC))
6381	{
6382	case IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_ATOMIC:
6383	Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapAtomic); break;
6384	case IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE:
6385	Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRw); break;
6386	case IEM_ACCESS_TYPE_WRITE:
6387	Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapWo); break;
6388	case IEM_ACCESS_TYPE_READ:
6389	Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRo); break;
6390	default: AssertFailed();
6391	}
6392	#else
6393	RT_NOREF(fAccess);
6394	#endif
6395
6396	/*
6397	* To keep things simple we have to commit any pending writes first as we
6398	* may end up making calls (there shouldn't be any at this point, so this
6399	* is just for consistency).
6400	*/
6401	/** @todo we could postpone this till we make the call and reload the
6402	* registers after returning from the call. Not sure if that's sensible or
6403	* not, though. */
6404	off = iemNativeRegFlushPendingWrites(pReNative, off);
6405
6406	/*
6407	* Move/spill/flush stuff out of call-volatile registers.
6408	*
6409	* We exclude any register holding the bUnmapInfo variable, as we'll be
6410	* checking it after returning from the call and will free it afterwards.
6411	*/
6412	/** @todo save+restore active registers and maybe guest shadows in miss
6413	* scenario. */
6414	off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /* vacate all non-volatile regs */,
6415	RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(idxVarUnmapInfo)));
6416
6417	/*
6418	* If idxVarUnmapInfo is zero, we can skip all this. Otherwise we'll have
6419	* to call the unmap helper function.
6420	*
6421	* The likelyhood of it being zero is higher than for the TLB hit when doing
6422	* the mapping, as a TLB miss for an well aligned and unproblematic memory
6423	* access should also end up with a mapping that won't need special unmapping.
6424	*/
6425	/** @todo Go over iemMemMapJmp and implement the no-unmap-needed case! That
6426	* should speed up things for the pure interpreter as well when TLBs
6427	* are enabled. */
6428	#ifdef RT_ARCH_AMD64
6429	if (pVarUnmapInfo->idxReg == UINT8_MAX)
6430	{
6431	/* test byte [rbp - xxx], 0ffh */
6432	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
6433	pbCodeBuf[off++] = 0xf6;
6434	uint8_t const idxStackSlot = pVarUnmapInfo->idxStackSlot;
6435	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, 0, iemNativeStackCalcBpDisp(idxStackSlot), pReNative);
6436	pbCodeBuf[off++] = 0xff;
6437	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
6438	}
6439	else
6440	#endif
6441	{
6442	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVarUnmapInfo, &off,
6443	true /fInitialized/, IEMNATIVE_CALL_ARG1_GREG /idxRegPref/);
6444	off = iemNativeEmitTestAnyBitsInGpr8(pReNative, off, idxVarReg, 0xff);
6445	iemNativeVarRegisterRelease(pReNative, idxVarUnmapInfo);
6446	}
6447	uint32_t const offJmpFixup = off;
6448	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices*/);
6449
6450	/*
6451	* Call the unmap helper function.
6452	*/
6453	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING /** @todo This should be unnecessary, the mapping call will already have set it! */
6454	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
6455	#else
6456	RT_NOREF(idxInstr);
6457	#endif
6458
6459	/* IEMNATIVE_CALL_ARG1_GREG = idxVarUnmapInfo (first!) */
6460	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxVarUnmapInfo,
6461	0 /offAddend/, IEMNATIVE_CALL_VOLATILE_GREG_MASK);
6462
6463	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
6464	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
6465
6466	/* Done setting up parameters, make the call. */
6467	off = iemNativeEmitCallImm(pReNative, off, pfnFunction);
6468
6469	/* The bUnmapInfo variable is implictly free by these MCs. */
6470	iemNativeVarFreeLocal(pReNative, idxVarUnmapInfo);
6471
6472	/*
6473	* Done, just fixup the jump for the non-call case.
6474	*/
6475	iemNativeFixupFixedJump(pReNative, offJmpFixup, off);
6476
6477	return off;
6478	}
6479
6480
6481
6482	/*********************************************************************************************************************************
6483	* State and Exceptions *
6484	*********************************************************************************************************************************/
6485
6486	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() off = iemNativeEmitPrepareFpuForUse(pReNative, off, true /fForChange/)
6487	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() off = iemNativeEmitPrepareFpuForUse(pReNative, off, false /fForChange/)
6488
6489	#define IEM_MC_PREPARE_SSE_USAGE() off = iemNativeEmitPrepareFpuForUse(pReNative, off, true /fForChange/)
6490	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() off = iemNativeEmitPrepareFpuForUse(pReNative, off, true /fForChange/)
6491	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() off = iemNativeEmitPrepareFpuForUse(pReNative, off, false /fForChange/)
6492
6493	#define IEM_MC_PREPARE_AVX_USAGE() off = iemNativeEmitPrepareFpuForUse(pReNative, off, true /fForChange/)
6494	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() off = iemNativeEmitPrepareFpuForUse(pReNative, off, true /fForChange/)
6495	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() off = iemNativeEmitPrepareFpuForUse(pReNative, off, false /fForChange/)
6496
6497
6498	DECL_INLINE_THROW(uint32_t) iemNativeEmitPrepareFpuForUse(PIEMRECOMPILERSTATE pReNative, uint32_t off, bool fForChange)
6499	{
6500	/** @todo this needs a lot more work later. */
6501	RT_NOREF(pReNative, fForChange);
6502	return off;
6503	}
6504
6505
6506
6507	/*********************************************************************************************************************************
6508	* Emitters for FPU related operations. *
6509	*********************************************************************************************************************************/
6510
6511	#define IEM_MC_FETCH_FCW(a_u16Fcw) \
6512	off = iemNativeEmitFetchFpuFcw(pReNative, off, a_u16Fcw)
6513
6514	/** Emits code for IEM_MC_FETCH_FCW. */
6515	DECL_INLINE_THROW(uint32_t)
6516	iemNativeEmitFetchFpuFcw(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar)
6517	{
6518	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6519	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint16_t));
6520
6521	uint8_t const idxReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6522
6523	/* Allocate a temporary FCW register. */
6524	/** @todo eliminate extra register */
6525	uint8_t const idxFcwReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFcw,
6526	kIemNativeGstRegUse_ReadOnly);
6527
6528	off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxReg, idxFcwReg);
6529
6530	/* Free but don't flush the FCW register. */
6531	iemNativeRegFreeTmp(pReNative, idxFcwReg);
6532	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6533
6534	return off;
6535	}
6536
6537
6538	#define IEM_MC_FETCH_FSW(a_u16Fsw) \
6539	off = iemNativeEmitFetchFpuFsw(pReNative, off, a_u16Fsw)
6540
6541	/** Emits code for IEM_MC_FETCH_FSW. */
6542	DECL_INLINE_THROW(uint32_t)
6543	iemNativeEmitFetchFpuFsw(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar)
6544	{
6545	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6546	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint16_t));
6547
6548	uint8_t const idxReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off, false /fInitialized/);
6549	/* Allocate a temporary FSW register. */
6550	/** @todo eliminate extra register */
6551	uint8_t const idxFswReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFsw,
6552	kIemNativeGstRegUse_ReadOnly);
6553
6554	off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxReg, idxFswReg);
6555
6556	/* Free but don't flush the FSW register. */
6557	iemNativeRegFreeTmp(pReNative, idxFswReg);
6558	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6559
6560	return off;
6561	}
6562
6563
6564
6565	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
6566
6567
6568	/*********************************************************************************************************************************
6569	* Emitters for SSE/AVX specific operations. *
6570	*********************************************************************************************************************************/
6571
6572	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
6573	off = iemNativeEmitSimdCopyXregU128(pReNative, off, a_iXRegDst, a_iXRegSrc)
6574
6575	/** Emits code for IEM_MC_COPY_XREG_U128. */
6576	DECL_INLINE_THROW(uint32_t)
6577	iemNativeEmitSimdCopyXregU128(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXRegDst, uint8_t iXRegSrc)
6578	{
6579	/* Allocate destination and source register. */
6580	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXRegDst),
6581	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForFullWrite);
6582	uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXRegSrc),
6583	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
6584
6585	off = iemNativeEmitSimdLoadVecRegFromVecRegU128(pReNative, off, idxSimdRegDst, idxSimdRegSrc);
6586	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXRegDst);
6587	/* We don't need to write everything back here as the destination is marked as dirty and will be flushed automatically. */
6588
6589	/* Free but don't flush the source and destination register. */
6590	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6591	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrc);
6592
6593	return off;
6594	}
6595
6596
6597	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
6598	off = iemNativeEmitSimdFetchXregU64(pReNative, off, a_u64Value, a_iXReg, a_iQWord)
6599
6600	/** Emits code for IEM_MC_FETCH_XREG_U64. */
6601	DECL_INLINE_THROW(uint32_t)
6602	iemNativeEmitSimdFetchXregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iXReg, uint8_t iQWord)
6603	{
6604	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6605	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
6606
6607	uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6608	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
6609
6610	iemNativeVarSetKindToStack(pReNative, idxDstVar);
6611	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6612
6613	off = iemNativeEmitSimdLoadGprFromVecRegU64(pReNative, off, idxVarReg, idxSimdRegSrc, iQWord);
6614
6615	/* Free but don't flush the source register. */
6616	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrc);
6617	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6618
6619	return off;
6620	}
6621
6622
6623	#define IEM_MC_FETCH_XREG_U32(a_u64Value, a_iXReg, a_iDWord) \
6624	off = iemNativeEmitSimdFetchXregU32(pReNative, off, a_u64Value, a_iXReg, a_iDWord)
6625
6626	/** Emits code for IEM_MC_FETCH_XREG_U32. */
6627	DECL_INLINE_THROW(uint32_t)
6628	iemNativeEmitSimdFetchXregU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iXReg, uint8_t iDWord)
6629	{
6630	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6631	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint32_t));
6632
6633	uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6634	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
6635
6636	iemNativeVarSetKindToStack(pReNative, idxDstVar);
6637	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6638
6639	off = iemNativeEmitSimdLoadGprFromVecRegU32(pReNative, off, idxVarReg, idxSimdRegSrc, iDWord);
6640
6641	/* Free but don't flush the source register. */
6642	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrc);
6643	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6644
6645	return off;
6646	}
6647
6648
6649	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQWord, a_u64Value) \
6650	off = iemNativeEmitSimdStoreXregU64(pReNative, off, a_iXReg, a_u64Value, a_iQWord)
6651
6652	/** Emits code for IEM_MC_STORE_XREG_U64. */
6653	DECL_INLINE_THROW(uint32_t)
6654	iemNativeEmitSimdStoreXregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxDstVar, uint8_t iQWord)
6655	{
6656	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6657	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
6658
6659	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6660	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
6661
6662	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6663
6664	off = iemNativeEmitSimdStoreGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, iQWord);
6665	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6666
6667	/* Free but don't flush the source register. */
6668	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6669	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6670
6671	return off;
6672	}
6673
6674
6675	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDWord, a_u32Value) \
6676	off = iemNativeEmitSimdStoreXregU32(pReNative, off, a_iXReg, a_u32Value, a_iDWord)
6677
6678	/** Emits code for IEM_MC_STORE_XREG_U32. */
6679	DECL_INLINE_THROW(uint32_t)
6680	iemNativeEmitSimdStoreXregU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxDstVar, uint8_t iDWord)
6681	{
6682	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6683	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint32_t));
6684
6685	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6686	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
6687
6688	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6689
6690	off = iemNativeEmitSimdStoreGprToVecRegU32(pReNative, off, idxSimdRegDst, idxVarReg, iDWord);
6691	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6692
6693	/* Free but don't flush the source register. */
6694	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6695	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6696
6697	return off;
6698	}
6699
6700
6701	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
6702	off = iemNativeEmitSimdStoreXregU64ZxU128(pReNative, off, a_iXReg, a_u64Value)
6703
6704	/** Emits code for IEM_MC_STORE_XREG_U32. */
6705	DECL_INLINE_THROW(uint32_t)
6706	iemNativeEmitSimdStoreXregU64ZxU128(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxDstVar)
6707	{
6708	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6709	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
6710
6711	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6712	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
6713
6714	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6715
6716	/* Zero the vector register first, then store the 64-bit value to the lower 64-bit. */
6717	off = iemNativeEmitSimdZeroVecRegLowU128(pReNative, off, idxSimdRegDst);
6718	off = iemNativeEmitSimdStoreGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, 0);
6719	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6720
6721	/* Free but don't flush the source register. */
6722	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6723	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6724
6725	return off;
6726	}
6727
6728
6729	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
6730	off = iemNativeEmitSimdStoreXregU32ZxU128(pReNative, off, a_iXReg, a_u32Value)
6731
6732	/** Emits code for IEM_MC_STORE_XREG_U32. */
6733	DECL_INLINE_THROW(uint32_t)
6734	iemNativeEmitSimdStoreXregU32ZxU128(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxDstVar)
6735	{
6736	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6737	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint32_t));
6738
6739	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6740	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
6741
6742	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6743
6744	/* Zero the vector register first, then store the 32-bit value to the lowest 32-bit element. */
6745	off = iemNativeEmitSimdZeroVecRegLowU128(pReNative, off, idxSimdRegDst);
6746	off = iemNativeEmitSimdStoreGprToVecRegU32(pReNative, off, idxSimdRegDst, idxVarReg, 0);
6747	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6748
6749	/* Free but don't flush the source register. */
6750	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6751	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6752
6753	return off;
6754	}
6755
6756
6757	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
6758	off = iemNativeEmitSimdFetchYregU64(pReNative, off, a_u64Dst, a_iYRegSrc, a_iQWord)
6759
6760	/** Emits code for IEM_MC_FETCH_YREG_U64. */
6761	DECL_INLINE_THROW(uint32_t)
6762	iemNativeEmitSimdFetchYregU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iYReg, uint8_t iQWord)
6763	{
6764	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6765	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
6766
6767	uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6768	iQWord >= 2
6769	? kIemNativeGstSimdRegLdStSz_High128
6770	: kIemNativeGstSimdRegLdStSz_Low128,
6771	kIemNativeGstRegUse_ReadOnly);
6772
6773	iemNativeVarSetKindToStack(pReNative, idxDstVar);
6774	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6775
6776	off = iemNativeEmitSimdLoadGprFromVecRegU64(pReNative, off, idxVarReg, idxSimdRegSrc, iQWord);
6777
6778	/* Free but don't flush the source register. */
6779	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrc);
6780	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6781
6782	return off;
6783	}
6784
6785
6786	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
6787	off = iemNativeEmitSimdFetchYregU32(pReNative, off, a_u32Dst, a_iYRegSrc, 0)
6788
6789	/** Emits code for IEM_MC_FETCH_YREG_U32. */
6790	DECL_INLINE_THROW(uint32_t)
6791	iemNativeEmitSimdFetchYregU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iYReg, uint8_t iDWord)
6792	{
6793	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
6794	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint32_t));
6795
6796	uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6797	iDWord >= 4
6798	? kIemNativeGstSimdRegLdStSz_High128
6799	: kIemNativeGstSimdRegLdStSz_Low128,
6800	kIemNativeGstRegUse_ReadOnly);
6801
6802	iemNativeVarSetKindToStack(pReNative, idxDstVar);
6803	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off);
6804
6805	off = iemNativeEmitSimdLoadGprFromVecRegU32(pReNative, off, idxVarReg, idxSimdRegSrc, iDWord);
6806
6807	/* Free but don't flush the source register. */
6808	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrc);
6809	iemNativeVarRegisterRelease(pReNative, idxDstVar);
6810
6811	return off;
6812	}
6813
6814
6815	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
6816	off = iemNativeEmitSimdClearYregHighU128(pReNative, off, a_iYReg)
6817
6818	/** Emits code for IEM_MC_CLEAR_YREG_128_UP. */
6819	DECL_INLINE_THROW(uint32_t)
6820	iemNativeEmitSimdClearYregHighU128(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYReg)
6821	{
6822	uint8_t const idxSimdReg = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6823	kIemNativeGstSimdRegLdStSz_High128, kIemNativeGstRegUse_ForFullWrite);
6824
6825	off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdReg);
6826	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYReg);
6827
6828	/* Free but don't flush the register. */
6829	iemNativeSimdRegFreeTmp(pReNative, idxSimdReg);
6830
6831	return off;
6832	}
6833
6834
6835	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
6836	off = iemNativeEmitSimdBroadcastXregU32ZxVlmax(pReNative, off, a_iXRegDst, a_u32Src)
6837
6838	/** Emits code for IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX. */
6839	DECL_INLINE_THROW(uint32_t)
6840	iemNativeEmitSimdBroadcastXregU32ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxSrcVar)
6841	{
6842	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6843	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint32_t));
6844
6845	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6846	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6847
6848	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6849
6850	off = iemNativeEmitSimdBroadcastGprToVecRegU32(pReNative, off, idxSimdRegDst, idxVarReg, false /f256Bit/);
6851	off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdRegDst);
6852	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6853	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iXReg);
6854
6855	/* Free but don't flush the source register. */
6856	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6857	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6858
6859	return off;
6860	}
6861
6862
6863	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
6864	off = iemNativeEmitSimdBroadcastXregU64ZxVlmax(pReNative, off, a_iXRegDst, a_u64Src)
6865
6866	/** Emits code for IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX. */
6867	DECL_INLINE_THROW(uint32_t)
6868	iemNativeEmitSimdBroadcastXregU64ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iXReg, uint8_t idxSrcVar)
6869	{
6870	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6871	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint64_t));
6872
6873	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
6874	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6875
6876	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6877
6878	off = iemNativeEmitSimdBroadcastGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, false /f256Bit/);
6879	off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdRegDst);
6880	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iXReg);
6881	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iXReg);
6882
6883	/* Free but don't flush the source register. */
6884	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6885	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6886
6887	return off;
6888	}
6889
6890
6891	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
6892	off = iemNativeEmitSimdBroadcastYregU32ZxVlmax(pReNative, off, a_iYRegDst, a_u32Src)
6893
6894	/** Emits code for IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX. */
6895	DECL_INLINE_THROW(uint32_t)
6896	iemNativeEmitSimdBroadcastYregU32ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYReg, uint8_t idxSrcVar)
6897	{
6898	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6899	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint32_t));
6900
6901	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6902	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6903
6904	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6905
6906	off = iemNativeEmitSimdBroadcastGprToVecRegU32(pReNative, off, idxSimdRegDst, idxVarReg, true /f256Bit/);
6907	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYReg);
6908	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYReg);
6909
6910	/* Free but don't flush the source register. */
6911	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6912	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6913
6914	return off;
6915	}
6916
6917
6918	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
6919	off = iemNativeEmitSimdBroadcastYregU64ZxVlmax(pReNative, off, a_iYRegDst, a_u64Src)
6920
6921	/** Emits code for IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX. */
6922	DECL_INLINE_THROW(uint32_t)
6923	iemNativeEmitSimdBroadcastYregU64ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYReg, uint8_t idxSrcVar)
6924	{
6925	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6926	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint64_t));
6927
6928	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6929	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6930
6931	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6932
6933	off = iemNativeEmitSimdBroadcastGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, true /f256Bit/);
6934	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYReg);
6935	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYReg);
6936
6937	/* Free but don't flush the source register. */
6938	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6939	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6940
6941	return off;
6942	}
6943
6944
6945	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
6946	off = iemNativeEmitSimdStoreYregU32ZxVlmax(pReNative, off, a_iYRegDst, a_u32Src)
6947
6948	/** Emits code for IEM_MC_STORE_YREG_U32_ZX_VLMAX. */
6949	DECL_INLINE_THROW(uint32_t)
6950	iemNativeEmitSimdStoreYregU32ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYReg, uint8_t idxSrcVar)
6951	{
6952	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6953	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint32_t));
6954
6955	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6956	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6957
6958	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6959
6960	off = iemNativeEmitSimdZeroVecRegU256(pReNative, off, idxSimdRegDst);
6961	off = iemNativeEmitSimdStoreGprToVecRegU32(pReNative, off, idxSimdRegDst, idxVarReg, 0 /iDWord/);
6962	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYReg);
6963	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYReg);
6964
6965	/* Free but don't flush the source register. */
6966	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6967	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6968
6969	return off;
6970	}
6971
6972
6973	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
6974	off = iemNativeEmitSimdStoreYregU64ZxVlmax(pReNative, off, a_iYRegDst, a_u64Src)
6975
6976	/** Emits code for IEM_MC_STORE_YREG_U64_ZX_VLMAX. */
6977	DECL_INLINE_THROW(uint32_t)
6978	iemNativeEmitSimdStoreYregU64ZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYReg, uint8_t idxSrcVar)
6979	{
6980	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
6981	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint64_t));
6982
6983	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
6984	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
6985
6986	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
6987
6988	off = iemNativeEmitSimdZeroVecRegU256(pReNative, off, idxSimdRegDst);
6989	off = iemNativeEmitSimdStoreGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, 0 /iQWord/);
6990	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYReg);
6991	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYReg);
6992
6993	/* Free but don't flush the source register. */
6994	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
6995	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
6996
6997	return off;
6998	}
6999
7000
7001	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
7002	off = iemNativeEmitSimdMergeYregU64LocalU64HiZxVlmax(pReNative, off, a_iYRegDst, a_u64Local, a_iYRegSrcHx)
7003
7004	/** Emits code for IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX. */
7005	DECL_INLINE_THROW(uint32_t)
7006	iemNativeEmitSimdMergeYregU64LocalU64HiZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYRegDst, uint8_t idxSrcVar, uint8_t iYRegSrcHx)
7007	{
7008	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
7009	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint64_t));
7010
7011	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
7012	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
7013	uint8_t const idxSimdRegSrcHx = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrcHx),
7014	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
7015	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
7016
7017	off = iemNativeEmitSimdLoadVecRegFromVecRegU128(pReNative, off, idxSimdRegDst, idxSimdRegSrcHx);
7018	off = iemNativeEmitSimdStoreGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, 0 /iQWord/);
7019	off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdRegDst);
7020	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYRegDst);
7021	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYRegDst);
7022
7023	/* Free but don't flush the source and destination registers. */
7024	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrcHx);
7025	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
7026	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
7027
7028	return off;
7029	}
7030
7031
7032	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
7033	off = iemNativeEmitSimdMergeYregU64LoU64LocalZxVlmax(pReNative, off, a_iYRegDst, a_iYRegSrcHx, a_u64Local)
7034
7035	/** Emits code for IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX. */
7036	DECL_INLINE_THROW(uint32_t)
7037	iemNativeEmitSimdMergeYregU64LoU64LocalZxVlmax(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iYRegDst, uint8_t iYRegSrcHx, uint8_t idxSrcVar)
7038	{
7039	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxSrcVar);
7040	IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxSrcVar, sizeof(uint64_t));
7041
7042	uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
7043	kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
7044	uint8_t const idxSimdRegSrcHx = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrcHx),
7045	kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
7046	uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
7047
7048	off = iemNativeEmitSimdLoadVecRegFromVecRegU128(pReNative, off, idxSimdRegDst, idxSimdRegSrcHx);
7049	off = iemNativeEmitSimdStoreGprToVecRegU64(pReNative, off, idxSimdRegDst, idxVarReg, 1 /iQWord/);
7050	off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdRegDst);
7051	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(pReNative, iYRegDst);
7052	IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(pReNative, iYRegDst);
7053
7054	/* Free but don't flush the source and destination registers. */
7055	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegSrcHx);
7056	iemNativeSimdRegFreeTmp(pReNative, idxSimdRegDst);
7057	iemNativeVarRegisterRelease(pReNative, idxSrcVar);
7058
7059	return off;
7060	}
7061
7062	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
7063
7064
7065	/*********************************************************************************************************************************
7066	* Include instruction emitters. *
7067	*********************************************************************************************************************************/
7068	#include "target-x86/IEMAllN8veEmit-x86.h"
7069

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h@ 103827

Download in other formats: