IEMN8veRecompiler.h@ 101722

Last change on this file since 101722 was 101705, checked in by vboxsync, 15 months ago
VMM/IEM: Fixed incorrect rex prefix for the AND emitters. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 114.3 KB

Line
1	/* $Id: IEMN8veRecompiler.h 101705 2023-11-02 00:28:10Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46
47	/** @name Stack Frame Layout
48	*
49	* @{ */
50	/** The size of the area for stack variables and spills and stuff.
51	* @note This limit is duplicated in the python script(s). */
52	#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
53	#ifdef RT_ARCH_AMD64
54	/** Number of stack arguments slots for calls made from the frame. */
55	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
56	/** An stack alignment adjustment (between non-volatile register pushes and
57	* the stack variable area, so the latter better aligned). */
58	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
59	/** Number of any shadow arguments (spill area) for calls we make. */
60	# ifdef RT_OS_WINDOWS
61	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
62	# else
63	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
64	# endif
65
66	/** Frame pointer (RBP) relative offset of the last push. */
67	# ifdef RT_OS_WINDOWS
68	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
69	# else
70	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
71	# endif
72	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
73	* address for it). */
74	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
75	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
76	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
77	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
78	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
79	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
80	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
81	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
82	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
83
84	# ifdef RT_OS_WINDOWS
85	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
86	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
87	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
88	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
89	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
90	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
91	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
92	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
93	# endif
94
95	#elif RT_ARCH_ARM64
96	/** No stack argument slots, enough got 8 registers for arguments. */
97	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
98	/** There are no argument spill area. */
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100
101	/** Number of saved registers at the top of our stack frame.
102	* This includes the return address and old frame pointer, so x19 thru x30. */
103	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
104	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
105	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
106
107	/** Frame pointer (BP) relative offset of the last push. */
108	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
109
110	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
111	* address for it). */
112	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
113
114	#else
115	# error "port me"
116	#endif
117	/** @} */
118
119
120	/** @name Fixed Register Allocation(s)
121	* @{ */
122	/** @def IEMNATIVE_REG_FIXED_PVMCPU
123	* The number of the register holding the pVCpu pointer. */
124	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
125	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
126	* @note This not available on AMD64, only ARM64. */
127	/** @def IEMNATIVE_REG_FIXED_TMP0
128	* Dedicated temporary register.
129	* @todo replace this by a register allocator and content tracker. */
130	/** @def IEMNATIVE_REG_FIXED_MASK
131	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
132	* architecture. */
133	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
134	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
135	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
136	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
137	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
138	\| RT_BIT_32(X86_GREG_xSP) \
139	\| RT_BIT_32(X86_GREG_xBP) )
140
141	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
142	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
143	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
144	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
145	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
146	\| RT_BIT_32(ARMV8_A64_REG_LR) \
147	\| RT_BIT_32(ARMV8_A64_REG_BP) \
148	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
149	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
150	\| RT_BIT_32(ARMV8_A64_REG_X18) \
151	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
152
153	#else
154	# error "port me"
155	#endif
156	/** @} */
157
158	/** @name Call related registers.
159	* @{ */
160	/** @def IEMNATIVE_CALL_RET_GREG
161	* The return value register. */
162	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
163	* Number of arguments in registers. */
164	/** @def IEMNATIVE_CALL_ARG0_GREG
165	* The general purpose register carrying argument \#0. */
166	/** @def IEMNATIVE_CALL_ARG1_GREG
167	* The general purpose register carrying argument \#1. */
168	/** @def IEMNATIVE_CALL_ARG2_GREG
169	* The general purpose register carrying argument \#2. */
170	/** @def IEMNATIVE_CALL_ARG3_GREG
171	* The general purpose register carrying argument \#3. */
172	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
173	* Mask of registers the callee will not save and may trash. */
174	#ifdef RT_ARCH_AMD64
175	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
176
177	# ifdef RT_OS_WINDOWS
178	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
179	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
180	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
181	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
182	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
183	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
184	\| RT_BIT_32(X86_GREG_xCX) \
185	\| RT_BIT_32(X86_GREG_xDX) \
186	\| RT_BIT_32(X86_GREG_x8) \
187	\| RT_BIT_32(X86_GREG_x9) \
188	\| RT_BIT_32(X86_GREG_x10) \
189	\| RT_BIT_32(X86_GREG_x11) )
190	# else
191	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
192	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
193	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
194	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
195	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
196	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
197	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
198	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
199	\| RT_BIT_32(X86_GREG_xCX) \
200	\| RT_BIT_32(X86_GREG_xDX) \
201	\| RT_BIT_32(X86_GREG_xDI) \
202	\| RT_BIT_32(X86_GREG_xSI) \
203	\| RT_BIT_32(X86_GREG_x8) \
204	\| RT_BIT_32(X86_GREG_x9) \
205	\| RT_BIT_32(X86_GREG_x10) \
206	\| RT_BIT_32(X86_GREG_x11) )
207	# endif
208
209	#elif defined(RT_ARCH_ARM64)
210	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
211	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
212	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
213	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
214	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
215	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
216	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
217	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
218	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
219	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
220	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
221	\| RT_BIT_32(ARMV8_A64_REG_X1) \
222	\| RT_BIT_32(ARMV8_A64_REG_X2) \
223	\| RT_BIT_32(ARMV8_A64_REG_X3) \
224	\| RT_BIT_32(ARMV8_A64_REG_X4) \
225	\| RT_BIT_32(ARMV8_A64_REG_X5) \
226	\| RT_BIT_32(ARMV8_A64_REG_X6) \
227	\| RT_BIT_32(ARMV8_A64_REG_X7) \
228	\| RT_BIT_32(ARMV8_A64_REG_X8) \
229	\| RT_BIT_32(ARMV8_A64_REG_X9) \
230	\| RT_BIT_32(ARMV8_A64_REG_X10) \
231	\| RT_BIT_32(ARMV8_A64_REG_X11) \
232	\| RT_BIT_32(ARMV8_A64_REG_X12) \
233	\| RT_BIT_32(ARMV8_A64_REG_X13) \
234	\| RT_BIT_32(ARMV8_A64_REG_X14) \
235	\| RT_BIT_32(ARMV8_A64_REG_X15) \
236	\| RT_BIT_32(ARMV8_A64_REG_X16) \
237	\| RT_BIT_32(ARMV8_A64_REG_X17) )
238
239	#endif
240
241	/** @} */
242
243
244	/** @def IEMNATIVE_HST_GREG_COUNT
245	* Number of host general purpose registers we tracker. */
246	/** @def IEMNATIVE_HST_GREG_MASK
247	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
248	* inverted register masks and such to get down to a correct set of regs. */
249	#ifdef RT_ARCH_AMD64
250	# define IEMNATIVE_HST_GREG_COUNT 16
251	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
252
253	#elif defined(RT_ARCH_ARM64)
254	# define IEMNATIVE_HST_GREG_COUNT 32
255	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
256	#else
257	# error "Port me!"
258	#endif
259
260
261	/** Native code generator label types. */
262	typedef enum
263	{
264	kIemNativeLabelType_Invalid = 0,
265	/* Labels w/o data, only once instance per TB: */
266	kIemNativeLabelType_Return,
267	kIemNativeLabelType_ReturnBreak,
268	kIemNativeLabelType_ReturnWithFlags,
269	kIemNativeLabelType_NonZeroRetOrPassUp,
270	kIemNativeLabelType_RaiseGp0,
271	/* Labels with data, potentially multiple instances per TB: */
272	kIemNativeLabelType_If,
273	kIemNativeLabelType_Else,
274	kIemNativeLabelType_Endif,
275	kIemNativeLabelType_CheckIrq,
276	kIemNativeLabelType_End
277	} IEMNATIVELABELTYPE;
278
279	/** Native code generator label definition. */
280	typedef struct IEMNATIVELABEL
281	{
282	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
283	* the epilog. */
284	uint32_t off;
285	/** The type of label (IEMNATIVELABELTYPE). */
286	uint16_t enmType;
287	/** Additional label data, type specific. */
288	uint16_t uData;
289	} IEMNATIVELABEL;
290	/** Pointer to a label. */
291	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
292
293
294	/** Native code generator fixup types. */
295	typedef enum
296	{
297	kIemNativeFixupType_Invalid = 0,
298	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
299	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
300	kIemNativeFixupType_Rel32,
301	#elif defined(RT_ARCH_ARM64)
302	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
303	kIemNativeFixupType_RelImm19At5,
304	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
305	kIemNativeFixupType_RelImm14At5,
306	#endif
307	kIemNativeFixupType_End
308	} IEMNATIVEFIXUPTYPE;
309
310	/** Native code generator fixup. */
311	typedef struct IEMNATIVEFIXUP
312	{
313	/** Code offset of the fixup location. */
314	uint32_t off;
315	/** The IEMNATIVELABEL this is a fixup for. */
316	uint16_t idxLabel;
317	/** The fixup type (IEMNATIVEFIXUPTYPE). */
318	uint8_t enmType;
319	/** Addend or other data. */
320	int8_t offAddend;
321	} IEMNATIVEFIXUP;
322	/** Pointer to a native code generator fixup. */
323	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
324
325
326	/**
327	* Guest registers that can be shadowed in GPRs.
328	*/
329	typedef enum IEMNATIVEGSTREG : uint8_t
330	{
331	kIemNativeGstReg_GprFirst = 0,
332	kIemNativeGstReg_GprLast = 15,
333	kIemNativeGstReg_Pc,
334	kIemNativeGstReg_EFlags, /*< This one is problematic since the higher bits are used internally. /
335	/* gap: 18..23 */
336	kIemNativeGstReg_SegSelFirst = 24,
337	kIemNativeGstReg_SegSelLast = 29,
338	kIemNativeGstReg_SegBaseFirst = 30,
339	kIemNativeGstReg_SegBaseLast = 35,
340	kIemNativeGstReg_SegLimitFirst = 36,
341	kIemNativeGstReg_SegLimitLast = 41,
342	kIemNativeGstReg_End
343	} IEMNATIVEGSTREG;
344
345	/**
346	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
347	*/
348	typedef enum IEMNATIVEGSTREGUSE
349	{
350	/** The usage is read-only, the register holding the guest register
351	* shadow copy will not be modified by the caller. */
352	kIemNativeGstRegUse_ReadOnly = 0,
353	/** The caller will update the guest register (think: PC += cbInstr).
354	* The guest shadow copy will follow the returned register. */
355	kIemNativeGstRegUse_ForUpdate,
356	/** The caller will use the guest register value as input in a calculation
357	* and the host register will be modified.
358	* This means that the returned host register will not be marked as a shadow
359	* copy of the guest register. */
360	kIemNativeGstRegUse_Calculation
361	} IEMNATIVEGSTREGUSE;
362
363	/**
364	* Guest registers (classes) that can be referenced.
365	*/
366	typedef enum IEMNATIVEGSTREGREF : uint8_t
367	{
368	kIemNativeGstRegRef_Invalid = 0,
369	kIemNativeGstRegRef_Gpr,
370	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
371	kIemNativeGstRegRef_EFlags,
372	kIemNativeGstRegRef_MxCsr,
373	kIemNativeGstRegRef_FpuReg,
374	kIemNativeGstRegRef_MReg,
375	kIemNativeGstRegRef_XReg,
376	kIemNativeGstRegRef_YReg,
377	kIemNativeGstRegRef_End
378	} IEMNATIVEGSTREGREF;
379
380
381	/** Variable kinds. */
382	typedef enum IEMNATIVEVARKIND : uint8_t
383	{
384	/** Customary invalid zero value. */
385	kIemNativeVarKind_Invalid = 0,
386	/** This is either in a register or on the stack. */
387	kIemNativeVarKind_Stack,
388	/** Immediate value - loaded into register when needed, or can live on the
389	* stack if referenced (in theory). */
390	kIemNativeVarKind_Immediate,
391	/** Variable reference - loaded into register when needed, never stack. */
392	kIemNativeVarKind_VarRef,
393	/** Guest register reference - loaded into register when needed, never stack. */
394	kIemNativeVarKind_GstRegRef,
395	/** End of valid values. */
396	kIemNativeVarKind_End
397	} IEMNATIVEVARKIND;
398
399
400	/** Variable or argument. */
401	typedef struct IEMNATIVEVAR
402	{
403	/** The kind of variable. */
404	IEMNATIVEVARKIND enmKind;
405	/** The variable size in bytes. */
406	uint8_t cbVar;
407	/** The first stack slot (uint64_t), except for immediate and references
408	* where it usually is UINT8_MAX. */
409	uint8_t idxStackSlot;
410	/** The host register allocated for the variable, UINT8_MAX if not. */
411	uint8_t idxReg;
412	/** The argument number if argument, UINT8_MAX if regular variable. */
413	uint8_t uArgNo;
414	/** If referenced, the index of the variable referencing this one, otherwise
415	* UINT8_MAX. A referenced variable must only be placed on the stack and
416	* must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
417	uint8_t idxReferrerVar;
418	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
419	IEMNATIVEGSTREG enmGstReg;
420	uint8_t bAlign;
421
422	union
423	{
424	/** kIemNativeVarKind_Immediate: The immediate value. */
425	uint64_t uValue;
426	/** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
427	uint8_t idxRefVar;
428	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
429	struct
430	{
431	/** The class of register. */
432	IEMNATIVEGSTREGREF enmClass;
433	/** Index within the class. */
434	uint8_t idx;
435	} GstRegRef;
436	} u;
437	} IEMNATIVEVAR;
438
439	/** What is being kept in a host register. */
440	typedef enum IEMNATIVEWHAT : uint8_t
441	{
442	/** The traditional invalid zero value. */
443	kIemNativeWhat_Invalid = 0,
444	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
445	kIemNativeWhat_Var,
446	/** Temporary register, this is typically freed when a MC completes. */
447	kIemNativeWhat_Tmp,
448	/** Call argument w/o a variable mapping. This is free (via
449	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
450	kIemNativeWhat_Arg,
451	/** Return status code.
452	* @todo not sure if we need this... */
453	kIemNativeWhat_rc,
454	/** The fixed pVCpu (PVMCPUCC) register.
455	* @todo consider offsetting this on amd64 to use negative offsets to access
456	* more members using 8-byte disp. */
457	kIemNativeWhat_pVCpuFixed,
458	/** The fixed pCtx (PCPUMCTX) register.
459	* @todo consider offsetting this on amd64 to use negative offsets to access
460	* more members using 8-byte disp. */
461	kIemNativeWhat_pCtxFixed,
462	/** Fixed temporary register. */
463	kIemNativeWhat_FixedTmp,
464	/** Register reserved by the CPU or OS architecture. */
465	kIemNativeWhat_FixedReserved,
466	/** End of valid values. */
467	kIemNativeWhat_End
468	} IEMNATIVEWHAT;
469
470	/**
471	* Host general register entry.
472	*
473	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
474	*
475	* @todo Track immediate values in host registers similarlly to how we track the
476	* guest register shadow copies. For it to be real helpful, though,
477	* we probably need to know which will be reused and put them into
478	* non-volatile registers, otherwise it's going to be more or less
479	* restricted to an instruction or two.
480	*/
481	typedef struct IEMNATIVEHSTREG
482	{
483	/** Set of guest registers this one shadows.
484	*
485	* Using a bitmap here so we can designate the same host register as a copy
486	* for more than one guest register. This is expected to be useful in
487	* situations where one value is copied to several registers in a sequence.
488	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
489	* sequence we'd want to let this register follow to be a copy of and there
490	* will always be places where we'd be picking the wrong one.
491	*/
492	uint64_t fGstRegShadows;
493	/** What is being kept in this register. */
494	IEMNATIVEWHAT enmWhat;
495	/** Variable index if holding a variable, otherwise UINT8_MAX. */
496	uint8_t idxVar;
497	/** Alignment padding. */
498	uint8_t abAlign[6];
499	} IEMNATIVEHSTREG;
500
501
502	/**
503	* Core state for the native recompiler, that is, things that needs careful
504	* handling when dealing with branches.
505	*/
506	typedef struct IEMNATIVECORESTATE
507	{
508	/** Allocation bitmap for aHstRegs. */
509	uint32_t bmHstRegs;
510
511	/** Bitmap marking which host register contains guest register shadow copies.
512	* This is used during register allocation to try preserve copies. */
513	uint32_t bmHstRegsWithGstShadow;
514	/** Bitmap marking valid entries in aidxGstRegShadows. */
515	uint64_t bmGstRegShadows;
516
517	union
518	{
519	/** Index of variable arguments, UINT8_MAX if not valid. */
520	uint8_t aidxArgVars[8];
521	/** For more efficient resetting. */
522	uint64_t u64ArgVars;
523	};
524
525	/** Allocation bitmap for aVars. */
526	uint32_t bmVars;
527
528	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
529	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
530	* (A shadow copy of a guest register can only be held in a one host register,
531	* there are no duplicate copies or ambiguities like that). */
532	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
533
534	/** Host register allocation tracking. */
535	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
536
537	/** Variables and arguments. */
538	IEMNATIVEVAR aVars[9];
539	} IEMNATIVECORESTATE;
540	/** Pointer to core state. */
541	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
542	/** Pointer to const core state. */
543	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
544
545
546	/**
547	* Conditional stack entry.
548	*/
549	typedef struct IEMNATIVECOND
550	{
551	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
552	bool fInElse;
553	/** The label for the IEM_MC_ELSE. */
554	uint32_t idxLabelElse;
555	/** The label for the IEM_MC_ENDIF. */
556	uint32_t idxLabelEndIf;
557	/** The initial state snapshot as the if-block starts executing. */
558	IEMNATIVECORESTATE InitialState;
559	/** The state snapshot at the end of the if-block. */
560	IEMNATIVECORESTATE IfFinalState;
561	} IEMNATIVECOND;
562	/** Pointer to a condition stack entry. */
563	typedef IEMNATIVECOND *PIEMNATIVECOND;
564
565
566	/**
567	* Native recompiler state.
568	*/
569	typedef struct IEMRECOMPILERSTATE
570	{
571	/** Size of the buffer that pbNativeRecompileBufR3 points to in
572	* IEMNATIVEINSTR units. */
573	uint32_t cInstrBufAlloc;
574	#ifdef VBOX_STRICT
575	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
576	uint32_t offInstrBufChecked;
577	#else
578	uint32_t uPadding1; /* We don't keep track of the size here... */
579	#endif
580	/** Fixed temporary code buffer for native recompilation. */
581	PIEMNATIVEINSTR pInstrBuf;
582
583	/** Bitmaps with the label types used. */
584	uint64_t bmLabelTypes;
585	/** Actual number of labels in paLabels. */
586	uint32_t cLabels;
587	/** Max number of entries allowed in paLabels before reallocating it. */
588	uint32_t cLabelsAlloc;
589	/** Labels defined while recompiling (referenced by fixups). */
590	PIEMNATIVELABEL paLabels;
591
592	/** Actual number of fixups paFixups. */
593	uint32_t cFixups;
594	/** Max number of entries allowed in paFixups before reallocating it. */
595	uint32_t cFixupsAlloc;
596	/** Buffer used by the recompiler for recording fixups when generating code. */
597	PIEMNATIVEFIXUP paFixups;
598
599	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
600	/** Number of debug info entries allocated for pDbgInfo. */
601	uint32_t cDbgInfoAlloc;
602	uint32_t uPadding;
603	/** Debug info. */
604	PIEMTBDBG pDbgInfo;
605	#endif
606
607	/** The translation block being recompiled. */
608	PCIEMTB pTbOrg;
609
610	/** The current condition stack depth (aCondStack). */
611	uint8_t cCondDepth;
612	uint8_t bPadding2;
613	/** Condition sequence number (for generating unique labels). */
614	uint16_t uCondSeqNo;
615	/** Check IRQ seqeunce number (for generating unique lables). */
616	uint16_t uCheckIrqSeqNo;
617	uint16_t uPadding3;
618
619	/** Core state requiring care with branches. */
620	IEMNATIVECORESTATE Core;
621
622	/** The condition nesting stack. */
623	IEMNATIVECOND aCondStack[2];
624
625	#ifndef IEM_WITH_THROW_CATCH
626	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
627	* for recompilation error handling. */
628	jmp_buf JmpBuf;
629	#endif
630	} IEMRECOMPILERSTATE;
631	/** Pointer to a native recompiler state. */
632	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
633
634
635	/** @def IEMNATIVE_TRY_SETJMP
636	* Wrapper around setjmp / try, hiding all the ugly differences.
637	*
638	* @note Use with extreme care as this is a fragile macro.
639	* @param a_pReNative The native recompile state.
640	* @param a_rcTarget The variable that should receive the status code in case
641	* of a longjmp/throw.
642	*/
643	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
644	* Start wrapper for catch / setjmp-else.
645	*
646	* This will set up a scope.
647	*
648	* @note Use with extreme care as this is a fragile macro.
649	* @param a_pReNative The native recompile state.
650	* @param a_rcTarget The variable that should receive the status code in case
651	* of a longjmp/throw.
652	*/
653	/** @def IEMNATIVE_CATCH_LONGJMP_END
654	* End wrapper for catch / setjmp-else.
655	*
656	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
657	* up the state.
658	*
659	* @note Use with extreme care as this is a fragile macro.
660	* @param a_pReNative The native recompile state.
661	*/
662	/** @def IEMNATIVE_DO_LONGJMP
663	*
664	* Wrapper around longjmp / throw.
665	*
666	* @param a_pReNative The native recompile state.
667	* @param a_rc The status code jump back with / throw.
668	*/
669	#ifdef IEM_WITH_THROW_CATCH
670	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
671	a_rcTarget = VINF_SUCCESS; \
672	try
673	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
674	catch (int rcThrown) \
675	{ \
676	a_rcTarget = rcThrown
677	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
678	} \
679	((void)0)
680	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
681	#else /* !IEM_WITH_THROW_CATCH */
682	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
683	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
684	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
685	else \
686	{ \
687	((void)0)
688	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
689	}
690	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
691	#endif /* !IEM_WITH_THROW_CATCH */
692
693
694	/**
695	* Native recompiler worker for a threaded function.
696	*
697	* @returns New code buffer offset; throws VBox status code in case of a failure.
698	* @param pReNative The native recompiler state.
699	* @param off The current code buffer offset.
700	* @param pCallEntry The threaded call entry.
701	*
702	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
703	*/
704	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
705	/** Pointer to a native recompiler worker for a threaded function. */
706	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
707
708	/** Defines a native recompiler worker for a threaded function.
709	* @see FNIEMNATIVERECOMPFUNC */
710	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
711	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
712
713	/** Prototypes a native recompiler function for a threaded function.
714	* @see FNIEMNATIVERECOMPFUNC */
715	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
716
717	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
718	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
719	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
720	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
721	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
722	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
723
724	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
725	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
726	bool fPreferVolatile = true);
727	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
728	IEMNATIVEGSTREG enmGstReg, IEMNATIVEGSTREGUSE enmIntendedUse);
729	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
730	IEMNATIVEGSTREG enmGstReg);
731
732	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
733	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
734	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
735	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
736	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
737	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
738	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
739	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
740
741	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
742	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
743	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
744
745
746	/**
747	* Ensures that there is sufficient space in the instruction output buffer.
748	*
749	* This will reallocate the buffer if needed and allowed.
750	*
751	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
752	* allocation size.
753	*
754	* @returns Pointer to the instruction output buffer on success; throws VBox
755	* status code on failure, so no need to check it.
756	* @param pReNative The native recompile state.
757	* @param off Current instruction offset. Works safely for UINT32_MAX
758	* as well.
759	* @param cInstrReq Number of instruction about to be added. It's okay to
760	* overestimate this a bit.
761	*/
762	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
763	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
764	{
765	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
766	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
767	{
768	#ifdef VBOX_STRICT
769	pReNative->offInstrBufChecked = offChecked;
770	#endif
771	return pReNative->pInstrBuf;
772	}
773	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
774	}
775
776	/**
777	* Checks that we didn't exceed the space requested in the last
778	* iemNativeInstrBufEnsure() call. */
779	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
780	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
781	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
782
783
784	/**
785	* Emit a simple marker instruction to more easily tell where something starts
786	* in the disassembly.
787	*/
788	DECL_INLINE_THROW(uint32_t)
789	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
790	{
791	#ifdef RT_ARCH_AMD64
792	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
793	if (uInfo == 0)
794	{
795	/* nop */
796	pbCodeBuf[off++] = 0x90;
797	}
798	else
799	{
800	/* nop [disp32] */
801	pbCodeBuf[off++] = 0x0f;
802	pbCodeBuf[off++] = 0x1f;
803	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
804	pbCodeBuf[off++] = RT_BYTE1(uInfo);
805	pbCodeBuf[off++] = RT_BYTE2(uInfo);
806	pbCodeBuf[off++] = RT_BYTE3(uInfo);
807	pbCodeBuf[off++] = RT_BYTE4(uInfo);
808	}
809	#elif RT_ARCH_ARM64
810	/* nop */
811	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
812	pu32CodeBuf[off++] = 0xd503201f;
813
814	RT_NOREF(uInfo);
815	#else
816	# error "port me"
817	#endif
818	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
819	return off;
820	}
821
822
823	/*********************************************************************************************************************************
824	* Loads, Stores and Related Stuff. *
825	*********************************************************************************************************************************/
826
827	/**
828	* Emits setting a GPR to zero.
829	*/
830	DECL_INLINE_THROW(uint32_t)
831	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
832	{
833	#ifdef RT_ARCH_AMD64
834	/* xor gpr32, gpr32 */
835	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
836	if (iGpr >= 8)
837	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
838	pbCodeBuf[off++] = 0x33;
839	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
840
841	#elif RT_ARCH_ARM64
842	/* mov gpr, #0x0 */
843	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
844	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
845
846	#else
847	# error "port me"
848	#endif
849	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
850	return off;
851	}
852
853
854	/**
855	* Emits loading a constant into a 64-bit GPR
856	*/
857	DECL_INLINE_THROW(uint32_t)
858	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
859	{
860	if (!uImm64)
861	return iemNativeEmitGprZero(pReNative, off, iGpr);
862
863	#ifdef RT_ARCH_AMD64
864	if (uImm64 <= UINT32_MAX)
865	{
866	/* mov gpr, imm32 */
867	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
868	if (iGpr >= 8)
869	pbCodeBuf[off++] = X86_OP_REX_B;
870	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
871	pbCodeBuf[off++] = RT_BYTE1(uImm64);
872	pbCodeBuf[off++] = RT_BYTE2(uImm64);
873	pbCodeBuf[off++] = RT_BYTE3(uImm64);
874	pbCodeBuf[off++] = RT_BYTE4(uImm64);
875	}
876	else
877	{
878	/* mov gpr, imm64 */
879	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
880	if (iGpr < 8)
881	pbCodeBuf[off++] = X86_OP_REX_W;
882	else
883	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
884	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
885	pbCodeBuf[off++] = RT_BYTE1(uImm64);
886	pbCodeBuf[off++] = RT_BYTE2(uImm64);
887	pbCodeBuf[off++] = RT_BYTE3(uImm64);
888	pbCodeBuf[off++] = RT_BYTE4(uImm64);
889	pbCodeBuf[off++] = RT_BYTE5(uImm64);
890	pbCodeBuf[off++] = RT_BYTE6(uImm64);
891	pbCodeBuf[off++] = RT_BYTE7(uImm64);
892	pbCodeBuf[off++] = RT_BYTE8(uImm64);
893	}
894
895	#elif RT_ARCH_ARM64
896	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
897
898	/*
899	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
900	* supply remaining bits using 'movk grp, imm16, lsl #x'.
901	*
902	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
903	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
904	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
905	* after the first non-zero immediate component so we switch to movk for
906	* the remainder.
907	*/
908	uint32_t fMovK = 0;
909	/* mov gpr, imm16 */
910	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
911	if (uImmPart)
912	{
913	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
914	fMovK \|= RT_BIT_32(29);
915	}
916	/* mov[k] gpr, imm16, lsl #16 */
917	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
918	if (uImmPart)
919	{
920	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
921	fMovK \|= RT_BIT_32(29);
922	}
923	/* mov[k] gpr, imm16, lsl #32 */
924	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
925	if (uImmPart)
926	{
927	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
928	fMovK \|= RT_BIT_32(29);
929	}
930	/* mov[k] gpr, imm16, lsl #48 */
931	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
932	if (uImmPart)
933	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
934
935	/** @todo there is an inverted mask variant we might want to explore if it
936	* reduces the number of instructions... */
937	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
938	* clang 12.x does that, only to use the 'x' version for the
939	* addressing in the following ldr). */
940
941	#else
942	# error "port me"
943	#endif
944	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
945	return off;
946	}
947
948
949	/**
950	* Emits loading a constant into a 8-bit GPR
951	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
952	* only the ARM64 version does that.
953	*/
954	DECL_INLINE_THROW(uint32_t)
955	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
956	{
957	#ifdef RT_ARCH_AMD64
958	/* mov gpr, imm8 */
959	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
960	if (iGpr >= 8)
961	pbCodeBuf[off++] = X86_OP_REX_B;
962	else if (iGpr >= 4)
963	pbCodeBuf[off++] = X86_OP_REX;
964	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
965	pbCodeBuf[off++] = RT_BYTE1(uImm8);
966
967	#elif RT_ARCH_ARM64
968	/* movz gpr, imm16, lsl #0 */
969	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
970	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
971
972	#else
973	# error "port me"
974	#endif
975	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
976	return off;
977	}
978
979
980	#ifdef RT_ARCH_AMD64
981	/**
982	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
983	*/
984	DECL_FORCE_INLINE(uint32_t)
985	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
986	{
987	if (offVCpu < 128)
988	{
989	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
990	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
991	}
992	else
993	{
994	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
995	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
996	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
997	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
998	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
999	}
1000	return off;
1001	}
1002	#elif RT_ARCH_ARM64
1003	/**
1004	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1005	*/
1006	DECL_FORCE_INLINE_THROW(uint32_t)
1007	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1008	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1009	{
1010	/*
1011	* There are a couple of ldr variants that takes an immediate offset, so
1012	* try use those if we can, otherwise we have to use the temporary register
1013	* help with the addressing.
1014	*/
1015	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
1016	{
1017	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1018	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1019	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
1020	}
1021	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
1022	{
1023	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1024	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
1025	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
1026	}
1027	else
1028	{
1029	/* The offset is too large, so we must load it into a register and use
1030	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1031	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1032	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
1033
1034	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1035	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
1036	}
1037	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1038	return off;
1039	}
1040	#endif
1041
1042
1043	/**
1044	* Emits a 64-bit GPR load of a VCpu value.
1045	*/
1046	DECL_INLINE_THROW(uint32_t)
1047	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1048	{
1049	#ifdef RT_ARCH_AMD64
1050	/* mov reg64, mem64 */
1051	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1052	if (iGpr < 8)
1053	pbCodeBuf[off++] = X86_OP_REX_W;
1054	else
1055	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1056	pbCodeBuf[off++] = 0x8b;
1057	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
1058	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1059
1060	#elif RT_ARCH_ARM64
1061	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1062
1063	#else
1064	# error "port me"
1065	#endif
1066	return off;
1067	}
1068
1069
1070	/**
1071	* Emits a 32-bit GPR load of a VCpu value.
1072	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1073	*/
1074	DECL_INLINE_THROW(uint32_t)
1075	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1076	{
1077	#ifdef RT_ARCH_AMD64
1078	/* mov reg32, mem32 */
1079	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1080	if (iGpr >= 8)
1081	pbCodeBuf[off++] = X86_OP_REX_R;
1082	pbCodeBuf[off++] = 0x8b;
1083	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1084	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1085
1086	#elif RT_ARCH_ARM64
1087	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1088
1089	#else
1090	# error "port me"
1091	#endif
1092	return off;
1093	}
1094
1095
1096	/**
1097	* Emits a 16-bit GPR load of a VCpu value.
1098	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
1099	*/
1100	DECL_INLINE_THROW(uint32_t)
1101	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1102	{
1103	#ifdef RT_ARCH_AMD64
1104	/* movzx reg32, mem16 */
1105	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1106	if (iGpr >= 8)
1107	pbCodeBuf[off++] = X86_OP_REX_R;
1108	pbCodeBuf[off++] = 0x0f;
1109	pbCodeBuf[off++] = 0xb7;
1110	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1111	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1112
1113	#elif RT_ARCH_ARM64
1114	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1115
1116	#else
1117	# error "port me"
1118	#endif
1119	return off;
1120	}
1121
1122
1123	/**
1124	* Emits a 8-bit GPR load of a VCpu value.
1125	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
1126	*/
1127	DECL_INLINE_THROW(uint32_t)
1128	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1129	{
1130	#ifdef RT_ARCH_AMD64
1131	/* movzx reg32, mem8 */
1132	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1133	if (iGpr >= 8)
1134	pbCodeBuf[off++] = X86_OP_REX_R;
1135	pbCodeBuf[off++] = 0x0f;
1136	pbCodeBuf[off++] = 0xb6;
1137	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1139
1140	#elif RT_ARCH_ARM64
1141	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1142
1143	#else
1144	# error "port me"
1145	#endif
1146	return off;
1147	}
1148
1149
1150	/**
1151	* Emits a store of a GPR value to a 64-bit VCpu field.
1152	*/
1153	DECL_INLINE_THROW(uint32_t)
1154	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1155	{
1156	#ifdef RT_ARCH_AMD64
1157	/* mov mem64, reg64 */
1158	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1159	if (iGpr < 8)
1160	pbCodeBuf[off++] = X86_OP_REX_W;
1161	else
1162	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1163	pbCodeBuf[off++] = 0x89;
1164	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1165	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1166
1167	#elif RT_ARCH_ARM64
1168	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1169
1170	#else
1171	# error "port me"
1172	#endif
1173	return off;
1174	}
1175
1176
1177	/**
1178	* Emits a store of a GPR value to a 32-bit VCpu field.
1179	*/
1180	DECL_INLINE_THROW(uint32_t)
1181	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1182	{
1183	#ifdef RT_ARCH_AMD64
1184	/* mov mem32, reg32 */
1185	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1186	if (iGpr >= 8)
1187	pbCodeBuf[off++] = X86_OP_REX_R;
1188	pbCodeBuf[off++] = 0x89;
1189	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1190	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1191
1192	#elif RT_ARCH_ARM64
1193	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1194
1195	#else
1196	# error "port me"
1197	#endif
1198	return off;
1199	}
1200
1201
1202	/**
1203	* Emits a store of a GPR value to a 16-bit VCpu field.
1204	*/
1205	DECL_INLINE_THROW(uint32_t)
1206	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1207	{
1208	#ifdef RT_ARCH_AMD64
1209	/* mov mem16, reg16 */
1210	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1211	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1212	if (iGpr >= 8)
1213	pbCodeBuf[off++] = X86_OP_REX_R;
1214	pbCodeBuf[off++] = 0x89;
1215	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1216	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1217
1218	#elif RT_ARCH_ARM64
1219	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1220
1221	#else
1222	# error "port me"
1223	#endif
1224	return off;
1225	}
1226
1227
1228	/**
1229	* Emits a store of a GPR value to a 8-bit VCpu field.
1230	*/
1231	DECL_INLINE_THROW(uint32_t)
1232	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1233	{
1234	#ifdef RT_ARCH_AMD64
1235	/* mov mem8, reg8 */
1236	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1237	if (iGpr >= 8)
1238	pbCodeBuf[off++] = X86_OP_REX_R;
1239	pbCodeBuf[off++] = 0x88;
1240	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1241	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1242
1243	#elif RT_ARCH_ARM64
1244	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1245
1246	#else
1247	# error "port me"
1248	#endif
1249	return off;
1250	}
1251
1252
1253	/**
1254	* Emits a gprdst = gprsrc load.
1255	*/
1256	DECL_INLINE_THROW(uint32_t)
1257	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1258	{
1259	#ifdef RT_ARCH_AMD64
1260	/* mov gprdst, gprsrc */
1261	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1262	if ((iGprDst \| iGprSrc) >= 8)
1263	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
1264	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
1265	: X86_OP_REX_W \| X86_OP_REX_R;
1266	else
1267	pbCodeBuf[off++] = X86_OP_REX_W;
1268	pbCodeBuf[off++] = 0x8b;
1269	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1270
1271	#elif RT_ARCH_ARM64
1272	/* mov dst, src; alias for: orr dst, xzr, src */
1273	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1274	pu32CodeBuf[off++] = UINT32_C(0xaa000000) \| ((uint32_t)iGprSrc << 16) \| ((uint32_t)ARMV8_A64_REG_XZR << 5) \| iGprDst;
1275
1276	#else
1277	# error "port me"
1278	#endif
1279	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1280	return off;
1281	}
1282
1283	#ifdef RT_ARCH_AMD64
1284	/**
1285	* Common bit of iemNativeEmitLoadGprByBp and friends.
1286	*/
1287	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1288	PIEMRECOMPILERSTATE pReNativeAssert)
1289	{
1290	if (offDisp < 128 && offDisp >= -128)
1291	{
1292	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1293	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1294	}
1295	else
1296	{
1297	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1298	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1299	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1300	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1301	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1302	}
1303	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1304	return off;
1305	}
1306	#endif
1307
1308
1309	#ifdef RT_ARCH_AMD64
1310	/**
1311	* Emits a 64-bit GRP load instruction with an BP relative source address.
1312	*/
1313	DECL_INLINE_THROW(uint32_t)
1314	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1315	{
1316	/* mov gprdst, qword [rbp + offDisp] */
1317	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1318	if (iGprDst < 8)
1319	pbCodeBuf[off++] = X86_OP_REX_W;
1320	else
1321	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1322	pbCodeBuf[off++] = 0x8b;
1323	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1324	}
1325	#endif
1326
1327
1328	#ifdef RT_ARCH_AMD64
1329	/**
1330	* Emits a 32-bit GRP load instruction with an BP relative source address.
1331	*/
1332	DECL_INLINE_THROW(uint32_t)
1333	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1334	{
1335	/* mov gprdst, dword [rbp + offDisp] */
1336	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1337	if (iGprDst >= 8)
1338	pbCodeBuf[off++] = X86_OP_REX_R;
1339	pbCodeBuf[off++] = 0x8b;
1340	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1341	}
1342	#endif
1343
1344
1345	#ifdef RT_ARCH_AMD64
1346	/**
1347	* Emits a load effective address to a GRP with an BP relative source address.
1348	*/
1349	DECL_INLINE_THROW(uint32_t)
1350	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1351	{
1352	/* lea gprdst, [rbp + offDisp] */
1353	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1354	if (iGprDst < 8)
1355	pbCodeBuf[off++] = X86_OP_REX_W;
1356	else
1357	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1358	pbCodeBuf[off++] = 0x8d;
1359	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1360	}
1361	#endif
1362
1363
1364	/**
1365	* Emits a 64-bit GPR store with an BP relative destination address.
1366	*
1367	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1368	*/
1369	DECL_INLINE_THROW(uint32_t)
1370	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1371	{
1372	#ifdef RT_ARCH_AMD64
1373	/* mov qword [rbp + offDisp], gprdst */
1374	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1375	if (iGprSrc < 8)
1376	pbCodeBuf[off++] = X86_OP_REX_W;
1377	else
1378	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1379	pbCodeBuf[off++] = 0x89;
1380	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1381
1382	#elif defined(RT_ARCH_ARM64)
1383	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1384	{
1385	/* str w/ unsigned imm12 (scaled) */
1386	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1387	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1388	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1389	}
1390	else if (offDisp >= -256 && offDisp <= 256)
1391	{
1392	/* stur w/ signed imm9 (unscaled) */
1393	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1394	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1395	}
1396	else
1397	{
1398	/* Use temporary indexing register. */
1399	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1400	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1401	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1402	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1403	}
1404	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1405	return off;
1406
1407	#else
1408	# error "Port me!"
1409	#endif
1410	}
1411
1412
1413	/**
1414	* Emits a 64-bit immediate store with an BP relative destination address.
1415	*
1416	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1417	*/
1418	DECL_INLINE_THROW(uint32_t)
1419	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1420	{
1421	#ifdef RT_ARCH_AMD64
1422	if ((int64_t)uImm64 == (int32_t)uImm64)
1423	{
1424	/* mov qword [rbp + offDisp], imm32 - sign extended */
1425	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1426	pbCodeBuf[off++] = X86_OP_REX_W;
1427	pbCodeBuf[off++] = 0xc7;
1428	if (offDisp < 128 && offDisp >= -128)
1429	{
1430	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1431	pbCodeBuf[off++] = (uint8_t)offDisp;
1432	}
1433	else
1434	{
1435	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1436	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1437	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1438	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1439	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1440	}
1441	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1442	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1443	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1444	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1445	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1446	return off;
1447	}
1448	#endif
1449
1450	/* Load tmp0, imm64; Store tmp to bp+disp. */
1451	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1452	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1453	}
1454
1455
1456	#ifdef RT_ARCH_AMD64
1457	/**
1458	* Common bit of iemNativeEmitLoadGprByGpr and friends.
1459	*/
1460	DECL_FORCE_INLINE(uint32_t)
1461	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1462	{
1463	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1464	{
1465	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1466	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1467	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1468	}
1469	else if (offDisp == (int8_t)offDisp)
1470	{
1471	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1472	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1473	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1474	pbCodeBuf[off++] = (uint8_t)offDisp;
1475	}
1476	else
1477	{
1478	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1479	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1480	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1481	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1482	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1483	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1484	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1485	}
1486	return off;
1487	}
1488	#elif RT_ARCH_ARM64
1489	/**
1490	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1491	*/
1492	DECL_FORCE_INLINE_THROW(uint32_t)
1493	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1494	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1495	{
1496	/*
1497	* There are a couple of ldr variants that takes an immediate offset, so
1498	* try use those if we can, otherwise we have to use the temporary register
1499	* help with the addressing.
1500	*/
1501	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1502	{
1503	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1504	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1505	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1506	}
1507	else
1508	{
1509	/* The offset is too large, so we must load it into a register and use
1510	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1511	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1512	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, off, (uint64)offDisp);
1513
1514	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1515	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1516
1517	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1518	}
1519	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1520	return off;
1521	}
1522	#endif
1523
1524
1525	/**
1526	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1527	*/
1528	DECL_INLINE_THROW(uint32_t)
1529	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1530	{
1531	#ifdef RT_ARCH_AMD64
1532	/* mov reg64, mem64 */
1533	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1534	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1535	pbCodeBuf[off++] = 0x8b;
1536	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1537	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1538
1539	#elif RT_ARCH_ARM64
1540	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1541
1542	#else
1543	# error "port me"
1544	#endif
1545	return off;
1546	}
1547
1548
1549	/**
1550	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1551	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1552	*/
1553	DECL_INLINE_THROW(uint32_t)
1554	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1555	{
1556	#ifdef RT_ARCH_AMD64
1557	/* mov reg32, mem32 */
1558	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1559	if (iGprDst >= 8 \|\| iGprBase >= 8)
1560	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1561	pbCodeBuf[off++] = 0x8b;
1562	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1563	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1564
1565	#elif RT_ARCH_ARM64
1566	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1567
1568	#else
1569	# error "port me"
1570	#endif
1571	return off;
1572	}
1573
1574
1575	/*********************************************************************************************************************************
1576	* Subtraction and Additions *
1577	*********************************************************************************************************************************/
1578
1579
1580	#ifdef RT_ARCH_AMD64
1581	/**
1582	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1583	*/
1584	DECL_INLINE_THROW(uint32_t)
1585	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1586	{
1587	/* sub gprdst, imm8/imm32 */
1588	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1589	if (iGprDst < 8)
1590	pbCodeBuf[off++] = X86_OP_REX_W;
1591	else
1592	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1593	if (iSubtrahend < 128 && iSubtrahend >= -128)
1594	{
1595	pbCodeBuf[off++] = 0x83;
1596	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1597	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1598	}
1599	else
1600	{
1601	pbCodeBuf[off++] = 0x81;
1602	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1603	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1604	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1605	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1606	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1607	}
1608	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1609	return off;
1610	}
1611	#endif
1612
1613
1614	/**
1615	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1616	* @note The AMD64 version sets flags.
1617	*/
1618	DECL_INLINE_THROW(uint32_t)
1619	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1620	{
1621	#if defined(RT_ARCH_AMD64)
1622	/* add Gv,Ev */
1623	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1624	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1625	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1626	pbCodeBuf[off++] = 0x04;
1627	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1628
1629	#elif defined(RT_ARCH_ARM64)
1630	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1631	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1632
1633	#else
1634	# error "Port me"
1635	#endif
1636	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1637	return off;
1638	}
1639
1640
1641	/**
1642	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1643	*/
1644	DECL_INLINE_THROW(uint32_t)
1645	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1646	{
1647	#if defined(RT_ARCH_AMD64)
1648	/* add or inc */
1649	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1650	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1651	if (iImm8 != 1)
1652	{
1653	pbCodeBuf[off++] = 0x83;
1654	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1655	pbCodeBuf[off++] = (uint8_t)iImm8;
1656	}
1657	else
1658	{
1659	pbCodeBuf[off++] = 0xff;
1660	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1661	}
1662
1663	#elif defined(RT_ARCH_ARM64)
1664	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1665	if (iImm8 >= 0)
1666	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1667	else
1668	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1669
1670	#else
1671	# error "Port me"
1672	#endif
1673	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1674	return off;
1675	}
1676
1677
1678	/**
1679	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1680	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1681	*/
1682	DECL_INLINE_THROW(uint32_t)
1683	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1684	{
1685	#if defined(RT_ARCH_AMD64)
1686	/* add or inc */
1687	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1688	if (iGprDst >= 8)
1689	pbCodeBuf[off++] = X86_OP_REX_B;
1690	if (iImm8 != 1)
1691	{
1692	pbCodeBuf[off++] = 0x83;
1693	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1694	pbCodeBuf[off++] = (uint8_t)iImm8;
1695	}
1696	else
1697	{
1698	pbCodeBuf[off++] = 0xff;
1699	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1700	}
1701
1702	#elif defined(RT_ARCH_ARM64)
1703	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1704	if (iImm8 >= 0)
1705	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1706	else
1707	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1708
1709	#else
1710	# error "Port me"
1711	#endif
1712	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1713	return off;
1714	}
1715
1716
1717	/**
1718	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1719	*/
1720	DECL_INLINE_THROW(uint32_t)
1721	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1722	{
1723	#if defined(RT_ARCH_AMD64)
1724	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1725	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1726
1727	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1728	{
1729	/* add grp, imm32 */
1730	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1731	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1732	pbCodeBuf[off++] = 0x81;
1733	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1734	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1735	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1736	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1737	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1738	}
1739	else
1740	{
1741	/* Best to use a temporary register to deal with this in the simplest way: */
1742	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1743
1744	/* add dst, tmpreg */
1745	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1746	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1747	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1748	pbCodeBuf[off++] = 0x03;
1749	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1750
1751	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1752	}
1753
1754	#elif defined(RT_ARCH_ARM64)
1755	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1756	{
1757	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1758	if (iAddend >= 0)
1759	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1760	else
1761	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1762	}
1763	else
1764	{
1765	/* Use temporary register for the immediate. */
1766	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1767
1768	/* add gprdst, gprdst, tmpreg */
1769	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1770	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1771
1772	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1773	}
1774
1775	#else
1776	# error "Port me"
1777	#endif
1778	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1779	return off;
1780	}
1781
1782
1783	/**
1784	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1785	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1786	*/
1787	DECL_INLINE_THROW(uint32_t)
1788	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1789	{
1790	#if defined(RT_ARCH_AMD64)
1791	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1792	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1793
1794	/* add grp, imm32 */
1795	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1796	if (iGprDst >= 8)
1797	pbCodeBuf[off++] = X86_OP_REX_B;
1798	pbCodeBuf[off++] = 0x81;
1799	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1800	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1801	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1802	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1803	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1804
1805	#elif defined(RT_ARCH_ARM64)
1806	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1807	{
1808	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1809	if (iAddend >= 0)
1810	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1811	else
1812	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1813	}
1814	else
1815	{
1816	/* Use temporary register for the immediate. */
1817	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1818
1819	/* add gprdst, gprdst, tmpreg */
1820	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1821	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1822
1823	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1824	}
1825
1826	#else
1827	# error "Port me"
1828	#endif
1829	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1830	return off;
1831	}
1832
1833
1834
1835	/*********************************************************************************************************************************
1836	* Bit Operations *
1837	*********************************************************************************************************************************/
1838
1839	/**
1840	* Emits code for clearing bits 16 thru 63 in the GPR.
1841	*/
1842	DECL_INLINE_THROW(uint32_t)
1843	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1844	{
1845	#if defined(RT_ARCH_AMD64)
1846	/* movzx reg32, reg16 */
1847	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1848	if (iGprDst >= 8)
1849	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1850	pbCodeBuf[off++] = 0x0f;
1851	pbCodeBuf[off++] = 0xb7;
1852	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1853
1854	#elif defined(RT_ARCH_ARM64)
1855	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1856	# if 1
1857	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1858	# else
1859	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1860	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1861	# endif
1862	#else
1863	# error "Port me"
1864	#endif
1865	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1866	return off;
1867	}
1868
1869
1870	/**
1871	* Emits code for AND'ing two 64-bit GPRs.
1872	*
1873	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1874	* and ARM64 hosts.
1875	*/
1876	DECL_INLINE_THROW(uint32_t)
1877	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1878	{
1879	#if defined(RT_ARCH_AMD64)
1880	/* and Gv, Ev */
1881	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1882	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1883	pbCodeBuf[off++] = 0x23;
1884	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1885	RT_NOREF(fSetFlags);
1886
1887	#elif defined(RT_ARCH_ARM64)
1888	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1889	if (!fSetFlags)
1890	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1891	else
1892	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
1893
1894	#else
1895	# error "Port me"
1896	#endif
1897	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1898	return off;
1899	}
1900
1901
1902	/**
1903	* Emits code for AND'ing two 32-bit GPRs.
1904	*/
1905	DECL_INLINE_THROW(uint32_t)
1906	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1907	{
1908	#if defined(RT_ARCH_AMD64)
1909	/* and Gv, Ev */
1910	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1911	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1912	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1913	pbCodeBuf[off++] = 0x23;
1914	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1915
1916	#elif defined(RT_ARCH_ARM64)
1917	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1918	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1919
1920	#else
1921	# error "Port me"
1922	#endif
1923	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1924	return off;
1925	}
1926
1927
1928	/**
1929	* Emits code for AND'ing a 64-bit GPRs with a constant.
1930	*
1931	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1932	* and ARM64 hosts.
1933	*/
1934	DECL_INLINE_THROW(uint32_t)
1935	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
1936	{
1937	#if defined(RT_ARCH_AMD64)
1938	if ((int64_t)uImm == (int8_t)uImm)
1939	{
1940	/* and Ev, imm8 */
1941	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1942	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1943	pbCodeBuf[off++] = 0x83;
1944	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1945	pbCodeBuf[off++] = (uint8_t)uImm;
1946	}
1947	else if ((int64_t)uImm == (int32_t)uImm)
1948	{
1949	/* and Ev, imm32 */
1950	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1951	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1952	pbCodeBuf[off++] = 0x81;
1953	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1954	pbCodeBuf[off++] = RT_BYTE1(uImm);
1955	pbCodeBuf[off++] = RT_BYTE2(uImm);
1956	pbCodeBuf[off++] = RT_BYTE3(uImm);
1957	pbCodeBuf[off++] = RT_BYTE4(uImm);
1958	}
1959	else
1960	{
1961	/* Use temporary register for the 64-bit immediate. */
1962	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1963	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
1964	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1965	}
1966	RT_NOREF(fSetFlags);
1967
1968	#elif defined(RT_ARCH_ARM64)
1969	uint32_t uImmR = 0;
1970	uint32_t uImmNandS = 0;
1971	if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
1972	{
1973	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1974	if (!fSetFlags)
1975	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
1976	else
1977	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
1978	}
1979	else
1980	{
1981	/* Use temporary register for the 64-bit immediate. */
1982	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1983	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1984	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1985	}
1986
1987	#else
1988	# error "Port me"
1989	#endif
1990	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1991	return off;
1992	}
1993
1994
1995	/**
1996	* Emits code for AND'ing an 32-bit GPRs with a constant.
1997	*/
1998	DECL_INLINE_THROW(uint32_t)
1999	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2000	{
2001	#if defined(RT_ARCH_AMD64)
2002	/* and Ev, imm */
2003	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2004	if (iGprDst >= 8)
2005	pbCodeBuf[off++] = X86_OP_REX_B;
2006	if ((int32_t)uImm == (int8_t)uImm)
2007	{
2008	pbCodeBuf[off++] = 0x83;
2009	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2010	pbCodeBuf[off++] = (uint8_t)uImm;
2011	}
2012	else
2013	{
2014	pbCodeBuf[off++] = 0x81;
2015	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2016	pbCodeBuf[off++] = RT_BYTE1(uImm);
2017	pbCodeBuf[off++] = RT_BYTE2(uImm);
2018	pbCodeBuf[off++] = RT_BYTE3(uImm);
2019	pbCodeBuf[off++] = RT_BYTE4(uImm);
2020	}
2021	RT_NOREF(fSetFlags);
2022
2023	#elif defined(RT_ARCH_ARM64)
2024	uint32_t uImmR = 0;
2025	uint32_t uImmNandS = 0;
2026	if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
2027	{
2028	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2029	if (!fSetFlags)
2030	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2031	else
2032	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2033	}
2034	else
2035	{
2036	/* Use temporary register for the 64-bit immediate. */
2037	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2038	if (!fSetFlags)
2039	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
2040	else
2041	off = iemNativeEmitAndsGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
2042	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2043	}
2044
2045	#else
2046	# error "Port me"
2047	#endif
2048	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2049	return off;
2050	}
2051
2052
2053	/**
2054	* Emits code for XOR'ing two 64-bit GPRs.
2055	*/
2056	DECL_INLINE_THROW(uint32_t)
2057	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2058	{
2059	#if defined(RT_ARCH_AMD64)
2060	/* and Gv, Ev */
2061	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2062	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2063	pbCodeBuf[off++] = 0x33;
2064	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2065
2066	#elif defined(RT_ARCH_ARM64)
2067	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2068	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2069
2070	#else
2071	# error "Port me"
2072	#endif
2073	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2074	return off;
2075	}
2076
2077
2078	/**
2079	* Emits code for XOR'ing two 32-bit GPRs.
2080	*/
2081	DECL_INLINE_THROW(uint32_t)
2082	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2083	{
2084	#if defined(RT_ARCH_AMD64)
2085	/* and Gv, Ev */
2086	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2087	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2088	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2089	pbCodeBuf[off++] = 0x33;
2090	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2091
2092	#elif defined(RT_ARCH_ARM64)
2093	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2094	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2095
2096	#else
2097	# error "Port me"
2098	#endif
2099	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2100	return off;
2101	}
2102
2103
2104	/*********************************************************************************************************************************
2105	* Shifting *
2106	*********************************************************************************************************************************/
2107
2108	/**
2109	* Emits code for shifting a GPR a fixed number of bits to the left.
2110	*/
2111	DECL_INLINE_THROW(uint32_t)
2112	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2113	{
2114	Assert(cShift > 0 && cShift < 64);
2115
2116	#if defined(RT_ARCH_AMD64)
2117	/* shl dst, cShift */
2118	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2119	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2120	if (cShift != 1)
2121	{
2122	pbCodeBuf[off++] = 0xc1;
2123	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2124	pbCodeBuf[off++] = cShift;
2125	}
2126	else
2127	{
2128	pbCodeBuf[off++] = 0xd1;
2129	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2130	}
2131
2132	#elif defined(RT_ARCH_ARM64)
2133	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2134	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2135
2136	#else
2137	# error "Port me"
2138	#endif
2139	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2140	return off;
2141	}
2142
2143
2144	/**
2145	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2146	*/
2147	DECL_INLINE_THROW(uint32_t)
2148	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2149	{
2150	Assert(cShift > 0 && cShift < 32);
2151
2152	#if defined(RT_ARCH_AMD64)
2153	/* shl dst, cShift */
2154	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2155	if (iGprDst >= 8)
2156	pbCodeBuf[off++] = X86_OP_REX_B;
2157	if (cShift != 1)
2158	{
2159	pbCodeBuf[off++] = 0xc1;
2160	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2161	pbCodeBuf[off++] = cShift;
2162	}
2163	else
2164	{
2165	pbCodeBuf[off++] = 0xd1;
2166	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2167	}
2168
2169	#elif defined(RT_ARCH_ARM64)
2170	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2171	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2172
2173	#else
2174	# error "Port me"
2175	#endif
2176	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2177	return off;
2178	}
2179
2180
2181	/**
2182	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2183	*/
2184	DECL_INLINE_THROW(uint32_t)
2185	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2186	{
2187	Assert(cShift > 0 && cShift < 64);
2188
2189	#if defined(RT_ARCH_AMD64)
2190	/* shr dst, cShift */
2191	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2192	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2193	if (cShift != 1)
2194	{
2195	pbCodeBuf[off++] = 0xc1;
2196	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2197	pbCodeBuf[off++] = cShift;
2198	}
2199	else
2200	{
2201	pbCodeBuf[off++] = 0xd1;
2202	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2203	}
2204
2205	#elif defined(RT_ARCH_ARM64)
2206	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2207	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2208
2209	#else
2210	# error "Port me"
2211	#endif
2212	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2213	return off;
2214	}
2215
2216
2217	/**
2218	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2219	* right.
2220	*/
2221	DECL_INLINE_THROW(uint32_t)
2222	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2223	{
2224	Assert(cShift > 0 && cShift < 32);
2225
2226	#if defined(RT_ARCH_AMD64)
2227	/* shr dst, cShift */
2228	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2229	if (iGprDst >= 8)
2230	pbCodeBuf[off++] = X86_OP_REX_B;
2231	if (cShift != 1)
2232	{
2233	pbCodeBuf[off++] = 0xc1;
2234	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2235	pbCodeBuf[off++] = cShift;
2236	}
2237	else
2238	{
2239	pbCodeBuf[off++] = 0xd1;
2240	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2241	}
2242
2243	#elif defined(RT_ARCH_ARM64)
2244	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2245	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2246
2247	#else
2248	# error "Port me"
2249	#endif
2250	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2251	return off;
2252	}
2253
2254
2255
2256	/*********************************************************************************************************************************
2257	* Compare and Testing *
2258	*********************************************************************************************************************************/
2259
2260
2261	#ifdef RT_ARCH_ARM64
2262	/**
2263	* Emits an ARM64 compare instruction.
2264	*/
2265	DECL_INLINE_THROW(uint32_t)
2266	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2267	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2268	{
2269	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2270	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2271	f64Bit, true /fSetFlags/, cShift, enmShift);
2272	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2273	return off;
2274	}
2275	#endif
2276
2277
2278	/**
2279	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2280	* with conditional instruction.
2281	*/
2282	DECL_INLINE_THROW(uint32_t)
2283	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2284	{
2285	#ifdef RT_ARCH_AMD64
2286	/* cmp Gv, Ev */
2287	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2288	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2289	pbCodeBuf[off++] = 0x3b;
2290	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2291	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2292
2293	#elif defined(RT_ARCH_ARM64)
2294	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2295
2296	#else
2297	# error "Port me!"
2298	#endif
2299	return off;
2300	}
2301
2302
2303	/**
2304	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2305	* with conditional instruction.
2306	*/
2307	DECL_INLINE_THROW(uint32_t)
2308	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2309	{
2310	#ifdef RT_ARCH_AMD64
2311	/* cmp Gv, Ev */
2312	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2313	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2314	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2315	pbCodeBuf[off++] = 0x3b;
2316	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2317	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2318
2319	#elif defined(RT_ARCH_ARM64)
2320	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2321
2322	#else
2323	# error "Port me!"
2324	#endif
2325	return off;
2326	}
2327
2328
2329	/**
2330	* Emits a compare of a 64-bit GPR with a constant value, settings status
2331	* flags/whatever for use with conditional instruction.
2332	*/
2333	DECL_INLINE_THROW(uint32_t)
2334	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2335	{
2336	#ifdef RT_ARCH_AMD64
2337	if (uImm <= UINT32_C(0xff))
2338	{
2339	/* cmp Ev, Ib */
2340	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2341	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2342	pbCodeBuf[off++] = 0x83;
2343	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2344	pbCodeBuf[off++] = (uint8_t)uImm;
2345	}
2346	else if ((int64_t)uImm == (int32_t)uImm)
2347	{
2348	/* cmp Ev, imm */
2349	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2350	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2351	pbCodeBuf[off++] = 0x81;
2352	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2353	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2354	pbCodeBuf[off++] = RT_BYTE1(uImm);
2355	pbCodeBuf[off++] = RT_BYTE2(uImm);
2356	pbCodeBuf[off++] = RT_BYTE3(uImm);
2357	pbCodeBuf[off++] = RT_BYTE4(uImm);
2358	}
2359	else
2360	{
2361	/* Use temporary register for the immediate. */
2362	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2363	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2364	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2365	}
2366
2367	#elif defined(RT_ARCH_ARM64)
2368	/** @todo guess there are clevere things we can do here... */
2369	if (uImm < _4K)
2370	{
2371	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2372	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2373	true /64Bit/, true /fSetFlags/);
2374	}
2375	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2376	{
2377	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2378	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2379	true /64Bit/, true /fSetFlags/, true /fShift12/);
2380	}
2381	else
2382	{
2383	/* Use temporary register for the immediate. */
2384	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2385	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2386	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2387	}
2388
2389	#else
2390	# error "Port me!"
2391	#endif
2392
2393	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2394	return off;
2395	}
2396
2397
2398	/**
2399	* Emits a compare of a 32-bit GPR with a constant value, settings status
2400	* flags/whatever for use with conditional instruction.
2401	*/
2402	DECL_INLINE_THROW(uint32_t)
2403	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2404	{
2405	#ifdef RT_ARCH_AMD64
2406	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2407	if (iGprLeft >= 8)
2408	pbCodeBuf[off++] = X86_OP_REX_B;
2409	if (uImm <= UINT32_C(0xff))
2410	{
2411	/* cmp Ev, Ib */
2412	pbCodeBuf[off++] = 0x83;
2413	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2414	pbCodeBuf[off++] = (uint8_t)uImm;
2415	}
2416	else
2417	{
2418	/* cmp Ev, imm */
2419	pbCodeBuf[off++] = 0x81;
2420	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2421	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2422	pbCodeBuf[off++] = RT_BYTE1(uImm);
2423	pbCodeBuf[off++] = RT_BYTE2(uImm);
2424	pbCodeBuf[off++] = RT_BYTE3(uImm);
2425	pbCodeBuf[off++] = RT_BYTE4(uImm);
2426	}
2427
2428	#elif defined(RT_ARCH_ARM64)
2429	/** @todo guess there are clevere things we can do here... */
2430	if (uImm < _4K)
2431	{
2432	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2433	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2434	false /64Bit/, true /fSetFlags/);
2435	}
2436	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2437	{
2438	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2439	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2440	false /64Bit/, true /fSetFlags/, true /fShift12/);
2441	}
2442	else
2443	{
2444	/* Use temporary register for the immediate. */
2445	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2446	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2447	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2448	}
2449
2450	#else
2451	# error "Port me!"
2452	#endif
2453
2454	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2455	return off;
2456	}
2457
2458
2459
2460	/*********************************************************************************************************************************
2461	* Branching *
2462	*********************************************************************************************************************************/
2463
2464	/**
2465	* Emits a JMP rel32 / B imm19 to the given label.
2466	*/
2467	DECL_INLINE_THROW(uint32_t)
2468	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2469	{
2470	Assert(idxLabel < pReNative->cLabels);
2471
2472	#ifdef RT_ARCH_AMD64
2473	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2474	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2475	{
2476	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2477	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2478	{
2479	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2480	pbCodeBuf[off++] = (uint8_t)offRel;
2481	}
2482	else
2483	{
2484	offRel -= 3;
2485	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2486	pbCodeBuf[off++] = RT_BYTE1(offRel);
2487	pbCodeBuf[off++] = RT_BYTE2(offRel);
2488	pbCodeBuf[off++] = RT_BYTE3(offRel);
2489	pbCodeBuf[off++] = RT_BYTE4(offRel);
2490	}
2491	}
2492	else
2493	{
2494	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2495	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2496	pbCodeBuf[off++] = 0xfe;
2497	pbCodeBuf[off++] = 0xff;
2498	pbCodeBuf[off++] = 0xff;
2499	pbCodeBuf[off++] = 0xff;
2500	}
2501	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2502
2503	#elif defined(RT_ARCH_ARM64)
2504	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2505	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2506	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
2507	else
2508	{
2509	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2510	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2511	}
2512
2513	#else
2514	# error "Port me!"
2515	#endif
2516	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2517	return off;
2518	}
2519
2520
2521	/**
2522	* Emits a JMP rel32 / B imm19 to a new undefined label.
2523	*/
2524	DECL_INLINE_THROW(uint32_t)
2525	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2526	{
2527	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2528	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2529	}
2530
2531	/** Condition type. */
2532	#ifdef RT_ARCH_AMD64
2533	typedef enum IEMNATIVEINSTRCOND : uint8_t
2534	{
2535	kIemNativeInstrCond_o = 0,
2536	kIemNativeInstrCond_no,
2537	kIemNativeInstrCond_c,
2538	kIemNativeInstrCond_nc,
2539	kIemNativeInstrCond_e,
2540	kIemNativeInstrCond_ne,
2541	kIemNativeInstrCond_be,
2542	kIemNativeInstrCond_nbe,
2543	kIemNativeInstrCond_s,
2544	kIemNativeInstrCond_ns,
2545	kIemNativeInstrCond_p,
2546	kIemNativeInstrCond_np,
2547	kIemNativeInstrCond_l,
2548	kIemNativeInstrCond_nl,
2549	kIemNativeInstrCond_le,
2550	kIemNativeInstrCond_nle
2551	} IEMNATIVEINSTRCOND;
2552	#elif defined(RT_ARCH_ARM64)
2553	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2554	#else
2555	# error "Port me!"
2556	#endif
2557
2558
2559	/**
2560	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2561	*/
2562	DECL_INLINE_THROW(uint32_t)
2563	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2564	{
2565	Assert(idxLabel < pReNative->cLabels);
2566
2567	#ifdef RT_ARCH_AMD64
2568	/* jcc rel32 */
2569	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2570	pbCodeBuf[off++] = 0x0f;
2571	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2572	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2573	pbCodeBuf[off++] = 0x00;
2574	pbCodeBuf[off++] = 0x00;
2575	pbCodeBuf[off++] = 0x00;
2576	pbCodeBuf[off++] = 0x00;
2577
2578	#elif defined(RT_ARCH_ARM64)
2579	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2580	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2581	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2582
2583	#else
2584	# error "Port me!"
2585	#endif
2586	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2587	return off;
2588	}
2589
2590
2591	/**
2592	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2593	*/
2594	DECL_INLINE_THROW(uint32_t)
2595	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2596	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2597	{
2598	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2599	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2600	}
2601
2602
2603	/**
2604	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2605	*/
2606	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2607	{
2608	#ifdef RT_ARCH_AMD64
2609	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2610	#elif defined(RT_ARCH_ARM64)
2611	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2612	#else
2613	# error "Port me!"
2614	#endif
2615	}
2616
2617	/**
2618	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2619	*/
2620	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2621	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2622	{
2623	#ifdef RT_ARCH_AMD64
2624	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2625	#elif defined(RT_ARCH_ARM64)
2626	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2627	#else
2628	# error "Port me!"
2629	#endif
2630	}
2631
2632
2633	/**
2634	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2635	*/
2636	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2637	{
2638	#ifdef RT_ARCH_AMD64
2639	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2640	#elif defined(RT_ARCH_ARM64)
2641	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2642	#else
2643	# error "Port me!"
2644	#endif
2645	}
2646
2647	/**
2648	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2649	*/
2650	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2651	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2652	{
2653	#ifdef RT_ARCH_AMD64
2654	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2655	#elif defined(RT_ARCH_ARM64)
2656	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2657	#else
2658	# error "Port me!"
2659	#endif
2660	}
2661
2662
2663	/**
2664	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2665	*/
2666	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2667	{
2668	#ifdef RT_ARCH_AMD64
2669	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2670	#elif defined(RT_ARCH_ARM64)
2671	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2672	#else
2673	# error "Port me!"
2674	#endif
2675	}
2676
2677	/**
2678	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2679	*/
2680	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2681	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2682	{
2683	#ifdef RT_ARCH_AMD64
2684	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2685	#elif defined(RT_ARCH_ARM64)
2686	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2687	#else
2688	# error "Port me!"
2689	#endif
2690	}
2691
2692
2693	/**
2694	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2695	*/
2696	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2697	{
2698	#ifdef RT_ARCH_AMD64
2699	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2700	#elif defined(RT_ARCH_ARM64)
2701	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2702	#else
2703	# error "Port me!"
2704	#endif
2705	}
2706
2707	/**
2708	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2709	*/
2710	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2711	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2712	{
2713	#ifdef RT_ARCH_AMD64
2714	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2715	#elif defined(RT_ARCH_ARM64)
2716	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2717	#else
2718	# error "Port me!"
2719	#endif
2720	}
2721
2722
2723	/**
2724	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2725	* How @a offJmp is applied is are target specific.
2726	*/
2727	DECL_INLINE_THROW(uint32_t)
2728	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2729	{
2730	#ifdef RT_ARCH_AMD64
2731	/* jcc rel32 */
2732	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2733	if (offTarget < 128 && offTarget >= -128)
2734	{
2735	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2736	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2737	}
2738	else
2739	{
2740	pbCodeBuf[off++] = 0x0f;
2741	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2742	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2743	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2744	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2745	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2746	}
2747
2748	#elif defined(RT_ARCH_ARM64)
2749	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2750	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2751
2752	#else
2753	# error "Port me!"
2754	#endif
2755	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2756	return off;
2757	}
2758
2759
2760	/**
2761	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2762	* How @a offJmp is applied is are target specific.
2763	*/
2764	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2765	{
2766	#ifdef RT_ARCH_AMD64
2767	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2768	#elif defined(RT_ARCH_ARM64)
2769	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2770	#else
2771	# error "Port me!"
2772	#endif
2773	}
2774
2775
2776	/**
2777	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2778	* How @a offJmp is applied is are target specific.
2779	*/
2780	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2781	{
2782	#ifdef RT_ARCH_AMD64
2783	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2784	#elif defined(RT_ARCH_ARM64)
2785	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2786	#else
2787	# error "Port me!"
2788	#endif
2789	}
2790
2791
2792	/**
2793	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2794	* How @a offJmp is applied is are target specific.
2795	*/
2796	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2797	{
2798	#ifdef RT_ARCH_AMD64
2799	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2800	#elif defined(RT_ARCH_ARM64)
2801	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2802	#else
2803	# error "Port me!"
2804	#endif
2805	}
2806
2807
2808	/**
2809	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2810	* How @a offJmp is applied is are target specific.
2811	*/
2812	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2813	{
2814	#ifdef RT_ARCH_AMD64
2815	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2816	#elif defined(RT_ARCH_ARM64)
2817	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2818	#else
2819	# error "Port me!"
2820	#endif
2821	}
2822
2823
2824	/**
2825	* Fixes up a conditional jump to a fixed label.
2826	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2827	*/
2828	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2829	{
2830	# if defined(RT_ARCH_AMD64)
2831	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2832	if (pbCodeBuf[offFixup] != 0x0f)
2833	{
2834	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2835	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2836	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2837	}
2838	else
2839	{
2840	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2841	uint32_t const offRel32 = offTarget - (offFixup + 6);
2842	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2843	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2844	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2845	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2846	}
2847
2848	# elif defined(RT_ARCH_ARM64)
2849	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2850	Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
2851	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
2852	\| (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
2853
2854	# endif
2855	}
2856
2857
2858	/**
2859	* Internal helper, don't call directly.
2860	*/
2861	DECL_INLINE_THROW(uint32_t)
2862	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2863	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
2864	{
2865	Assert(iBitNo < 64);
2866	#ifdef RT_ARCH_AMD64
2867	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2868	if (iBitNo < 8)
2869	{
2870	/* test Eb, imm8 */
2871	if (iGprSrc >= 4)
2872	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2873	pbCodeBuf[off++] = 0xf6;
2874	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2875	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2876	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2877	}
2878	else
2879	{
2880	/* bt Ev, imm8 */
2881	if (iBitNo >= 32)
2882	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2883	else if (iGprSrc >= 8)
2884	pbCodeBuf[off++] = X86_OP_REX_B;
2885	pbCodeBuf[off++] = 0x0f;
2886	pbCodeBuf[off++] = 0xba;
2887	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2888	pbCodeBuf[off++] = iBitNo;
2889	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2890	}
2891
2892	#elif defined(RT_ARCH_ARM64)
2893	/* Use the TBNZ instruction here. */
2894	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2895	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
2896	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2897
2898	#else
2899	# error "Port me!"
2900	#endif
2901	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2902	return off;
2903	}
2904
2905
2906	/**
2907	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2908	* @a iGprSrc.
2909	*
2910	* @note On ARM64 the range is only +/-8191 instructions.
2911	*/
2912	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2913	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2914	{
2915	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
2916	}
2917
2918
2919	/**
2920	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2921	* _set_ in @a iGprSrc.
2922	*
2923	* @note On ARM64 the range is only +/-8191 instructions.
2924	*/
2925	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2926	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2927	{
2928	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
2929	}
2930
2931
2932	/**
2933	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2934	* flags accordingly.
2935	*/
2936	DECL_INLINE_THROW(uint32_t)
2937	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2938	{
2939	Assert(fBits != 0);
2940	#ifdef RT_ARCH_AMD64
2941
2942	if (fBits >= UINT32_MAX)
2943	{
2944	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2945
2946	/* test Ev,Gv */
2947	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2948	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2949	pbCodeBuf[off++] = 0x85;
2950	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2951
2952	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2953	}
2954	else if (fBits <= UINT32_MAX)
2955	{
2956	/* test Eb, imm8 or test Ev, imm32 */
2957	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2958	if (fBits <= UINT8_MAX)
2959	{
2960	if (iGprSrc >= 4)
2961	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2962	pbCodeBuf[off++] = 0xf6;
2963	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2964	pbCodeBuf[off++] = (uint8_t)fBits;
2965	}
2966	else
2967	{
2968	if (iGprSrc >= 8)
2969	pbCodeBuf[off++] = X86_OP_REX_B;
2970	pbCodeBuf[off++] = 0xf7;
2971	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2972	pbCodeBuf[off++] = RT_BYTE1(fBits);
2973	pbCodeBuf[off++] = RT_BYTE2(fBits);
2974	pbCodeBuf[off++] = RT_BYTE3(fBits);
2975	pbCodeBuf[off++] = RT_BYTE4(fBits);
2976	}
2977	}
2978	/** @todo implement me. */
2979	else
2980	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
2981
2982	#elif defined(RT_ARCH_ARM64)
2983
2984	if (false)
2985	{
2986	/** @todo figure out how to work the immr / N:imms constants. */
2987	}
2988	else
2989	{
2990	/* ands Zr, iGprSrc, iTmpReg */
2991	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2992	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2993	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2994	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2995	}
2996
2997	#else
2998	# error "Port me!"
2999	#endif
3000	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3001	return off;
3002	}
3003
3004
3005	/**
3006	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
3007	* are set in @a iGprSrc.
3008	*/
3009	DECL_INLINE_THROW(uint32_t)
3010	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3011	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3012	{
3013	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3014
3015	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3016	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3017
3018	return off;
3019	}
3020
3021
3022	/**
3023	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
3024	* are set in @a iGprSrc.
3025	*/
3026	DECL_INLINE_THROW(uint32_t)
3027	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3028	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3029	{
3030	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3031
3032	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3033	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3034
3035	return off;
3036	}
3037
3038
3039	/**
3040	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
3041	*
3042	* The operand size is given by @a f64Bit.
3043	*/
3044	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3045	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3046	{
3047	Assert(idxLabel < pReNative->cLabels);
3048
3049	#ifdef RT_ARCH_AMD64
3050	/* test reg32,reg32 / test reg64,reg64 */
3051	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3052	if (f64Bit)
3053	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
3054	else if (iGprSrc >= 8)
3055	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
3056	pbCodeBuf[off++] = 0x85;
3057	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3058	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3059
3060	/* jz idxLabel */
3061	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3062
3063	#elif defined(RT_ARCH_ARM64)
3064	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3065	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3066	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, f64Bit);
3067	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3068
3069	#else
3070	# error "Port me!"
3071	#endif
3072	return off;
3073	}
3074
3075
3076	/**
3077	* Emits code that jumps to a new label if @a iGprSrc is zero.
3078	*
3079	* The operand size is given by @a f64Bit.
3080	*/
3081	DECL_INLINE_THROW(uint32_t)
3082	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3083	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3084	{
3085	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3086	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3087	}
3088
3089
3090	/**
3091	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3092	* differs.
3093	*/
3094	DECL_INLINE_THROW(uint32_t)
3095	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3096	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3097	{
3098	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3099	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3100	return off;
3101	}
3102
3103
3104	/**
3105	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3106	*/
3107	DECL_INLINE_THROW(uint32_t)
3108	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3109	uint8_t iGprLeft, uint8_t iGprRight,
3110	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3111	{
3112	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3113	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3114	}
3115
3116
3117	/**
3118	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3119	*/
3120	DECL_INLINE_THROW(uint32_t)
3121	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3122	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3123	{
3124	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3125	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3126	return off;
3127	}
3128
3129
3130	/**
3131	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3132	*/
3133	DECL_INLINE_THROW(uint32_t)
3134	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3135	uint8_t iGprSrc, uint64_t uImm,
3136	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3137	{
3138	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3139	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3140	}
3141
3142
3143	/**
3144	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3145	* @a uImm.
3146	*/
3147	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3148	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3149	{
3150	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3151	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3152	return off;
3153	}
3154
3155
3156	/**
3157	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3158	* @a uImm.
3159	*/
3160	DECL_INLINE_THROW(uint32_t)
3161	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3162	uint8_t iGprSrc, uint32_t uImm,
3163	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3164	{
3165	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3166	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3167	}
3168
3169
3170
3171	/**
3172	* Emits a call to a 64-bit address.
3173	*/
3174	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3175	{
3176	#ifdef RT_ARCH_AMD64
3177	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3178
3179	/* call rax */
3180	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3181	pbCodeBuf[off++] = 0xff;
3182	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3183
3184	#elif defined(RT_ARCH_ARM64)
3185	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3186
3187	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3188	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3189
3190	#else
3191	# error "port me"
3192	#endif
3193	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3194	return off;
3195	}
3196
3197
3198	/** @} */
3199
3200	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3201

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101722

Download in other formats: