IEMN8veRecompiler.h@ 101844

Last change on this file since 101844 was 101844, checked in by vboxsync, 17 months ago
VMM/IEM: Native translation of IEM_MC_CALL_CIMPL_1_THREADED and IEM_MC_FETCH_GREG_U16, enabling recompilation of 16-bit indirect calls and ltr/lldt reg16 instructions. This also includes basic argument and variable handling. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 122.5 KB

Line
1	/* $Id: IEMN8veRecompiler.h 101844 2023-11-06 02:05:35Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46
47	/** @name Stack Frame Layout
48	*
49	* @{ */
50	/** The size of the area for stack variables and spills and stuff.
51	* @note This limit is duplicated in the python script(s). We add 0x40 for
52	* alignment padding. */
53	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
54	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
55	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
56	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
57
58	#ifdef RT_ARCH_AMD64
59	/** An stack alignment adjustment (between non-volatile register pushes and
60	* the stack variable area, so the latter better aligned). */
61	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
62
63	/** Number of stack arguments slots for calls made from the frame. */
64	# ifdef RT_OS_WINDOWS
65	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
66	# else
67	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
68	# endif
69	/** Number of any shadow arguments (spill area) for calls we make. */
70	# ifdef RT_OS_WINDOWS
71	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
72	# else
73	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
74	# endif
75
76	/** Frame pointer (RBP) relative offset of the last push. */
77	# ifdef RT_OS_WINDOWS
78	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
79	# else
80	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
81	# endif
82	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
83	* address for it). */
84	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
85	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
86	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
87	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
88	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
89	# ifdef RT_OS_WINDOWS
90	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
91	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
92	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
93	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
94	# endif
95
96	# ifdef RT_OS_WINDOWS
97	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
98	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
99	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
100	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
101	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
102	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
103	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
104	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
105	# endif
106
107	#elif RT_ARCH_ARM64
108	/** No alignment padding needed for arm64. */
109	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
110	/** No stack argument slots, got 8 registers for arguments will suffice. */
111	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
112	/** There are no argument spill area. */
113	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
114
115	/** Number of saved registers at the top of our stack frame.
116	* This includes the return address and old frame pointer, so x19 thru x30. */
117	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
118	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
119	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
120
121	/** Frame pointer (BP) relative offset of the last push. */
122	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
123
124	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
125	* address for it). */
126	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
127
128	#else
129	# error "port me"
130	#endif
131	/** @} */
132
133
134	/** @name Fixed Register Allocation(s)
135	* @{ */
136	/** @def IEMNATIVE_REG_FIXED_PVMCPU
137	* The number of the register holding the pVCpu pointer. */
138	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
139	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
140	* @note This not available on AMD64, only ARM64. */
141	/** @def IEMNATIVE_REG_FIXED_TMP0
142	* Dedicated temporary register.
143	* @todo replace this by a register allocator and content tracker. */
144	/** @def IEMNATIVE_REG_FIXED_MASK
145	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
146	* architecture. */
147	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
148	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
149	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
150	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
151	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
152	\| RT_BIT_32(X86_GREG_xSP) \
153	\| RT_BIT_32(X86_GREG_xBP) )
154
155	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
156	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
157	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
158	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
159	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
160	\| RT_BIT_32(ARMV8_A64_REG_LR) \
161	\| RT_BIT_32(ARMV8_A64_REG_BP) \
162	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
163	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
164	\| RT_BIT_32(ARMV8_A64_REG_X18) \
165	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
166
167	#else
168	# error "port me"
169	#endif
170	/** @} */
171
172	/** @name Call related registers.
173	* @{ */
174	/** @def IEMNATIVE_CALL_RET_GREG
175	* The return value register. */
176	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
177	* Number of arguments in registers. */
178	/** @def IEMNATIVE_CALL_ARG0_GREG
179	* The general purpose register carrying argument \#0. */
180	/** @def IEMNATIVE_CALL_ARG1_GREG
181	* The general purpose register carrying argument \#1. */
182	/** @def IEMNATIVE_CALL_ARG2_GREG
183	* The general purpose register carrying argument \#2. */
184	/** @def IEMNATIVE_CALL_ARG3_GREG
185	* The general purpose register carrying argument \#3. */
186	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
187	* Mask of registers the callee will not save and may trash. */
188	#ifdef RT_ARCH_AMD64
189	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
190
191	# ifdef RT_OS_WINDOWS
192	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
193	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
194	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
195	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
196	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
197	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
198	\| RT_BIT_32(X86_GREG_xCX) \
199	\| RT_BIT_32(X86_GREG_xDX) \
200	\| RT_BIT_32(X86_GREG_x8) \
201	\| RT_BIT_32(X86_GREG_x9) \
202	\| RT_BIT_32(X86_GREG_x10) \
203	\| RT_BIT_32(X86_GREG_x11) )
204	# else
205	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
206	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
207	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
208	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
209	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
210	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
211	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
212	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
213	\| RT_BIT_32(X86_GREG_xCX) \
214	\| RT_BIT_32(X86_GREG_xDX) \
215	\| RT_BIT_32(X86_GREG_xDI) \
216	\| RT_BIT_32(X86_GREG_xSI) \
217	\| RT_BIT_32(X86_GREG_x8) \
218	\| RT_BIT_32(X86_GREG_x9) \
219	\| RT_BIT_32(X86_GREG_x10) \
220	\| RT_BIT_32(X86_GREG_x11) )
221	# endif
222
223	#elif defined(RT_ARCH_ARM64)
224	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
225	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
226	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
227	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
228	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
229	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
230	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
231	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
232	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
233	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
234	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
235	\| RT_BIT_32(ARMV8_A64_REG_X1) \
236	\| RT_BIT_32(ARMV8_A64_REG_X2) \
237	\| RT_BIT_32(ARMV8_A64_REG_X3) \
238	\| RT_BIT_32(ARMV8_A64_REG_X4) \
239	\| RT_BIT_32(ARMV8_A64_REG_X5) \
240	\| RT_BIT_32(ARMV8_A64_REG_X6) \
241	\| RT_BIT_32(ARMV8_A64_REG_X7) \
242	\| RT_BIT_32(ARMV8_A64_REG_X8) \
243	\| RT_BIT_32(ARMV8_A64_REG_X9) \
244	\| RT_BIT_32(ARMV8_A64_REG_X10) \
245	\| RT_BIT_32(ARMV8_A64_REG_X11) \
246	\| RT_BIT_32(ARMV8_A64_REG_X12) \
247	\| RT_BIT_32(ARMV8_A64_REG_X13) \
248	\| RT_BIT_32(ARMV8_A64_REG_X14) \
249	\| RT_BIT_32(ARMV8_A64_REG_X15) \
250	\| RT_BIT_32(ARMV8_A64_REG_X16) \
251	\| RT_BIT_32(ARMV8_A64_REG_X17) )
252
253	#endif
254
255	/** This is the maximum argument count we'll ever be needing. */
256	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
257	/** @} */
258
259
260	/** @def IEMNATIVE_HST_GREG_COUNT
261	* Number of host general purpose registers we tracker. */
262	/** @def IEMNATIVE_HST_GREG_MASK
263	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
264	* inverted register masks and such to get down to a correct set of regs. */
265	#ifdef RT_ARCH_AMD64
266	# define IEMNATIVE_HST_GREG_COUNT 16
267	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
268
269	#elif defined(RT_ARCH_ARM64)
270	# define IEMNATIVE_HST_GREG_COUNT 32
271	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
272	#else
273	# error "Port me!"
274	#endif
275
276
277	/** Native code generator label types. */
278	typedef enum
279	{
280	kIemNativeLabelType_Invalid = 0,
281	/* Labels w/o data, only once instance per TB: */
282	kIemNativeLabelType_Return,
283	kIemNativeLabelType_ReturnBreak,
284	kIemNativeLabelType_ReturnWithFlags,
285	kIemNativeLabelType_NonZeroRetOrPassUp,
286	kIemNativeLabelType_RaiseGp0,
287	/* Labels with data, potentially multiple instances per TB: */
288	kIemNativeLabelType_If,
289	kIemNativeLabelType_Else,
290	kIemNativeLabelType_Endif,
291	kIemNativeLabelType_CheckIrq,
292	kIemNativeLabelType_End
293	} IEMNATIVELABELTYPE;
294
295	/** Native code generator label definition. */
296	typedef struct IEMNATIVELABEL
297	{
298	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
299	* the epilog. */
300	uint32_t off;
301	/** The type of label (IEMNATIVELABELTYPE). */
302	uint16_t enmType;
303	/** Additional label data, type specific. */
304	uint16_t uData;
305	} IEMNATIVELABEL;
306	/** Pointer to a label. */
307	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
308
309
310	/** Native code generator fixup types. */
311	typedef enum
312	{
313	kIemNativeFixupType_Invalid = 0,
314	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
315	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
316	kIemNativeFixupType_Rel32,
317	#elif defined(RT_ARCH_ARM64)
318	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
319	kIemNativeFixupType_RelImm19At5,
320	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
321	kIemNativeFixupType_RelImm14At5,
322	#endif
323	kIemNativeFixupType_End
324	} IEMNATIVEFIXUPTYPE;
325
326	/** Native code generator fixup. */
327	typedef struct IEMNATIVEFIXUP
328	{
329	/** Code offset of the fixup location. */
330	uint32_t off;
331	/** The IEMNATIVELABEL this is a fixup for. */
332	uint16_t idxLabel;
333	/** The fixup type (IEMNATIVEFIXUPTYPE). */
334	uint8_t enmType;
335	/** Addend or other data. */
336	int8_t offAddend;
337	} IEMNATIVEFIXUP;
338	/** Pointer to a native code generator fixup. */
339	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
340
341
342	/**
343	* Guest registers that can be shadowed in GPRs.
344	*/
345	typedef enum IEMNATIVEGSTREG : uint8_t
346	{
347	kIemNativeGstReg_GprFirst = 0,
348	kIemNativeGstReg_GprLast = 15,
349	kIemNativeGstReg_Pc,
350	kIemNativeGstReg_EFlags, /*< This one is problematic since the higher bits are used internally. /
351	/* gap: 18..23 */
352	kIemNativeGstReg_SegSelFirst = 24,
353	kIemNativeGstReg_SegSelLast = 29,
354	kIemNativeGstReg_SegBaseFirst = 30,
355	kIemNativeGstReg_SegBaseLast = 35,
356	kIemNativeGstReg_SegLimitFirst = 36,
357	kIemNativeGstReg_SegLimitLast = 41,
358	kIemNativeGstReg_End
359	} IEMNATIVEGSTREG;
360
361	/**
362	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
363	*/
364	typedef enum IEMNATIVEGSTREGUSE
365	{
366	/** The usage is read-only, the register holding the guest register
367	* shadow copy will not be modified by the caller. */
368	kIemNativeGstRegUse_ReadOnly = 0,
369	/** The caller will update the guest register (think: PC += cbInstr).
370	* The guest shadow copy will follow the returned register. */
371	kIemNativeGstRegUse_ForUpdate,
372	/** The caller will use the guest register value as input in a calculation
373	* and the host register will be modified.
374	* This means that the returned host register will not be marked as a shadow
375	* copy of the guest register. */
376	kIemNativeGstRegUse_Calculation
377	} IEMNATIVEGSTREGUSE;
378
379	/**
380	* Guest registers (classes) that can be referenced.
381	*/
382	typedef enum IEMNATIVEGSTREGREF : uint8_t
383	{
384	kIemNativeGstRegRef_Invalid = 0,
385	kIemNativeGstRegRef_Gpr,
386	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
387	kIemNativeGstRegRef_EFlags,
388	kIemNativeGstRegRef_MxCsr,
389	kIemNativeGstRegRef_FpuReg,
390	kIemNativeGstRegRef_MReg,
391	kIemNativeGstRegRef_XReg,
392	//kIemNativeGstRegRef_YReg, - doesn't work.
393	kIemNativeGstRegRef_End
394	} IEMNATIVEGSTREGREF;
395
396
397	/** Variable kinds. */
398	typedef enum IEMNATIVEVARKIND : uint8_t
399	{
400	/** Customary invalid zero value. */
401	kIemNativeVarKind_Invalid = 0,
402	/** This is either in a register or on the stack. */
403	kIemNativeVarKind_Stack,
404	/** Immediate value - loaded into register when needed, or can live on the
405	* stack if referenced (in theory). */
406	kIemNativeVarKind_Immediate,
407	/** Variable reference - loaded into register when needed, never stack. */
408	kIemNativeVarKind_VarRef,
409	/** Guest register reference - loaded into register when needed, never stack. */
410	kIemNativeVarKind_GstRegRef,
411	/** End of valid values. */
412	kIemNativeVarKind_End
413	} IEMNATIVEVARKIND;
414
415
416	/** Variable or argument. */
417	typedef struct IEMNATIVEVAR
418	{
419	/** The kind of variable. */
420	IEMNATIVEVARKIND enmKind;
421	/** The variable size in bytes. */
422	uint8_t cbVar;
423	/** The first stack slot (uint64_t), except for immediate and references
424	* where it usually is UINT8_MAX. */
425	uint8_t idxStackSlot;
426	/** The host register allocated for the variable, UINT8_MAX if not. */
427	uint8_t idxReg;
428	/** The argument number if argument, UINT8_MAX if regular variable. */
429	uint8_t uArgNo;
430	/** If referenced, the index of the variable referencing this one, otherwise
431	* UINT8_MAX. A referenced variable must only be placed on the stack and
432	* must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
433	uint8_t idxReferrerVar;
434	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
435	IEMNATIVEGSTREG enmGstReg;
436	uint8_t bAlign;
437
438	union
439	{
440	/** kIemNativeVarKind_Immediate: The immediate value. */
441	uint64_t uValue;
442	/** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
443	uint8_t idxRefVar;
444	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
445	struct
446	{
447	/** The class of register. */
448	IEMNATIVEGSTREGREF enmClass;
449	/** Index within the class. */
450	uint8_t idx;
451	} GstRegRef;
452	} u;
453	} IEMNATIVEVAR;
454
455	/** What is being kept in a host register. */
456	typedef enum IEMNATIVEWHAT : uint8_t
457	{
458	/** The traditional invalid zero value. */
459	kIemNativeWhat_Invalid = 0,
460	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
461	kIemNativeWhat_Var,
462	/** Temporary register, this is typically freed when a MC completes. */
463	kIemNativeWhat_Tmp,
464	/** Call argument w/o a variable mapping. This is free (via
465	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
466	kIemNativeWhat_Arg,
467	/** Return status code.
468	* @todo not sure if we need this... */
469	kIemNativeWhat_rc,
470	/** The fixed pVCpu (PVMCPUCC) register.
471	* @todo consider offsetting this on amd64 to use negative offsets to access
472	* more members using 8-byte disp. */
473	kIemNativeWhat_pVCpuFixed,
474	/** The fixed pCtx (PCPUMCTX) register.
475	* @todo consider offsetting this on amd64 to use negative offsets to access
476	* more members using 8-byte disp. */
477	kIemNativeWhat_pCtxFixed,
478	/** Fixed temporary register. */
479	kIemNativeWhat_FixedTmp,
480	/** Register reserved by the CPU or OS architecture. */
481	kIemNativeWhat_FixedReserved,
482	/** End of valid values. */
483	kIemNativeWhat_End
484	} IEMNATIVEWHAT;
485
486	/**
487	* Host general register entry.
488	*
489	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
490	*
491	* @todo Track immediate values in host registers similarlly to how we track the
492	* guest register shadow copies. For it to be real helpful, though,
493	* we probably need to know which will be reused and put them into
494	* non-volatile registers, otherwise it's going to be more or less
495	* restricted to an instruction or two.
496	*/
497	typedef struct IEMNATIVEHSTREG
498	{
499	/** Set of guest registers this one shadows.
500	*
501	* Using a bitmap here so we can designate the same host register as a copy
502	* for more than one guest register. This is expected to be useful in
503	* situations where one value is copied to several registers in a sequence.
504	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
505	* sequence we'd want to let this register follow to be a copy of and there
506	* will always be places where we'd be picking the wrong one.
507	*/
508	uint64_t fGstRegShadows;
509	/** What is being kept in this register. */
510	IEMNATIVEWHAT enmWhat;
511	/** Variable index if holding a variable, otherwise UINT8_MAX. */
512	uint8_t idxVar;
513	/** Alignment padding. */
514	uint8_t abAlign[6];
515	} IEMNATIVEHSTREG;
516
517
518	/**
519	* Core state for the native recompiler, that is, things that needs careful
520	* handling when dealing with branches.
521	*/
522	typedef struct IEMNATIVECORESTATE
523	{
524	/** Allocation bitmap for aHstRegs. */
525	uint32_t bmHstRegs;
526
527	/** Bitmap marking which host register contains guest register shadow copies.
528	* This is used during register allocation to try preserve copies. */
529	uint32_t bmHstRegsWithGstShadow;
530	/** Bitmap marking valid entries in aidxGstRegShadows. */
531	uint64_t bmGstRegShadows;
532
533	union
534	{
535	/** Index of variable arguments, UINT8_MAX if not valid. */
536	uint8_t aidxArgVars[8];
537	/** For more efficient resetting. */
538	uint64_t u64ArgVars;
539	};
540
541	/** Allocation bitmap for the stack. */
542	uint32_t bmStack;
543	/** Allocation bitmap for aVars. */
544	uint32_t bmVars;
545
546	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
547	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
548	* (A shadow copy of a guest register can only be held in a one host register,
549	* there are no duplicate copies or ambiguities like that). */
550	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
551
552	/** Host register allocation tracking. */
553	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
554
555	/** Variables and arguments. */
556	IEMNATIVEVAR aVars[9];
557	} IEMNATIVECORESTATE;
558	/** Pointer to core state. */
559	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
560	/** Pointer to const core state. */
561	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
562
563
564	/**
565	* Conditional stack entry.
566	*/
567	typedef struct IEMNATIVECOND
568	{
569	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
570	bool fInElse;
571	/** The label for the IEM_MC_ELSE. */
572	uint32_t idxLabelElse;
573	/** The label for the IEM_MC_ENDIF. */
574	uint32_t idxLabelEndIf;
575	/** The initial state snapshot as the if-block starts executing. */
576	IEMNATIVECORESTATE InitialState;
577	/** The state snapshot at the end of the if-block. */
578	IEMNATIVECORESTATE IfFinalState;
579	} IEMNATIVECOND;
580	/** Pointer to a condition stack entry. */
581	typedef IEMNATIVECOND *PIEMNATIVECOND;
582
583
584	/**
585	* Native recompiler state.
586	*/
587	typedef struct IEMRECOMPILERSTATE
588	{
589	/** Size of the buffer that pbNativeRecompileBufR3 points to in
590	* IEMNATIVEINSTR units. */
591	uint32_t cInstrBufAlloc;
592	#ifdef VBOX_STRICT
593	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
594	uint32_t offInstrBufChecked;
595	#else
596	uint32_t uPadding1; /* We don't keep track of the size here... */
597	#endif
598	/** Fixed temporary code buffer for native recompilation. */
599	PIEMNATIVEINSTR pInstrBuf;
600
601	/** Bitmaps with the label types used. */
602	uint64_t bmLabelTypes;
603	/** Actual number of labels in paLabels. */
604	uint32_t cLabels;
605	/** Max number of entries allowed in paLabels before reallocating it. */
606	uint32_t cLabelsAlloc;
607	/** Labels defined while recompiling (referenced by fixups). */
608	PIEMNATIVELABEL paLabels;
609
610	/** Actual number of fixups paFixups. */
611	uint32_t cFixups;
612	/** Max number of entries allowed in paFixups before reallocating it. */
613	uint32_t cFixupsAlloc;
614	/** Buffer used by the recompiler for recording fixups when generating code. */
615	PIEMNATIVEFIXUP paFixups;
616
617	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
618	/** Number of debug info entries allocated for pDbgInfo. */
619	uint32_t cDbgInfoAlloc;
620	uint32_t uPadding;
621	/** Debug info. */
622	PIEMTBDBG pDbgInfo;
623	#endif
624
625	/** The translation block being recompiled. */
626	PCIEMTB pTbOrg;
627
628	/** The current condition stack depth (aCondStack). */
629	uint8_t cCondDepth;
630	uint8_t bPadding2;
631	/** Condition sequence number (for generating unique labels). */
632	uint16_t uCondSeqNo;
633	/** Check IRQ seqeunce number (for generating unique lables). */
634	uint16_t uCheckIrqSeqNo;
635	uint8_t bPadding3;
636
637	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
638	uint8_t cArgs;
639	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
640	uint32_t fCImpl;
641	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
642	uint32_t fMc;
643
644	/** Core state requiring care with branches. */
645	IEMNATIVECORESTATE Core;
646
647	/** The condition nesting stack. */
648	IEMNATIVECOND aCondStack[2];
649
650	#ifndef IEM_WITH_THROW_CATCH
651	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
652	* for recompilation error handling. */
653	jmp_buf JmpBuf;
654	#endif
655	} IEMRECOMPILERSTATE;
656	/** Pointer to a native recompiler state. */
657	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
658
659
660	/** @def IEMNATIVE_TRY_SETJMP
661	* Wrapper around setjmp / try, hiding all the ugly differences.
662	*
663	* @note Use with extreme care as this is a fragile macro.
664	* @param a_pReNative The native recompile state.
665	* @param a_rcTarget The variable that should receive the status code in case
666	* of a longjmp/throw.
667	*/
668	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
669	* Start wrapper for catch / setjmp-else.
670	*
671	* This will set up a scope.
672	*
673	* @note Use with extreme care as this is a fragile macro.
674	* @param a_pReNative The native recompile state.
675	* @param a_rcTarget The variable that should receive the status code in case
676	* of a longjmp/throw.
677	*/
678	/** @def IEMNATIVE_CATCH_LONGJMP_END
679	* End wrapper for catch / setjmp-else.
680	*
681	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
682	* up the state.
683	*
684	* @note Use with extreme care as this is a fragile macro.
685	* @param a_pReNative The native recompile state.
686	*/
687	/** @def IEMNATIVE_DO_LONGJMP
688	*
689	* Wrapper around longjmp / throw.
690	*
691	* @param a_pReNative The native recompile state.
692	* @param a_rc The status code jump back with / throw.
693	*/
694	#ifdef IEM_WITH_THROW_CATCH
695	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
696	a_rcTarget = VINF_SUCCESS; \
697	try
698	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
699	catch (int rcThrown) \
700	{ \
701	a_rcTarget = rcThrown
702	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
703	} \
704	((void)0)
705	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
706	#else /* !IEM_WITH_THROW_CATCH */
707	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
708	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
709	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
710	else \
711	{ \
712	((void)0)
713	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
714	}
715	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
716	#endif /* !IEM_WITH_THROW_CATCH */
717
718
719	/**
720	* Native recompiler worker for a threaded function.
721	*
722	* @returns New code buffer offset; throws VBox status code in case of a failure.
723	* @param pReNative The native recompiler state.
724	* @param off The current code buffer offset.
725	* @param pCallEntry The threaded call entry.
726	*
727	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
728	*/
729	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
730	/** Pointer to a native recompiler worker for a threaded function. */
731	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
732
733	/** Defines a native recompiler worker for a threaded function.
734	* @see FNIEMNATIVERECOMPFUNC */
735	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
736	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
737
738	/** Prototypes a native recompiler function for a threaded function.
739	* @see FNIEMNATIVERECOMPFUNC */
740	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
741
742	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
743	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
744	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
745	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
746	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
747	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
748
749	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
750	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
751	bool fPreferVolatile = true);
752	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
753	IEMNATIVEGSTREG enmGstReg, IEMNATIVEGSTREGUSE enmIntendedUse);
754	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
755	IEMNATIVEGSTREG enmGstReg);
756
757	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
758	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
759	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
760	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
761	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
762	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
763	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
764	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
765
766	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
767	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
768	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
769
770
771	/**
772	* Ensures that there is sufficient space in the instruction output buffer.
773	*
774	* This will reallocate the buffer if needed and allowed.
775	*
776	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
777	* allocation size.
778	*
779	* @returns Pointer to the instruction output buffer on success; throws VBox
780	* status code on failure, so no need to check it.
781	* @param pReNative The native recompile state.
782	* @param off Current instruction offset. Works safely for UINT32_MAX
783	* as well.
784	* @param cInstrReq Number of instruction about to be added. It's okay to
785	* overestimate this a bit.
786	*/
787	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
788	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
789	{
790	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
791	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
792	{
793	#ifdef VBOX_STRICT
794	pReNative->offInstrBufChecked = offChecked;
795	#endif
796	return pReNative->pInstrBuf;
797	}
798	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
799	}
800
801	/**
802	* Checks that we didn't exceed the space requested in the last
803	* iemNativeInstrBufEnsure() call. */
804	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
805	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
806	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
807
808
809	/**
810	* Emit a simple marker instruction to more easily tell where something starts
811	* in the disassembly.
812	*/
813	DECL_INLINE_THROW(uint32_t)
814	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
815	{
816	#ifdef RT_ARCH_AMD64
817	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
818	if (uInfo == 0)
819	{
820	/* nop */
821	pbCodeBuf[off++] = 0x90;
822	}
823	else
824	{
825	/* nop [disp32] */
826	pbCodeBuf[off++] = 0x0f;
827	pbCodeBuf[off++] = 0x1f;
828	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
829	pbCodeBuf[off++] = RT_BYTE1(uInfo);
830	pbCodeBuf[off++] = RT_BYTE2(uInfo);
831	pbCodeBuf[off++] = RT_BYTE3(uInfo);
832	pbCodeBuf[off++] = RT_BYTE4(uInfo);
833	}
834	#elif RT_ARCH_ARM64
835	/* nop */
836	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
837	pu32CodeBuf[off++] = 0xd503201f;
838
839	RT_NOREF(uInfo);
840	#else
841	# error "port me"
842	#endif
843	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
844	return off;
845	}
846
847
848	/*********************************************************************************************************************************
849	* Loads, Stores and Related Stuff. *
850	*********************************************************************************************************************************/
851
852	/**
853	* Emits setting a GPR to zero.
854	*/
855	DECL_INLINE_THROW(uint32_t)
856	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
857	{
858	#ifdef RT_ARCH_AMD64
859	/* xor gpr32, gpr32 */
860	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
861	if (iGpr >= 8)
862	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
863	pbCodeBuf[off++] = 0x33;
864	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
865
866	#elif RT_ARCH_ARM64
867	/* mov gpr, #0x0 */
868	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
869	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
870
871	#else
872	# error "port me"
873	#endif
874	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
875	return off;
876	}
877
878
879	/**
880	* Emits loading a constant into a 64-bit GPR
881	*/
882	DECL_INLINE_THROW(uint32_t)
883	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
884	{
885	if (!uImm64)
886	return iemNativeEmitGprZero(pReNative, off, iGpr);
887
888	#ifdef RT_ARCH_AMD64
889	if (uImm64 <= UINT32_MAX)
890	{
891	/* mov gpr, imm32 */
892	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
893	if (iGpr >= 8)
894	pbCodeBuf[off++] = X86_OP_REX_B;
895	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
896	pbCodeBuf[off++] = RT_BYTE1(uImm64);
897	pbCodeBuf[off++] = RT_BYTE2(uImm64);
898	pbCodeBuf[off++] = RT_BYTE3(uImm64);
899	pbCodeBuf[off++] = RT_BYTE4(uImm64);
900	}
901	else
902	{
903	/* mov gpr, imm64 */
904	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
905	if (iGpr < 8)
906	pbCodeBuf[off++] = X86_OP_REX_W;
907	else
908	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
909	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
910	pbCodeBuf[off++] = RT_BYTE1(uImm64);
911	pbCodeBuf[off++] = RT_BYTE2(uImm64);
912	pbCodeBuf[off++] = RT_BYTE3(uImm64);
913	pbCodeBuf[off++] = RT_BYTE4(uImm64);
914	pbCodeBuf[off++] = RT_BYTE5(uImm64);
915	pbCodeBuf[off++] = RT_BYTE6(uImm64);
916	pbCodeBuf[off++] = RT_BYTE7(uImm64);
917	pbCodeBuf[off++] = RT_BYTE8(uImm64);
918	}
919
920	#elif RT_ARCH_ARM64
921	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
922
923	/*
924	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
925	* supply remaining bits using 'movk grp, imm16, lsl #x'.
926	*
927	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
928	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
929	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
930	* after the first non-zero immediate component so we switch to movk for
931	* the remainder.
932	*/
933	uint32_t fMovK = 0;
934	/* mov gpr, imm16 */
935	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
936	if (uImmPart)
937	{
938	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
939	fMovK \|= RT_BIT_32(29);
940	}
941	/* mov[k] gpr, imm16, lsl #16 */
942	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
943	if (uImmPart)
944	{
945	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
946	fMovK \|= RT_BIT_32(29);
947	}
948	/* mov[k] gpr, imm16, lsl #32 */
949	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
950	if (uImmPart)
951	{
952	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
953	fMovK \|= RT_BIT_32(29);
954	}
955	/* mov[k] gpr, imm16, lsl #48 */
956	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
957	if (uImmPart)
958	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
959
960	/** @todo there is an inverted mask variant we might want to explore if it
961	* reduces the number of instructions... */
962	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
963	* clang 12.x does that, only to use the 'x' version for the
964	* addressing in the following ldr). */
965
966	#else
967	# error "port me"
968	#endif
969	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
970	return off;
971	}
972
973
974	/**
975	* Emits loading a constant into a 8-bit GPR
976	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
977	* only the ARM64 version does that.
978	*/
979	DECL_INLINE_THROW(uint32_t)
980	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
981	{
982	#ifdef RT_ARCH_AMD64
983	/* mov gpr, imm8 */
984	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
985	if (iGpr >= 8)
986	pbCodeBuf[off++] = X86_OP_REX_B;
987	else if (iGpr >= 4)
988	pbCodeBuf[off++] = X86_OP_REX;
989	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
990	pbCodeBuf[off++] = RT_BYTE1(uImm8);
991
992	#elif RT_ARCH_ARM64
993	/* movz gpr, imm16, lsl #0 */
994	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
995	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
996
997	#else
998	# error "port me"
999	#endif
1000	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1001	return off;
1002	}
1003
1004
1005	#ifdef RT_ARCH_AMD64
1006	/**
1007	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1008	*/
1009	DECL_FORCE_INLINE(uint32_t)
1010	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
1011	{
1012	if (offVCpu < 128)
1013	{
1014	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
1015	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
1016	}
1017	else
1018	{
1019	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
1020	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
1021	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
1022	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
1023	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
1024	}
1025	return off;
1026	}
1027	#elif RT_ARCH_ARM64
1028	/**
1029	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1030	*/
1031	DECL_FORCE_INLINE_THROW(uint32_t)
1032	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1033	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1034	{
1035	/*
1036	* There are a couple of ldr variants that takes an immediate offset, so
1037	* try use those if we can, otherwise we have to use the temporary register
1038	* help with the addressing.
1039	*/
1040	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
1041	{
1042	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1043	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1044	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
1045	}
1046	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
1047	{
1048	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1049	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
1050	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
1051	}
1052	else
1053	{
1054	/* The offset is too large, so we must load it into a register and use
1055	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1056	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1057	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
1058
1059	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1060	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
1061	}
1062	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1063	return off;
1064	}
1065	#endif
1066
1067
1068	/**
1069	* Emits a 64-bit GPR load of a VCpu value.
1070	*/
1071	DECL_INLINE_THROW(uint32_t)
1072	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1073	{
1074	#ifdef RT_ARCH_AMD64
1075	/* mov reg64, mem64 */
1076	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1077	if (iGpr < 8)
1078	pbCodeBuf[off++] = X86_OP_REX_W;
1079	else
1080	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1081	pbCodeBuf[off++] = 0x8b;
1082	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
1083	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1084
1085	#elif RT_ARCH_ARM64
1086	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1087
1088	#else
1089	# error "port me"
1090	#endif
1091	return off;
1092	}
1093
1094
1095	/**
1096	* Emits a 32-bit GPR load of a VCpu value.
1097	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1098	*/
1099	DECL_INLINE_THROW(uint32_t)
1100	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1101	{
1102	#ifdef RT_ARCH_AMD64
1103	/* mov reg32, mem32 */
1104	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1105	if (iGpr >= 8)
1106	pbCodeBuf[off++] = X86_OP_REX_R;
1107	pbCodeBuf[off++] = 0x8b;
1108	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1109	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1110
1111	#elif RT_ARCH_ARM64
1112	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1113
1114	#else
1115	# error "port me"
1116	#endif
1117	return off;
1118	}
1119
1120
1121	/**
1122	* Emits a 16-bit GPR load of a VCpu value.
1123	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
1124	*/
1125	DECL_INLINE_THROW(uint32_t)
1126	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1127	{
1128	#ifdef RT_ARCH_AMD64
1129	/* movzx reg32, mem16 */
1130	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1131	if (iGpr >= 8)
1132	pbCodeBuf[off++] = X86_OP_REX_R;
1133	pbCodeBuf[off++] = 0x0f;
1134	pbCodeBuf[off++] = 0xb7;
1135	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1136	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1137
1138	#elif RT_ARCH_ARM64
1139	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1140
1141	#else
1142	# error "port me"
1143	#endif
1144	return off;
1145	}
1146
1147
1148	/**
1149	* Emits a 8-bit GPR load of a VCpu value.
1150	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
1151	*/
1152	DECL_INLINE_THROW(uint32_t)
1153	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1154	{
1155	#ifdef RT_ARCH_AMD64
1156	/* movzx reg32, mem8 */
1157	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1158	if (iGpr >= 8)
1159	pbCodeBuf[off++] = X86_OP_REX_R;
1160	pbCodeBuf[off++] = 0x0f;
1161	pbCodeBuf[off++] = 0xb6;
1162	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1164
1165	#elif RT_ARCH_ARM64
1166	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1167
1168	#else
1169	# error "port me"
1170	#endif
1171	return off;
1172	}
1173
1174
1175	/**
1176	* Emits a store of a GPR value to a 64-bit VCpu field.
1177	*/
1178	DECL_INLINE_THROW(uint32_t)
1179	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1180	{
1181	#ifdef RT_ARCH_AMD64
1182	/* mov mem64, reg64 */
1183	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1184	if (iGpr < 8)
1185	pbCodeBuf[off++] = X86_OP_REX_W;
1186	else
1187	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1188	pbCodeBuf[off++] = 0x89;
1189	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1190	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1191
1192	#elif RT_ARCH_ARM64
1193	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1194
1195	#else
1196	# error "port me"
1197	#endif
1198	return off;
1199	}
1200
1201
1202	/**
1203	* Emits a store of a GPR value to a 32-bit VCpu field.
1204	*/
1205	DECL_INLINE_THROW(uint32_t)
1206	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1207	{
1208	#ifdef RT_ARCH_AMD64
1209	/* mov mem32, reg32 */
1210	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1211	if (iGpr >= 8)
1212	pbCodeBuf[off++] = X86_OP_REX_R;
1213	pbCodeBuf[off++] = 0x89;
1214	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1215	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1216
1217	#elif RT_ARCH_ARM64
1218	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1219
1220	#else
1221	# error "port me"
1222	#endif
1223	return off;
1224	}
1225
1226
1227	/**
1228	* Emits a store of a GPR value to a 16-bit VCpu field.
1229	*/
1230	DECL_INLINE_THROW(uint32_t)
1231	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1232	{
1233	#ifdef RT_ARCH_AMD64
1234	/* mov mem16, reg16 */
1235	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1236	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1237	if (iGpr >= 8)
1238	pbCodeBuf[off++] = X86_OP_REX_R;
1239	pbCodeBuf[off++] = 0x89;
1240	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1241	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1242
1243	#elif RT_ARCH_ARM64
1244	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1245
1246	#else
1247	# error "port me"
1248	#endif
1249	return off;
1250	}
1251
1252
1253	/**
1254	* Emits a store of a GPR value to a 8-bit VCpu field.
1255	*/
1256	DECL_INLINE_THROW(uint32_t)
1257	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1258	{
1259	#ifdef RT_ARCH_AMD64
1260	/* mov mem8, reg8 */
1261	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1262	if (iGpr >= 8)
1263	pbCodeBuf[off++] = X86_OP_REX_R;
1264	pbCodeBuf[off++] = 0x88;
1265	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1266	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1267
1268	#elif RT_ARCH_ARM64
1269	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1270
1271	#else
1272	# error "port me"
1273	#endif
1274	return off;
1275	}
1276
1277
1278	/**
1279	* Emits a load effective address to a GRP of a VCpu field.
1280	*/
1281	DECL_INLINE_THROW(uint32_t)
1282	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
1283	{
1284	#ifdef RT_ARCH_AMD64
1285	/* lea gprdst, [rbx + offDisp] */
1286	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1287	if (iGprDst < 8)
1288	pbCodeBuf[off++] = X86_OP_REX_W;
1289	else
1290	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1291	pbCodeBuf[off++] = 0x8d;
1292	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
1293
1294	#elif defined(RT_ARCH_ARM64)
1295	if (offVCpu < (unsigned)_4K)
1296	{
1297	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1298	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
1299	}
1300	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
1301	{
1302	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1303	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
1304	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
1305	}
1306	else
1307	{
1308	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1309	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
1310	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1311	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
1312	}
1313
1314	#else
1315	# error "port me"
1316	#endif
1317	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1318	return off;
1319	}
1320
1321
1322	/**
1323	* Emits a gprdst = gprsrc load.
1324	*/
1325	DECL_INLINE_THROW(uint32_t)
1326	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1327	{
1328	#ifdef RT_ARCH_AMD64
1329	/* mov gprdst, gprsrc */
1330	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1331	if ((iGprDst \| iGprSrc) >= 8)
1332	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
1333	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
1334	: X86_OP_REX_W \| X86_OP_REX_R;
1335	else
1336	pbCodeBuf[off++] = X86_OP_REX_W;
1337	pbCodeBuf[off++] = 0x8b;
1338	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1339
1340	#elif RT_ARCH_ARM64
1341	/* mov dst, src; alias for: orr dst, xzr, src */
1342	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1343	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
1344
1345	#else
1346	# error "port me"
1347	#endif
1348	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1349	return off;
1350	}
1351
1352
1353	/**
1354	* Emits a gprdst = gprsrc[31:0] load.
1355	* @note Bits 63 thru 32 are cleared.
1356	*/
1357	DECL_INLINE_THROW(uint32_t)
1358	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1359	{
1360	#ifdef RT_ARCH_AMD64
1361	/* mov gprdst, gprsrc */
1362	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1363	if ((iGprDst \| iGprSrc) >= 8)
1364	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1365	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1366	: X86_OP_REX_R;
1367	pbCodeBuf[off++] = 0x8b;
1368	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1369
1370	#elif RT_ARCH_ARM64
1371	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
1372	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1373	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
1374
1375	#else
1376	# error "port me"
1377	#endif
1378	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1379	return off;
1380	}
1381
1382
1383	/**
1384	* Emits a gprdst = gprsrc[15:0] load.
1385	* @note Bits 63 thru 15 are cleared.
1386	*/
1387	DECL_INLINE_THROW(uint32_t)
1388	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1389	{
1390	#ifdef RT_ARCH_AMD64
1391	/* movzx Gv,Ew */
1392	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1393	if ((iGprDst \| iGprSrc) >= 8)
1394	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1395	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1396	: X86_OP_REX_R;
1397	pbCodeBuf[off++] = 0x0f;
1398	pbCodeBuf[off++] = 0xb7;
1399	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1400
1401	#elif RT_ARCH_ARM64
1402	/* and gprdst, gprsrc, #0xffff */
1403	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1404	# if 1
1405	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
1406	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
1407	# else
1408	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
1409	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1410	# endif
1411
1412	#else
1413	# error "port me"
1414	#endif
1415	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1416	return off;
1417	}
1418
1419
1420	/**
1421	* Emits a gprdst = gprsrc[7:0] load.
1422	* @note Bits 63 thru 8 are cleared.
1423	*/
1424	DECL_INLINE_THROW(uint32_t)
1425	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1426	{
1427	#ifdef RT_ARCH_AMD64
1428	/* movzx Gv,Eb */
1429	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1430	if ((iGprDst \| iGprSrc) >= 8)
1431	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1432	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1433	: X86_OP_REX_R;
1434	pbCodeBuf[off++] = 0x0f;
1435	pbCodeBuf[off++] = 0xb6;
1436	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1437
1438	#elif RT_ARCH_ARM64
1439	/* and gprdst, gprsrc, #0xff */
1440	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1441	# if 1
1442	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1443	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
1444	# else
1445	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
1446	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
1447	# endif
1448
1449	#else
1450	# error "port me"
1451	#endif
1452	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1453	return off;
1454	}
1455
1456	#if 0 /** @todo */
1457	/**
1458	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1459	* @note Bits 63 thru 8 are cleared.
1460	*/
1461	DECL_INLINE_THROW(uint32_t)
1462	iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1463	{
1464	#ifdef RT_ARCH_AMD64
1465	/* movzx Gv,Eb */
1466	/** @todo */
1467	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1468	if ((iGprDst \| iGprSrc) >= 8)
1469	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1470	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1471	: X86_OP_REX_R;
1472	pbCodeBuf[off++] = 0x0f;
1473	pbCodeBuf[off++] = 0xb6;
1474	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1475
1476	#elif RT_ARCH_ARM64
1477	/* ubfx gprdst, gprsrc, #8, #8 */
1478	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1479	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
1480	pu32CodeBuf[off++] = /** @todo ubfx */;
1481
1482	#else
1483	# error "port me"
1484	#endif
1485	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1486	return off;
1487	}
1488	#endif
1489
1490	#ifdef RT_ARCH_AMD64
1491	/**
1492	* Common bit of iemNativeEmitLoadGprByBp and friends.
1493	*/
1494	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1495	PIEMRECOMPILERSTATE pReNativeAssert)
1496	{
1497	if (offDisp < 128 && offDisp >= -128)
1498	{
1499	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1500	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1501	}
1502	else
1503	{
1504	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1505	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1506	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1507	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1508	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1509	}
1510	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1511	return off;
1512	}
1513	#endif
1514
1515
1516	#ifdef RT_ARCH_AMD64
1517	/**
1518	* Emits a 64-bit GRP load instruction with an BP relative source address.
1519	*/
1520	DECL_INLINE_THROW(uint32_t)
1521	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1522	{
1523	/* mov gprdst, qword [rbp + offDisp] */
1524	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1525	if (iGprDst < 8)
1526	pbCodeBuf[off++] = X86_OP_REX_W;
1527	else
1528	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1529	pbCodeBuf[off++] = 0x8b;
1530	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1531	}
1532	#endif
1533
1534
1535	#ifdef RT_ARCH_AMD64
1536	/**
1537	* Emits a 32-bit GRP load instruction with an BP relative source address.
1538	*/
1539	DECL_INLINE_THROW(uint32_t)
1540	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1541	{
1542	/* mov gprdst, dword [rbp + offDisp] */
1543	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1544	if (iGprDst >= 8)
1545	pbCodeBuf[off++] = X86_OP_REX_R;
1546	pbCodeBuf[off++] = 0x8b;
1547	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1548	}
1549	#endif
1550
1551
1552	/**
1553	* Emits a load effective address to a GRP with an BP relative source address.
1554	*/
1555	DECL_INLINE_THROW(uint32_t)
1556	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1557	{
1558	#ifdef RT_ARCH_AMD64
1559	/* lea gprdst, [rbp + offDisp] */
1560	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1561	if (iGprDst < 8)
1562	pbCodeBuf[off++] = X86_OP_REX_W;
1563	else
1564	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1565	pbCodeBuf[off++] = 0x8d;
1566	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1567
1568	#elif defined(RT_ARCH_ARM64)
1569	if ((uint32_t)offDisp < (unsigned)_4K)
1570	{
1571	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1572	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);
1573	}
1574	else if ((uint32_t)-offDisp < (unsigned)_4K)
1575	{
1576	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1577	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);
1578	}
1579	else
1580	{
1581	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1582	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1583	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1584	if (offDisp >= 0)
1585	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
1586	else
1587	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
1588	}
1589
1590	#else
1591	# error "port me"
1592	#endif
1593
1594	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1595	return off;
1596	}
1597
1598
1599	/**
1600	* Emits a 64-bit GPR store with an BP relative destination address.
1601	*
1602	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1603	*/
1604	DECL_INLINE_THROW(uint32_t)
1605	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1606	{
1607	#ifdef RT_ARCH_AMD64
1608	/* mov qword [rbp + offDisp], gprdst */
1609	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1610	if (iGprSrc < 8)
1611	pbCodeBuf[off++] = X86_OP_REX_W;
1612	else
1613	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1614	pbCodeBuf[off++] = 0x89;
1615	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1616
1617	#elif defined(RT_ARCH_ARM64)
1618	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1619	{
1620	/* str w/ unsigned imm12 (scaled) */
1621	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1622	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1623	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1624	}
1625	else if (offDisp >= -256 && offDisp <= 256)
1626	{
1627	/* stur w/ signed imm9 (unscaled) */
1628	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1629	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1630	}
1631	else
1632	{
1633	/* Use temporary indexing register. */
1634	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1635	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1636	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1637	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1638	}
1639	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1640	return off;
1641
1642	#else
1643	# error "Port me!"
1644	#endif
1645	}
1646
1647
1648	/**
1649	* Emits a 64-bit immediate store with an BP relative destination address.
1650	*
1651	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1652	*/
1653	DECL_INLINE_THROW(uint32_t)
1654	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1655	{
1656	#ifdef RT_ARCH_AMD64
1657	if ((int64_t)uImm64 == (int32_t)uImm64)
1658	{
1659	/* mov qword [rbp + offDisp], imm32 - sign extended */
1660	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1661	pbCodeBuf[off++] = X86_OP_REX_W;
1662	pbCodeBuf[off++] = 0xc7;
1663	if (offDisp < 128 && offDisp >= -128)
1664	{
1665	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1666	pbCodeBuf[off++] = (uint8_t)offDisp;
1667	}
1668	else
1669	{
1670	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1671	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1672	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1673	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1674	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1675	}
1676	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1677	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1678	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1679	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1680	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1681	return off;
1682	}
1683	#endif
1684
1685	/* Load tmp0, imm64; Store tmp to bp+disp. */
1686	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1687	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1688	}
1689
1690
1691	#ifdef RT_ARCH_AMD64
1692	/**
1693	* Common bit of iemNativeEmitLoadGprByGpr and friends.
1694	*/
1695	DECL_FORCE_INLINE(uint32_t)
1696	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1697	{
1698	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1699	{
1700	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1701	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1702	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1703	}
1704	else if (offDisp == (int8_t)offDisp)
1705	{
1706	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1707	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1708	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1709	pbCodeBuf[off++] = (uint8_t)offDisp;
1710	}
1711	else
1712	{
1713	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1714	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1715	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1716	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1717	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1718	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1719	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1720	}
1721	return off;
1722	}
1723	#elif RT_ARCH_ARM64
1724	/**
1725	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1726	*/
1727	DECL_FORCE_INLINE_THROW(uint32_t)
1728	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1729	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1730	{
1731	/*
1732	* There are a couple of ldr variants that takes an immediate offset, so
1733	* try use those if we can, otherwise we have to use the temporary register
1734	* help with the addressing.
1735	*/
1736	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1737	{
1738	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1739	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1740	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1741	}
1742	else
1743	{
1744	/* The offset is too large, so we must load it into a register and use
1745	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1746	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1747	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, off, (uint64)offDisp);
1748
1749	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1750	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1751
1752	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1753	}
1754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1755	return off;
1756	}
1757	#endif
1758
1759
1760	/**
1761	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1762	*/
1763	DECL_INLINE_THROW(uint32_t)
1764	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1765	{
1766	#ifdef RT_ARCH_AMD64
1767	/* mov reg64, mem64 */
1768	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1769	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1770	pbCodeBuf[off++] = 0x8b;
1771	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1772	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1773
1774	#elif RT_ARCH_ARM64
1775	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1776
1777	#else
1778	# error "port me"
1779	#endif
1780	return off;
1781	}
1782
1783
1784	/**
1785	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1786	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1787	*/
1788	DECL_INLINE_THROW(uint32_t)
1789	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1790	{
1791	#ifdef RT_ARCH_AMD64
1792	/* mov reg32, mem32 */
1793	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1794	if (iGprDst >= 8 \|\| iGprBase >= 8)
1795	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1796	pbCodeBuf[off++] = 0x8b;
1797	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1798	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1799
1800	#elif RT_ARCH_ARM64
1801	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1802
1803	#else
1804	# error "port me"
1805	#endif
1806	return off;
1807	}
1808
1809
1810	/*********************************************************************************************************************************
1811	* Subtraction and Additions *
1812	*********************************************************************************************************************************/
1813
1814
1815	#ifdef RT_ARCH_AMD64
1816	/**
1817	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1818	*/
1819	DECL_INLINE_THROW(uint32_t)
1820	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1821	{
1822	/* sub gprdst, imm8/imm32 */
1823	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1824	if (iGprDst < 8)
1825	pbCodeBuf[off++] = X86_OP_REX_W;
1826	else
1827	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1828	if (iSubtrahend < 128 && iSubtrahend >= -128)
1829	{
1830	pbCodeBuf[off++] = 0x83;
1831	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1832	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1833	}
1834	else
1835	{
1836	pbCodeBuf[off++] = 0x81;
1837	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1838	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1839	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1840	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1841	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1842	}
1843	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1844	return off;
1845	}
1846	#endif
1847
1848
1849	/**
1850	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1851	* @note The AMD64 version sets flags.
1852	*/
1853	DECL_INLINE_THROW(uint32_t)
1854	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1855	{
1856	#if defined(RT_ARCH_AMD64)
1857	/* add Gv,Ev */
1858	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1859	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1860	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1861	pbCodeBuf[off++] = 0x04;
1862	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1863
1864	#elif defined(RT_ARCH_ARM64)
1865	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1866	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1867
1868	#else
1869	# error "Port me"
1870	#endif
1871	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1872	return off;
1873	}
1874
1875
1876	/**
1877	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1878	*/
1879	DECL_INLINE_THROW(uint32_t)
1880	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1881	{
1882	#if defined(RT_ARCH_AMD64)
1883	/* add or inc */
1884	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1885	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1886	if (iImm8 != 1)
1887	{
1888	pbCodeBuf[off++] = 0x83;
1889	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1890	pbCodeBuf[off++] = (uint8_t)iImm8;
1891	}
1892	else
1893	{
1894	pbCodeBuf[off++] = 0xff;
1895	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1896	}
1897
1898	#elif defined(RT_ARCH_ARM64)
1899	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1900	if (iImm8 >= 0)
1901	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1902	else
1903	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1904
1905	#else
1906	# error "Port me"
1907	#endif
1908	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1909	return off;
1910	}
1911
1912
1913	/**
1914	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1915	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1916	*/
1917	DECL_INLINE_THROW(uint32_t)
1918	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1919	{
1920	#if defined(RT_ARCH_AMD64)
1921	/* add or inc */
1922	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1923	if (iGprDst >= 8)
1924	pbCodeBuf[off++] = X86_OP_REX_B;
1925	if (iImm8 != 1)
1926	{
1927	pbCodeBuf[off++] = 0x83;
1928	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1929	pbCodeBuf[off++] = (uint8_t)iImm8;
1930	}
1931	else
1932	{
1933	pbCodeBuf[off++] = 0xff;
1934	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1935	}
1936
1937	#elif defined(RT_ARCH_ARM64)
1938	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1939	if (iImm8 >= 0)
1940	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1941	else
1942	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1943
1944	#else
1945	# error "Port me"
1946	#endif
1947	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1948	return off;
1949	}
1950
1951
1952	/**
1953	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1954	*/
1955	DECL_INLINE_THROW(uint32_t)
1956	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1957	{
1958	#if defined(RT_ARCH_AMD64)
1959	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1960	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1961
1962	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1963	{
1964	/* add grp, imm32 */
1965	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1966	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1967	pbCodeBuf[off++] = 0x81;
1968	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1969	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1970	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1971	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1972	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1973	}
1974	else
1975	{
1976	/* Best to use a temporary register to deal with this in the simplest way: */
1977	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1978
1979	/* add dst, tmpreg */
1980	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1981	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1982	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1983	pbCodeBuf[off++] = 0x03;
1984	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1985
1986	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1987	}
1988
1989	#elif defined(RT_ARCH_ARM64)
1990	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1991	{
1992	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1993	if (iAddend >= 0)
1994	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1995	else
1996	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1997	}
1998	else
1999	{
2000	/* Use temporary register for the immediate. */
2001	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2002
2003	/* add gprdst, gprdst, tmpreg */
2004	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2005	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
2006
2007	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2008	}
2009
2010	#else
2011	# error "Port me"
2012	#endif
2013	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2014	return off;
2015	}
2016
2017
2018	/**
2019	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
2020	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2021	*/
2022	DECL_INLINE_THROW(uint32_t)
2023	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2024	{
2025	#if defined(RT_ARCH_AMD64)
2026	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2027	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
2028
2029	/* add grp, imm32 */
2030	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2031	if (iGprDst >= 8)
2032	pbCodeBuf[off++] = X86_OP_REX_B;
2033	pbCodeBuf[off++] = 0x81;
2034	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2035	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2036	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2037	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2038	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2039
2040	#elif defined(RT_ARCH_ARM64)
2041	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2042	{
2043	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2044	if (iAddend >= 0)
2045	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
2046	else
2047	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
2048	}
2049	else
2050	{
2051	/* Use temporary register for the immediate. */
2052	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
2053
2054	/* add gprdst, gprdst, tmpreg */
2055	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2056	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
2057
2058	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2059	}
2060
2061	#else
2062	# error "Port me"
2063	#endif
2064	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2065	return off;
2066	}
2067
2068
2069
2070	/*********************************************************************************************************************************
2071	* Bit Operations *
2072	*********************************************************************************************************************************/
2073
2074	/**
2075	* Emits code for clearing bits 16 thru 63 in the GPR.
2076	*/
2077	DECL_INLINE_THROW(uint32_t)
2078	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2079	{
2080	#if defined(RT_ARCH_AMD64)
2081	/* movzx Gv,Ew */
2082	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2083	if (iGprDst >= 8)
2084	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2085	pbCodeBuf[off++] = 0x0f;
2086	pbCodeBuf[off++] = 0xb7;
2087	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2088
2089	#elif defined(RT_ARCH_ARM64)
2090	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2091	# if 1
2092	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2093	# else
2094	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2095	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2096	# endif
2097	#else
2098	# error "Port me"
2099	#endif
2100	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2101	return off;
2102	}
2103
2104
2105	/**
2106	* Emits code for AND'ing two 64-bit GPRs.
2107	*
2108	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2109	* and ARM64 hosts.
2110	*/
2111	DECL_INLINE_THROW(uint32_t)
2112	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2113	{
2114	#if defined(RT_ARCH_AMD64)
2115	/* and Gv, Ev */
2116	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2117	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2118	pbCodeBuf[off++] = 0x23;
2119	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2120	RT_NOREF(fSetFlags);
2121
2122	#elif defined(RT_ARCH_ARM64)
2123	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2124	if (!fSetFlags)
2125	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2126	else
2127	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2128
2129	#else
2130	# error "Port me"
2131	#endif
2132	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2133	return off;
2134	}
2135
2136
2137	/**
2138	* Emits code for AND'ing two 32-bit GPRs.
2139	*/
2140	DECL_INLINE_THROW(uint32_t)
2141	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2142	{
2143	#if defined(RT_ARCH_AMD64)
2144	/* and Gv, Ev */
2145	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2146	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2147	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2148	pbCodeBuf[off++] = 0x23;
2149	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2150
2151	#elif defined(RT_ARCH_ARM64)
2152	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2153	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2154
2155	#else
2156	# error "Port me"
2157	#endif
2158	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2159	return off;
2160	}
2161
2162
2163	/**
2164	* Emits code for AND'ing a 64-bit GPRs with a constant.
2165	*
2166	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2167	* and ARM64 hosts.
2168	*/
2169	DECL_INLINE_THROW(uint32_t)
2170	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2171	{
2172	#if defined(RT_ARCH_AMD64)
2173	if ((int64_t)uImm == (int8_t)uImm)
2174	{
2175	/* and Ev, imm8 */
2176	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2177	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2178	pbCodeBuf[off++] = 0x83;
2179	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2180	pbCodeBuf[off++] = (uint8_t)uImm;
2181	}
2182	else if ((int64_t)uImm == (int32_t)uImm)
2183	{
2184	/* and Ev, imm32 */
2185	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2186	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2187	pbCodeBuf[off++] = 0x81;
2188	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2189	pbCodeBuf[off++] = RT_BYTE1(uImm);
2190	pbCodeBuf[off++] = RT_BYTE2(uImm);
2191	pbCodeBuf[off++] = RT_BYTE3(uImm);
2192	pbCodeBuf[off++] = RT_BYTE4(uImm);
2193	}
2194	else
2195	{
2196	/* Use temporary register for the 64-bit immediate. */
2197	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2198	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2199	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2200	}
2201	RT_NOREF(fSetFlags);
2202
2203	#elif defined(RT_ARCH_ARM64)
2204	uint32_t uImmR = 0;
2205	uint32_t uImmNandS = 0;
2206	if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
2207	{
2208	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2209	if (!fSetFlags)
2210	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2211	else
2212	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2213	}
2214	else
2215	{
2216	/* Use temporary register for the 64-bit immediate. */
2217	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2218	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2219	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2220	}
2221
2222	#else
2223	# error "Port me"
2224	#endif
2225	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2226	return off;
2227	}
2228
2229
2230	/**
2231	* Emits code for AND'ing an 32-bit GPRs with a constant.
2232	*/
2233	DECL_INLINE_THROW(uint32_t)
2234	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2235	{
2236	#if defined(RT_ARCH_AMD64)
2237	/* and Ev, imm */
2238	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2239	if (iGprDst >= 8)
2240	pbCodeBuf[off++] = X86_OP_REX_B;
2241	if ((int32_t)uImm == (int8_t)uImm)
2242	{
2243	pbCodeBuf[off++] = 0x83;
2244	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2245	pbCodeBuf[off++] = (uint8_t)uImm;
2246	}
2247	else
2248	{
2249	pbCodeBuf[off++] = 0x81;
2250	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2251	pbCodeBuf[off++] = RT_BYTE1(uImm);
2252	pbCodeBuf[off++] = RT_BYTE2(uImm);
2253	pbCodeBuf[off++] = RT_BYTE3(uImm);
2254	pbCodeBuf[off++] = RT_BYTE4(uImm);
2255	}
2256	RT_NOREF(fSetFlags);
2257
2258	#elif defined(RT_ARCH_ARM64)
2259	uint32_t uImmR = 0;
2260	uint32_t uImmNandS = 0;
2261	if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
2262	{
2263	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2264	if (!fSetFlags)
2265	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2266	else
2267	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2268	}
2269	else
2270	{
2271	/* Use temporary register for the 64-bit immediate. */
2272	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2273	if (!fSetFlags)
2274	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
2275	else
2276	off = iemNativeEmitAndsGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
2277	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2278	}
2279
2280	#else
2281	# error "Port me"
2282	#endif
2283	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2284	return off;
2285	}
2286
2287
2288	/**
2289	* Emits code for XOR'ing two 64-bit GPRs.
2290	*/
2291	DECL_INLINE_THROW(uint32_t)
2292	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2293	{
2294	#if defined(RT_ARCH_AMD64)
2295	/* and Gv, Ev */
2296	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2297	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2298	pbCodeBuf[off++] = 0x33;
2299	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2300
2301	#elif defined(RT_ARCH_ARM64)
2302	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2303	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2304
2305	#else
2306	# error "Port me"
2307	#endif
2308	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2309	return off;
2310	}
2311
2312
2313	/**
2314	* Emits code for XOR'ing two 32-bit GPRs.
2315	*/
2316	DECL_INLINE_THROW(uint32_t)
2317	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2318	{
2319	#if defined(RT_ARCH_AMD64)
2320	/* and Gv, Ev */
2321	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2322	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2323	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2324	pbCodeBuf[off++] = 0x33;
2325	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2326
2327	#elif defined(RT_ARCH_ARM64)
2328	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2329	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2330
2331	#else
2332	# error "Port me"
2333	#endif
2334	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2335	return off;
2336	}
2337
2338
2339	/*********************************************************************************************************************************
2340	* Shifting *
2341	*********************************************************************************************************************************/
2342
2343	/**
2344	* Emits code for shifting a GPR a fixed number of bits to the left.
2345	*/
2346	DECL_INLINE_THROW(uint32_t)
2347	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2348	{
2349	Assert(cShift > 0 && cShift < 64);
2350
2351	#if defined(RT_ARCH_AMD64)
2352	/* shl dst, cShift */
2353	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2354	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2355	if (cShift != 1)
2356	{
2357	pbCodeBuf[off++] = 0xc1;
2358	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2359	pbCodeBuf[off++] = cShift;
2360	}
2361	else
2362	{
2363	pbCodeBuf[off++] = 0xd1;
2364	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2365	}
2366
2367	#elif defined(RT_ARCH_ARM64)
2368	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2369	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2370
2371	#else
2372	# error "Port me"
2373	#endif
2374	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2375	return off;
2376	}
2377
2378
2379	/**
2380	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2381	*/
2382	DECL_INLINE_THROW(uint32_t)
2383	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2384	{
2385	Assert(cShift > 0 && cShift < 32);
2386
2387	#if defined(RT_ARCH_AMD64)
2388	/* shl dst, cShift */
2389	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2390	if (iGprDst >= 8)
2391	pbCodeBuf[off++] = X86_OP_REX_B;
2392	if (cShift != 1)
2393	{
2394	pbCodeBuf[off++] = 0xc1;
2395	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2396	pbCodeBuf[off++] = cShift;
2397	}
2398	else
2399	{
2400	pbCodeBuf[off++] = 0xd1;
2401	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2402	}
2403
2404	#elif defined(RT_ARCH_ARM64)
2405	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2406	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2407
2408	#else
2409	# error "Port me"
2410	#endif
2411	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2412	return off;
2413	}
2414
2415
2416	/**
2417	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2418	*/
2419	DECL_INLINE_THROW(uint32_t)
2420	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2421	{
2422	Assert(cShift > 0 && cShift < 64);
2423
2424	#if defined(RT_ARCH_AMD64)
2425	/* shr dst, cShift */
2426	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2427	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2428	if (cShift != 1)
2429	{
2430	pbCodeBuf[off++] = 0xc1;
2431	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2432	pbCodeBuf[off++] = cShift;
2433	}
2434	else
2435	{
2436	pbCodeBuf[off++] = 0xd1;
2437	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2438	}
2439
2440	#elif defined(RT_ARCH_ARM64)
2441	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2442	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2443
2444	#else
2445	# error "Port me"
2446	#endif
2447	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2448	return off;
2449	}
2450
2451
2452	/**
2453	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2454	* right.
2455	*/
2456	DECL_INLINE_THROW(uint32_t)
2457	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2458	{
2459	Assert(cShift > 0 && cShift < 32);
2460
2461	#if defined(RT_ARCH_AMD64)
2462	/* shr dst, cShift */
2463	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2464	if (iGprDst >= 8)
2465	pbCodeBuf[off++] = X86_OP_REX_B;
2466	if (cShift != 1)
2467	{
2468	pbCodeBuf[off++] = 0xc1;
2469	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2470	pbCodeBuf[off++] = cShift;
2471	}
2472	else
2473	{
2474	pbCodeBuf[off++] = 0xd1;
2475	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2476	}
2477
2478	#elif defined(RT_ARCH_ARM64)
2479	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2480	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2481
2482	#else
2483	# error "Port me"
2484	#endif
2485	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2486	return off;
2487	}
2488
2489
2490
2491	/*********************************************************************************************************************************
2492	* Compare and Testing *
2493	*********************************************************************************************************************************/
2494
2495
2496	#ifdef RT_ARCH_ARM64
2497	/**
2498	* Emits an ARM64 compare instruction.
2499	*/
2500	DECL_INLINE_THROW(uint32_t)
2501	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2502	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2503	{
2504	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2505	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2506	f64Bit, true /fSetFlags/, cShift, enmShift);
2507	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2508	return off;
2509	}
2510	#endif
2511
2512
2513	/**
2514	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2515	* with conditional instruction.
2516	*/
2517	DECL_INLINE_THROW(uint32_t)
2518	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2519	{
2520	#ifdef RT_ARCH_AMD64
2521	/* cmp Gv, Ev */
2522	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2523	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2524	pbCodeBuf[off++] = 0x3b;
2525	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2526	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2527
2528	#elif defined(RT_ARCH_ARM64)
2529	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2530
2531	#else
2532	# error "Port me!"
2533	#endif
2534	return off;
2535	}
2536
2537
2538	/**
2539	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2540	* with conditional instruction.
2541	*/
2542	DECL_INLINE_THROW(uint32_t)
2543	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2544	{
2545	#ifdef RT_ARCH_AMD64
2546	/* cmp Gv, Ev */
2547	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2548	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2549	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2550	pbCodeBuf[off++] = 0x3b;
2551	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2552	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2553
2554	#elif defined(RT_ARCH_ARM64)
2555	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2556
2557	#else
2558	# error "Port me!"
2559	#endif
2560	return off;
2561	}
2562
2563
2564	/**
2565	* Emits a compare of a 64-bit GPR with a constant value, settings status
2566	* flags/whatever for use with conditional instruction.
2567	*/
2568	DECL_INLINE_THROW(uint32_t)
2569	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2570	{
2571	#ifdef RT_ARCH_AMD64
2572	if (uImm <= UINT32_C(0xff))
2573	{
2574	/* cmp Ev, Ib */
2575	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2576	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2577	pbCodeBuf[off++] = 0x83;
2578	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2579	pbCodeBuf[off++] = (uint8_t)uImm;
2580	}
2581	else if ((int64_t)uImm == (int32_t)uImm)
2582	{
2583	/* cmp Ev, imm */
2584	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2585	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2586	pbCodeBuf[off++] = 0x81;
2587	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2588	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2589	pbCodeBuf[off++] = RT_BYTE1(uImm);
2590	pbCodeBuf[off++] = RT_BYTE2(uImm);
2591	pbCodeBuf[off++] = RT_BYTE3(uImm);
2592	pbCodeBuf[off++] = RT_BYTE4(uImm);
2593	}
2594	else
2595	{
2596	/* Use temporary register for the immediate. */
2597	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2598	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2599	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2600	}
2601
2602	#elif defined(RT_ARCH_ARM64)
2603	/** @todo guess there are clevere things we can do here... */
2604	if (uImm < _4K)
2605	{
2606	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2607	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2608	true /64Bit/, true /fSetFlags/);
2609	}
2610	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2611	{
2612	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2613	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2614	true /64Bit/, true /fSetFlags/, true /fShift12/);
2615	}
2616	else
2617	{
2618	/* Use temporary register for the immediate. */
2619	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2620	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2621	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2622	}
2623
2624	#else
2625	# error "Port me!"
2626	#endif
2627
2628	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2629	return off;
2630	}
2631
2632
2633	/**
2634	* Emits a compare of a 32-bit GPR with a constant value, settings status
2635	* flags/whatever for use with conditional instruction.
2636	*/
2637	DECL_INLINE_THROW(uint32_t)
2638	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2639	{
2640	#ifdef RT_ARCH_AMD64
2641	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2642	if (iGprLeft >= 8)
2643	pbCodeBuf[off++] = X86_OP_REX_B;
2644	if (uImm <= UINT32_C(0xff))
2645	{
2646	/* cmp Ev, Ib */
2647	pbCodeBuf[off++] = 0x83;
2648	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2649	pbCodeBuf[off++] = (uint8_t)uImm;
2650	}
2651	else
2652	{
2653	/* cmp Ev, imm */
2654	pbCodeBuf[off++] = 0x81;
2655	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2656	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2657	pbCodeBuf[off++] = RT_BYTE1(uImm);
2658	pbCodeBuf[off++] = RT_BYTE2(uImm);
2659	pbCodeBuf[off++] = RT_BYTE3(uImm);
2660	pbCodeBuf[off++] = RT_BYTE4(uImm);
2661	}
2662
2663	#elif defined(RT_ARCH_ARM64)
2664	/** @todo guess there are clevere things we can do here... */
2665	if (uImm < _4K)
2666	{
2667	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2668	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2669	false /64Bit/, true /fSetFlags/);
2670	}
2671	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2672	{
2673	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2674	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2675	false /64Bit/, true /fSetFlags/, true /fShift12/);
2676	}
2677	else
2678	{
2679	/* Use temporary register for the immediate. */
2680	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2681	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2682	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2683	}
2684
2685	#else
2686	# error "Port me!"
2687	#endif
2688
2689	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2690	return off;
2691	}
2692
2693
2694
2695	/*********************************************************************************************************************************
2696	* Branching *
2697	*********************************************************************************************************************************/
2698
2699	/**
2700	* Emits a JMP rel32 / B imm19 to the given label.
2701	*/
2702	DECL_INLINE_THROW(uint32_t)
2703	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2704	{
2705	Assert(idxLabel < pReNative->cLabels);
2706
2707	#ifdef RT_ARCH_AMD64
2708	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2709	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2710	{
2711	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2712	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2713	{
2714	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2715	pbCodeBuf[off++] = (uint8_t)offRel;
2716	}
2717	else
2718	{
2719	offRel -= 3;
2720	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2721	pbCodeBuf[off++] = RT_BYTE1(offRel);
2722	pbCodeBuf[off++] = RT_BYTE2(offRel);
2723	pbCodeBuf[off++] = RT_BYTE3(offRel);
2724	pbCodeBuf[off++] = RT_BYTE4(offRel);
2725	}
2726	}
2727	else
2728	{
2729	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2730	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2731	pbCodeBuf[off++] = 0xfe;
2732	pbCodeBuf[off++] = 0xff;
2733	pbCodeBuf[off++] = 0xff;
2734	pbCodeBuf[off++] = 0xff;
2735	}
2736	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2737
2738	#elif defined(RT_ARCH_ARM64)
2739	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2740	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2741	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
2742	else
2743	{
2744	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2745	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2746	}
2747
2748	#else
2749	# error "Port me!"
2750	#endif
2751	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2752	return off;
2753	}
2754
2755
2756	/**
2757	* Emits a JMP rel32 / B imm19 to a new undefined label.
2758	*/
2759	DECL_INLINE_THROW(uint32_t)
2760	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2761	{
2762	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2763	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2764	}
2765
2766	/** Condition type. */
2767	#ifdef RT_ARCH_AMD64
2768	typedef enum IEMNATIVEINSTRCOND : uint8_t
2769	{
2770	kIemNativeInstrCond_o = 0,
2771	kIemNativeInstrCond_no,
2772	kIemNativeInstrCond_c,
2773	kIemNativeInstrCond_nc,
2774	kIemNativeInstrCond_e,
2775	kIemNativeInstrCond_ne,
2776	kIemNativeInstrCond_be,
2777	kIemNativeInstrCond_nbe,
2778	kIemNativeInstrCond_s,
2779	kIemNativeInstrCond_ns,
2780	kIemNativeInstrCond_p,
2781	kIemNativeInstrCond_np,
2782	kIemNativeInstrCond_l,
2783	kIemNativeInstrCond_nl,
2784	kIemNativeInstrCond_le,
2785	kIemNativeInstrCond_nle
2786	} IEMNATIVEINSTRCOND;
2787	#elif defined(RT_ARCH_ARM64)
2788	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2789	#else
2790	# error "Port me!"
2791	#endif
2792
2793
2794	/**
2795	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2796	*/
2797	DECL_INLINE_THROW(uint32_t)
2798	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2799	{
2800	Assert(idxLabel < pReNative->cLabels);
2801
2802	#ifdef RT_ARCH_AMD64
2803	/* jcc rel32 */
2804	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2805	pbCodeBuf[off++] = 0x0f;
2806	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2807	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2808	pbCodeBuf[off++] = 0x00;
2809	pbCodeBuf[off++] = 0x00;
2810	pbCodeBuf[off++] = 0x00;
2811	pbCodeBuf[off++] = 0x00;
2812
2813	#elif defined(RT_ARCH_ARM64)
2814	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2815	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2816	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2817
2818	#else
2819	# error "Port me!"
2820	#endif
2821	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2822	return off;
2823	}
2824
2825
2826	/**
2827	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2828	*/
2829	DECL_INLINE_THROW(uint32_t)
2830	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2831	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2832	{
2833	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2834	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2835	}
2836
2837
2838	/**
2839	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2840	*/
2841	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2842	{
2843	#ifdef RT_ARCH_AMD64
2844	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2845	#elif defined(RT_ARCH_ARM64)
2846	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2847	#else
2848	# error "Port me!"
2849	#endif
2850	}
2851
2852	/**
2853	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2854	*/
2855	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2856	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2857	{
2858	#ifdef RT_ARCH_AMD64
2859	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2860	#elif defined(RT_ARCH_ARM64)
2861	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2862	#else
2863	# error "Port me!"
2864	#endif
2865	}
2866
2867
2868	/**
2869	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2870	*/
2871	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2872	{
2873	#ifdef RT_ARCH_AMD64
2874	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2875	#elif defined(RT_ARCH_ARM64)
2876	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2877	#else
2878	# error "Port me!"
2879	#endif
2880	}
2881
2882	/**
2883	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2884	*/
2885	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2886	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2887	{
2888	#ifdef RT_ARCH_AMD64
2889	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2890	#elif defined(RT_ARCH_ARM64)
2891	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2892	#else
2893	# error "Port me!"
2894	#endif
2895	}
2896
2897
2898	/**
2899	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2900	*/
2901	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2902	{
2903	#ifdef RT_ARCH_AMD64
2904	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2905	#elif defined(RT_ARCH_ARM64)
2906	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2907	#else
2908	# error "Port me!"
2909	#endif
2910	}
2911
2912	/**
2913	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2914	*/
2915	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2916	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2917	{
2918	#ifdef RT_ARCH_AMD64
2919	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2920	#elif defined(RT_ARCH_ARM64)
2921	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2922	#else
2923	# error "Port me!"
2924	#endif
2925	}
2926
2927
2928	/**
2929	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2930	*/
2931	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2932	{
2933	#ifdef RT_ARCH_AMD64
2934	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2935	#elif defined(RT_ARCH_ARM64)
2936	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2937	#else
2938	# error "Port me!"
2939	#endif
2940	}
2941
2942	/**
2943	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2944	*/
2945	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2946	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2947	{
2948	#ifdef RT_ARCH_AMD64
2949	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2950	#elif defined(RT_ARCH_ARM64)
2951	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2952	#else
2953	# error "Port me!"
2954	#endif
2955	}
2956
2957
2958	/**
2959	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2960	* How @a offJmp is applied is are target specific.
2961	*/
2962	DECL_INLINE_THROW(uint32_t)
2963	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2964	{
2965	#ifdef RT_ARCH_AMD64
2966	/* jcc rel32 */
2967	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2968	if (offTarget < 128 && offTarget >= -128)
2969	{
2970	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2971	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2972	}
2973	else
2974	{
2975	pbCodeBuf[off++] = 0x0f;
2976	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2977	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2978	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2979	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2980	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2981	}
2982
2983	#elif defined(RT_ARCH_ARM64)
2984	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2985	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2986
2987	#else
2988	# error "Port me!"
2989	#endif
2990	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2991	return off;
2992	}
2993
2994
2995	/**
2996	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2997	* How @a offJmp is applied is are target specific.
2998	*/
2999	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3000	{
3001	#ifdef RT_ARCH_AMD64
3002	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
3003	#elif defined(RT_ARCH_ARM64)
3004	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
3005	#else
3006	# error "Port me!"
3007	#endif
3008	}
3009
3010
3011	/**
3012	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
3013	* How @a offJmp is applied is are target specific.
3014	*/
3015	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3016	{
3017	#ifdef RT_ARCH_AMD64
3018	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
3019	#elif defined(RT_ARCH_ARM64)
3020	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
3021	#else
3022	# error "Port me!"
3023	#endif
3024	}
3025
3026
3027	/**
3028	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
3029	* How @a offJmp is applied is are target specific.
3030	*/
3031	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3032	{
3033	#ifdef RT_ARCH_AMD64
3034	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
3035	#elif defined(RT_ARCH_ARM64)
3036	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
3037	#else
3038	# error "Port me!"
3039	#endif
3040	}
3041
3042
3043	/**
3044	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
3045	* How @a offJmp is applied is are target specific.
3046	*/
3047	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3048	{
3049	#ifdef RT_ARCH_AMD64
3050	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3051	#elif defined(RT_ARCH_ARM64)
3052	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3053	#else
3054	# error "Port me!"
3055	#endif
3056	}
3057
3058
3059	/**
3060	* Fixes up a conditional jump to a fixed label.
3061	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
3062	*/
3063	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3064	{
3065	# if defined(RT_ARCH_AMD64)
3066	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3067	if (pbCodeBuf[offFixup] != 0x0f)
3068	{
3069	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
3070	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3071	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
3072	}
3073	else
3074	{
3075	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
3076	uint32_t const offRel32 = offTarget - (offFixup + 6);
3077	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
3078	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
3079	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
3080	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
3081	}
3082
3083	# elif defined(RT_ARCH_ARM64)
3084	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3085	Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
3086	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
3087	\| (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
3088
3089	# endif
3090	}
3091
3092
3093	/**
3094	* Internal helper, don't call directly.
3095	*/
3096	DECL_INLINE_THROW(uint32_t)
3097	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3098	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3099	{
3100	Assert(iBitNo < 64);
3101	#ifdef RT_ARCH_AMD64
3102	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3103	if (iBitNo < 8)
3104	{
3105	/* test Eb, imm8 */
3106	if (iGprSrc >= 4)
3107	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3108	pbCodeBuf[off++] = 0xf6;
3109	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3110	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3111	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3112	}
3113	else
3114	{
3115	/* bt Ev, imm8 */
3116	if (iBitNo >= 32)
3117	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3118	else if (iGprSrc >= 8)
3119	pbCodeBuf[off++] = X86_OP_REX_B;
3120	pbCodeBuf[off++] = 0x0f;
3121	pbCodeBuf[off++] = 0xba;
3122	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3123	pbCodeBuf[off++] = iBitNo;
3124	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3125	}
3126
3127	#elif defined(RT_ARCH_ARM64)
3128	/* Use the TBNZ instruction here. */
3129	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3130	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3131	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3132
3133	#else
3134	# error "Port me!"
3135	#endif
3136	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3137	return off;
3138	}
3139
3140
3141	/**
3142	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3143	* @a iGprSrc.
3144	*
3145	* @note On ARM64 the range is only +/-8191 instructions.
3146	*/
3147	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3148	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3149	{
3150	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
3151	}
3152
3153
3154	/**
3155	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3156	* _set_ in @a iGprSrc.
3157	*
3158	* @note On ARM64 the range is only +/-8191 instructions.
3159	*/
3160	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3161	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3162	{
3163	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
3164	}
3165
3166
3167	/**
3168	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3169	* flags accordingly.
3170	*/
3171	DECL_INLINE_THROW(uint32_t)
3172	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3173	{
3174	Assert(fBits != 0);
3175	#ifdef RT_ARCH_AMD64
3176
3177	if (fBits >= UINT32_MAX)
3178	{
3179	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3180
3181	/* test Ev,Gv */
3182	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3183	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3184	pbCodeBuf[off++] = 0x85;
3185	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3186
3187	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3188	}
3189	else if (fBits <= UINT32_MAX)
3190	{
3191	/* test Eb, imm8 or test Ev, imm32 */
3192	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3193	if (fBits <= UINT8_MAX)
3194	{
3195	if (iGprSrc >= 4)
3196	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3197	pbCodeBuf[off++] = 0xf6;
3198	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3199	pbCodeBuf[off++] = (uint8_t)fBits;
3200	}
3201	else
3202	{
3203	if (iGprSrc >= 8)
3204	pbCodeBuf[off++] = X86_OP_REX_B;
3205	pbCodeBuf[off++] = 0xf7;
3206	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3207	pbCodeBuf[off++] = RT_BYTE1(fBits);
3208	pbCodeBuf[off++] = RT_BYTE2(fBits);
3209	pbCodeBuf[off++] = RT_BYTE3(fBits);
3210	pbCodeBuf[off++] = RT_BYTE4(fBits);
3211	}
3212	}
3213	/** @todo implement me. */
3214	else
3215	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3216
3217	#elif defined(RT_ARCH_ARM64)
3218
3219	if (false)
3220	{
3221	/** @todo figure out how to work the immr / N:imms constants. */
3222	}
3223	else
3224	{
3225	/* ands Zr, iGprSrc, iTmpReg */
3226	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3227	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3228	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3229	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3230	}
3231
3232	#else
3233	# error "Port me!"
3234	#endif
3235	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3236	return off;
3237	}
3238
3239
3240	/**
3241	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
3242	* are set in @a iGprSrc.
3243	*/
3244	DECL_INLINE_THROW(uint32_t)
3245	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3246	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3247	{
3248	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3249
3250	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3251	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3252
3253	return off;
3254	}
3255
3256
3257	/**
3258	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
3259	* are set in @a iGprSrc.
3260	*/
3261	DECL_INLINE_THROW(uint32_t)
3262	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3263	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3264	{
3265	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3266
3267	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3268	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3269
3270	return off;
3271	}
3272
3273
3274	/**
3275	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
3276	*
3277	* The operand size is given by @a f64Bit.
3278	*/
3279	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3280	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3281	{
3282	Assert(idxLabel < pReNative->cLabels);
3283
3284	#ifdef RT_ARCH_AMD64
3285	/* test reg32,reg32 / test reg64,reg64 */
3286	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3287	if (f64Bit)
3288	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
3289	else if (iGprSrc >= 8)
3290	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
3291	pbCodeBuf[off++] = 0x85;
3292	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3293	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3294
3295	/* jz idxLabel */
3296	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3297
3298	#elif defined(RT_ARCH_ARM64)
3299	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3300	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3301	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, f64Bit);
3302	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3303
3304	#else
3305	# error "Port me!"
3306	#endif
3307	return off;
3308	}
3309
3310
3311	/**
3312	* Emits code that jumps to a new label if @a iGprSrc is zero.
3313	*
3314	* The operand size is given by @a f64Bit.
3315	*/
3316	DECL_INLINE_THROW(uint32_t)
3317	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3318	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3319	{
3320	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3321	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3322	}
3323
3324
3325	/**
3326	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3327	* differs.
3328	*/
3329	DECL_INLINE_THROW(uint32_t)
3330	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3331	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3332	{
3333	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3334	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3335	return off;
3336	}
3337
3338
3339	/**
3340	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3341	*/
3342	DECL_INLINE_THROW(uint32_t)
3343	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3344	uint8_t iGprLeft, uint8_t iGprRight,
3345	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3346	{
3347	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3348	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3349	}
3350
3351
3352	/**
3353	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3354	*/
3355	DECL_INLINE_THROW(uint32_t)
3356	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3357	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3358	{
3359	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3360	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3361	return off;
3362	}
3363
3364
3365	/**
3366	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3367	*/
3368	DECL_INLINE_THROW(uint32_t)
3369	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3370	uint8_t iGprSrc, uint64_t uImm,
3371	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3372	{
3373	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3374	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3375	}
3376
3377
3378	/**
3379	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3380	* @a uImm.
3381	*/
3382	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3383	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3384	{
3385	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3386	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3387	return off;
3388	}
3389
3390
3391	/**
3392	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3393	* @a uImm.
3394	*/
3395	DECL_INLINE_THROW(uint32_t)
3396	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3397	uint8_t iGprSrc, uint32_t uImm,
3398	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3399	{
3400	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3401	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3402	}
3403
3404
3405
3406	/**
3407	* Emits a call to a 64-bit address.
3408	*/
3409	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3410	{
3411	#ifdef RT_ARCH_AMD64
3412	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3413
3414	/* call rax */
3415	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3416	pbCodeBuf[off++] = 0xff;
3417	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3418
3419	#elif defined(RT_ARCH_ARM64)
3420	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3421
3422	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3423	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3424
3425	#else
3426	# error "port me"
3427	#endif
3428	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3429	return off;
3430	}
3431
3432
3433	/** @} */
3434
3435	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3436

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101844

Download in other formats: