IEMN8veRecompilerEmit.h@ 102801

Last change on this file since 102801 was 102800, checked in by vboxsync, 16 months ago
VMM/IEM: Stats, go straight for the safe fallback functions in the memory access helpers. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 210.5 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102800 2024-01-09 22:19:17Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	pCodeBuf[off++] = 0xcc;
88	RT_NOREF(uInfo); /** @todo use multibyte nop for info? */
89
90	#elif defined(RT_ARCH_ARM64)
91	pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
92
93	#else
94	# error "error"
95	#endif
96	return off;
97	}
98
99
100	/**
101	* Emit a breakpoint instruction.
102	*/
103	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
104	{
105	#ifdef RT_ARCH_AMD64
106	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
107	#elif defined(RT_ARCH_ARM64)
108	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
109	#else
110	# error "error"
111	#endif
112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
113	return off;
114	}
115
116
117	/*********************************************************************************************************************************
118	* Loads, Stores and Related Stuff. *
119	*********************************************************************************************************************************/
120
121	#ifdef RT_ARCH_AMD64
122	/**
123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
124	*/
125	DECL_FORCE_INLINE(uint32_t)
126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
127	{
128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
129	{
130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
133	}
134	else if (offDisp == (int8_t)offDisp)
135	{
136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
139	pbCodeBuf[off++] = (uint8_t)offDisp;
140	}
141	else
142	{
143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
150	}
151	return off;
152	}
153	#endif /* RT_ARCH_AMD64 */
154
155	/**
156	* Emits setting a GPR to zero.
157	*/
158	DECL_INLINE_THROW(uint32_t)
159	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
160	{
161	#ifdef RT_ARCH_AMD64
162	/* xor gpr32, gpr32 */
163	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
164	if (iGpr >= 8)
165	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
166	pbCodeBuf[off++] = 0x33;
167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
168
169	#elif defined(RT_ARCH_ARM64)
170	/* mov gpr, #0x0 */
171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
172	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
173
174	#else
175	# error "port me"
176	#endif
177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
178	return off;
179	}
180
181
182	/**
183	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
184	* buffer space.
185	*
186	* Max buffer consumption:
187	* - AMD64: 10 instruction bytes.
188	* - ARM64: 4 instruction words (16 bytes).
189	*/
190	DECL_FORCE_INLINE(uint32_t)
191	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
192	{
193	#ifdef RT_ARCH_AMD64
194	if (uImm64 == 0)
195	{
196	/* xor gpr, gpr */
197	if (iGpr >= 8)
198	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
199	pCodeBuf[off++] = 0x33;
200	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
201	}
202	else if (uImm64 <= UINT32_MAX)
203	{
204	/* mov gpr, imm32 */
205	if (iGpr >= 8)
206	pCodeBuf[off++] = X86_OP_REX_B;
207	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else if (uImm64 == (uint64_t)(int32_t)uImm64)
214	{
215	/* mov gpr, sx(imm32) */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xc7;
221	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
222	pCodeBuf[off++] = RT_BYTE1(uImm64);
223	pCodeBuf[off++] = RT_BYTE2(uImm64);
224	pCodeBuf[off++] = RT_BYTE3(uImm64);
225	pCodeBuf[off++] = RT_BYTE4(uImm64);
226	}
227	else
228	{
229	/* mov gpr, imm64 */
230	if (iGpr < 8)
231	pCodeBuf[off++] = X86_OP_REX_W;
232	else
233	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
234	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
235	pCodeBuf[off++] = RT_BYTE1(uImm64);
236	pCodeBuf[off++] = RT_BYTE2(uImm64);
237	pCodeBuf[off++] = RT_BYTE3(uImm64);
238	pCodeBuf[off++] = RT_BYTE4(uImm64);
239	pCodeBuf[off++] = RT_BYTE5(uImm64);
240	pCodeBuf[off++] = RT_BYTE6(uImm64);
241	pCodeBuf[off++] = RT_BYTE7(uImm64);
242	pCodeBuf[off++] = RT_BYTE8(uImm64);
243	}
244
245	#elif defined(RT_ARCH_ARM64)
246	/*
247	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
248	* supply remaining bits using 'movk grp, imm16, lsl #x'.
249	*
250	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
251	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
252	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
253	* after the first non-zero immediate component so we switch to movk for
254	* the remainder.
255	*/
256	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
257	+ !((uImm64 >> 16) & UINT16_MAX)
258	+ !((uImm64 >> 32) & UINT16_MAX)
259	+ !((uImm64 >> 48) & UINT16_MAX);
260	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
261	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
262	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
263	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
264	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
265	if (cFfffHalfWords <= cZeroHalfWords)
266	{
267	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
268
269	/* movz gpr, imm16 */
270	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
271	if (uImmPart \|\| cZeroHalfWords == 4)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #16 */
277	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
278	if (uImmPart)
279	{
280	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
281	fMovBase \|= RT_BIT_32(29);
282	}
283	/* mov[z/k] gpr, imm16, lsl #32 */
284	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
285	if (uImmPart)
286	{
287	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
288	fMovBase \|= RT_BIT_32(29);
289	}
290	/* mov[z/k] gpr, imm16, lsl #48 */
291	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
292	if (uImmPart)
293	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
294	}
295	else
296	{
297	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
298
299	/* find the first half-word that isn't UINT16_MAX. */
300	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
301	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
302	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
303
304	/* movn gpr, imm16, lsl #iHwNotFfff16 /
305	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
306	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
307	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
308	/* movk gpr, imm16 */
309	if (iHwNotFfff != 0)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #16 */
316	if (iHwNotFfff != 1)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
321	}
322	/* movk gpr, imm16, lsl #32 */
323	if (iHwNotFfff != 2)
324	{
325	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
326	if (uImmPart != UINT32_C(0xffff))
327	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
328	}
329	/* movk gpr, imm16, lsl #48 */
330	if (iHwNotFfff != 3)
331	{
332	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
333	if (uImmPart != UINT32_C(0xffff))
334	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
335	}
336	}
337
338	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
339	* clang 12.x does that, only to use the 'x' version for the
340	* addressing in the following ldr). */
341
342	#else
343	# error "port me"
344	#endif
345	return off;
346	}
347
348
349	/**
350	* Emits loading a constant into a 64-bit GPR
351	*/
352	DECL_INLINE_THROW(uint32_t)
353	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
354	{
355	#ifdef RT_ARCH_AMD64
356	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
357	#elif defined(RT_ARCH_ARM64)
358	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
359	#else
360	# error "port me"
361	#endif
362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
363	return off;
364	}
365
366
367	/**
368	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
369	* buffer space.
370	*
371	* Max buffer consumption:
372	* - AMD64: 6 instruction bytes.
373	* - ARM64: 2 instruction words (8 bytes).
374	*
375	* @note The top 32 bits will be cleared.
376	*/
377	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
378	{
379	#ifdef RT_ARCH_AMD64
380	if (uImm32 == 0)
381	{
382	/* xor gpr, gpr */
383	if (iGpr >= 8)
384	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
385	pCodeBuf[off++] = 0x33;
386	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
387	}
388	else
389	{
390	/* mov gpr, imm32 */
391	if (iGpr >= 8)
392	pCodeBuf[off++] = X86_OP_REX_B;
393	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
394	pCodeBuf[off++] = RT_BYTE1(uImm32);
395	pCodeBuf[off++] = RT_BYTE2(uImm32);
396	pCodeBuf[off++] = RT_BYTE3(uImm32);
397	pCodeBuf[off++] = RT_BYTE4(uImm32);
398	}
399
400	#elif defined(RT_ARCH_ARM64)
401	if ((uImm32 >> 16) == 0)
402	/* movz gpr, imm16 */
403	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
404	else if ((uImm32 & UINT32_C(0xffff)) == 0)
405	/* movz gpr, imm16, lsl #16 */
406	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
407	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
408	/* movn gpr, imm16, lsl #16 */
409	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
410	else if ((uImm32 >> 16) == UINT32_C(0xffff))
411	/* movn gpr, imm16 */
412	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
413	else
414	{
415	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
416	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
417	}
418
419	#else
420	# error "port me"
421	#endif
422	return off;
423	}
424
425
426	/**
427	* Emits loading a constant into a 32-bit GPR.
428	* @note The top 32 bits will be cleared.
429	*/
430	DECL_INLINE_THROW(uint32_t)
431	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
432	{
433	#ifdef RT_ARCH_AMD64
434	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
435	#elif defined(RT_ARCH_ARM64)
436	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
437	#else
438	# error "port me"
439	#endif
440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
441	return off;
442	}
443
444
445	/**
446	* Emits loading a constant into a 8-bit GPR
447	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
448	* only the ARM64 version does that.
449	*/
450	DECL_INLINE_THROW(uint32_t)
451	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
452	{
453	#ifdef RT_ARCH_AMD64
454	/* mov gpr, imm8 */
455	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
456	if (iGpr >= 8)
457	pbCodeBuf[off++] = X86_OP_REX_B;
458	else if (iGpr >= 4)
459	pbCodeBuf[off++] = X86_OP_REX;
460	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
461	pbCodeBuf[off++] = RT_BYTE1(uImm8);
462
463	#elif defined(RT_ARCH_ARM64)
464	/* movz gpr, imm16, lsl #0 */
465	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
466	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
467
468	#else
469	# error "port me"
470	#endif
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472	return off;
473	}
474
475
476	#ifdef RT_ARCH_AMD64
477	/**
478	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
479	*/
480	DECL_FORCE_INLINE(uint32_t)
481	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
482	{
483	if (offVCpu < 128)
484	{
485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
486	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
487	}
488	else
489	{
490	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
491	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
492	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
493	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
494	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
495	}
496	return off;
497	}
498
499	#elif defined(RT_ARCH_ARM64)
500
501	/**
502	* Common bit of iemNativeEmitLoadGprFromVCpuU64Ex and friends.
503	*
504	* @note Loads can use @a iGprReg for large offsets, stores requires a temporary
505	* registers (@a iGprTmp).
506	* @note DON'T try this with prefetch.
507	*/
508	DECL_FORCE_INLINE_THROW(uint32_t)
509	iemNativeEmitGprByVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu,
510	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
511	{
512	/*
513	* There are a couple of ldr variants that takes an immediate offset, so
514	* try use those if we can, otherwise we have to use the temporary register
515	* help with the addressing.
516	*/
517	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
518	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
519	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
520	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
521	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
522	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
523	else if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) \|\| iGprTmp != UINT8_MAX)
524	{
525	/* The offset is too large, so we must load it into a register and use
526	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
527	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
528	if (iGprTmp == UINT8_MAX)
529	iGprTmp = iGprReg;
530	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, offVCpu);
531	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, iGprTmp);
532	}
533	else
534	# ifdef IEM_WITH_THROW_CATCH
535	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
536	# else
537	AssertReleaseFailedStmt(off = UINT32_MAX);
538	# endif
539
540	return off;
541	}
542
543	/**
544	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
545	*/
546	DECL_FORCE_INLINE_THROW(uint32_t)
547	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
548	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
549	{
550	/*
551	* There are a couple of ldr variants that takes an immediate offset, so
552	* try use those if we can, otherwise we have to use the temporary register
553	* help with the addressing.
554	*/
555	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
556	{
557	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
558	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
559	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
560	}
561	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
562	{
563	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
564	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
565	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
566	}
567	else
568	{
569	/* The offset is too large, so we must load it into a register and use
570	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
571	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
572	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
574	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
575	IEMNATIVE_REG_FIXED_TMP0);
576	}
577	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
578	return off;
579	}
580
581	#endif /* RT_ARCH_ARM64 */
582
583
584	/**
585	* Emits a 64-bit GPR load of a VCpu value.
586	*/
587	DECL_FORCE_INLINE_THROW(uint32_t)
588	iemNativeEmitLoadGprFromVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
589	{
590	#ifdef RT_ARCH_AMD64
591	/* mov reg64, mem64 */
592	if (iGpr < 8)
593	pCodeBuf[off++] = X86_OP_REX_W;
594	else
595	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
596	pCodeBuf[off++] = 0x8b;
597	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
598
599	#elif defined(RT_ARCH_ARM64)
600	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
601
602	#else
603	# error "port me"
604	#endif
605	return off;
606	}
607
608
609	/**
610	* Emits a 64-bit GPR load of a VCpu value.
611	*/
612	DECL_INLINE_THROW(uint32_t)
613	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
614	{
615	#ifdef RT_ARCH_AMD64
616	off = iemNativeEmitLoadGprFromVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
617	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
618
619	#elif defined(RT_ARCH_ARM64)
620	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
621
622	#else
623	# error "port me"
624	#endif
625	return off;
626	}
627
628
629	/**
630	* Emits a 32-bit GPR load of a VCpu value.
631	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
632	*/
633	DECL_INLINE_THROW(uint32_t)
634	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
635	{
636	#ifdef RT_ARCH_AMD64
637	/* mov reg32, mem32 */
638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
639	if (iGpr >= 8)
640	pbCodeBuf[off++] = X86_OP_REX_R;
641	pbCodeBuf[off++] = 0x8b;
642	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
644
645	#elif defined(RT_ARCH_ARM64)
646	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
647
648	#else
649	# error "port me"
650	#endif
651	return off;
652	}
653
654
655	/**
656	* Emits a 16-bit GPR load of a VCpu value.
657	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
658	*/
659	DECL_INLINE_THROW(uint32_t)
660	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
661	{
662	#ifdef RT_ARCH_AMD64
663	/* movzx reg32, mem16 */
664	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
665	if (iGpr >= 8)
666	pbCodeBuf[off++] = X86_OP_REX_R;
667	pbCodeBuf[off++] = 0x0f;
668	pbCodeBuf[off++] = 0xb7;
669	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
670	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
671
672	#elif defined(RT_ARCH_ARM64)
673	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
674
675	#else
676	# error "port me"
677	#endif
678	return off;
679	}
680
681
682	/**
683	* Emits a 8-bit GPR load of a VCpu value.
684	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
685	*/
686	DECL_INLINE_THROW(uint32_t)
687	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
688	{
689	#ifdef RT_ARCH_AMD64
690	/* movzx reg32, mem8 */
691	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
692	if (iGpr >= 8)
693	pbCodeBuf[off++] = X86_OP_REX_R;
694	pbCodeBuf[off++] = 0x0f;
695	pbCodeBuf[off++] = 0xb6;
696	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
697	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
698
699	#elif defined(RT_ARCH_ARM64)
700	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
701
702	#else
703	# error "port me"
704	#endif
705	return off;
706	}
707
708
709	/**
710	* Emits a store of a GPR value to a 64-bit VCpu field.
711	*/
712	DECL_INLINE_THROW(uint32_t)
713	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
714	{
715	#ifdef RT_ARCH_AMD64
716	/* mov mem64, reg64 */
717	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
718	if (iGpr < 8)
719	pbCodeBuf[off++] = X86_OP_REX_W;
720	else
721	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
722	pbCodeBuf[off++] = 0x89;
723	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
724	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
725
726	#elif defined(RT_ARCH_ARM64)
727	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
728
729	#else
730	# error "port me"
731	#endif
732	return off;
733	}
734
735
736	/**
737	* Emits a store of a GPR value to a 32-bit VCpu field.
738	*/
739	DECL_INLINE_THROW(uint32_t)
740	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
741	{
742	#ifdef RT_ARCH_AMD64
743	/* mov mem32, reg32 */
744	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
745	if (iGpr >= 8)
746	pbCodeBuf[off++] = X86_OP_REX_R;
747	pbCodeBuf[off++] = 0x89;
748	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
749	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
750
751	#elif defined(RT_ARCH_ARM64)
752	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
753
754	#else
755	# error "port me"
756	#endif
757	return off;
758	}
759
760
761	/**
762	* Emits a store of a GPR value to a 16-bit VCpu field.
763	*/
764	DECL_INLINE_THROW(uint32_t)
765	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
766	{
767	#ifdef RT_ARCH_AMD64
768	/* mov mem16, reg16 */
769	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
770	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
771	if (iGpr >= 8)
772	pbCodeBuf[off++] = X86_OP_REX_R;
773	pbCodeBuf[off++] = 0x89;
774	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
775	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
776
777	#elif defined(RT_ARCH_ARM64)
778	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
779
780	#else
781	# error "port me"
782	#endif
783	return off;
784	}
785
786
787	/**
788	* Emits a store of a GPR value to a 8-bit VCpu field.
789	*/
790	DECL_INLINE_THROW(uint32_t)
791	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
792	{
793	#ifdef RT_ARCH_AMD64
794	/* mov mem8, reg8 */
795	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
796	if (iGpr >= 8)
797	pbCodeBuf[off++] = X86_OP_REX_R;
798	pbCodeBuf[off++] = 0x88;
799	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
800	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
801
802	#elif defined(RT_ARCH_ARM64)
803	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
804
805	#else
806	# error "port me"
807	#endif
808	return off;
809	}
810
811
812	/**
813	* Emits a store of an immediate value to a 8-bit VCpu field.
814	*/
815	DECL_INLINE_THROW(uint32_t)
816	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
817	{
818	#ifdef RT_ARCH_AMD64
819	/* mov mem8, imm8 */
820	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
821	pbCodeBuf[off++] = 0xc6;
822	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
823	pbCodeBuf[off++] = bImm;
824	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
825
826	#elif defined(RT_ARCH_ARM64)
827	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
828	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
829	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
830	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
831
832	#else
833	# error "port me"
834	#endif
835	return off;
836	}
837
838
839	/**
840	* Emits a load effective address to a GRP of a VCpu field.
841	*/
842	DECL_INLINE_THROW(uint32_t)
843	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
844	{
845	#ifdef RT_ARCH_AMD64
846	/* lea gprdst, [rbx + offDisp] */
847	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
848	if (iGprDst < 8)
849	pbCodeBuf[off++] = X86_OP_REX_W;
850	else
851	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
852	pbCodeBuf[off++] = 0x8d;
853	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
854
855	#elif defined(RT_ARCH_ARM64)
856	if (offVCpu < (unsigned)_4K)
857	{
858	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
859	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
860	}
861	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
862	{
863	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
864	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
865	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
866	}
867	else
868	{
869	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
870	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
871	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
872	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
873	}
874
875	#else
876	# error "port me"
877	#endif
878	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
879	return off;
880	}
881
882
883	/** This is just as a typesafe alternative to RT_UOFFSETOF. */
884	DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromStamCounterPtr(PVMCPU pVCpu, PSTAMCOUNTER pStamCounter)
885	{
886	uintptr_t const off = (uintptr_t)pStamCounter - (uintptr_t)pVCpu;
887	Assert(off < sizeof(VMCPU));
888	return off;
889	}
890
891
892	/** This is just as a typesafe alternative to RT_UOFFSETOF. */
893	DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromU64Ptr(PVMCPU pVCpu, uint64_t *pu64)
894	{
895	uintptr_t const off = (uintptr_t)pu64 - (uintptr_t)pVCpu;
896	Assert(off < sizeof(VMCPU));
897	return off;
898	}
899
900
901	/**
902	* Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
903	*
904	* @note The two temp registers are not required for AMD64. ARM64 always
905	* requires the first, and the 2nd is needed if the offset cannot be
906	* encoded as an immediate.
907	*/
908	DECL_FORCE_INLINE(uint32_t)
909	iemNativeEmitIncStamCounterInVCpuEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
910	{
911	#ifdef RT_ARCH_AMD64
912	/* inc qword [pVCpu + off] */
913	pCodeBuf[off++] = X86_OP_REX_W;
914	pCodeBuf[off++] = 0xff;
915	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
916	RT_NOREF(idxTmp1, idxTmp2);
917
918	#elif defined(RT_ARCH_ARM64)
919	/* Determine how we're to access pVCpu first. */
920	uint32_t const cbData = sizeof(STAMCOUNTER);
921	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
922	{
923	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
924	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1,
925	IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
926	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
927	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1,
928	IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
929	}
930	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
931	{
932	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
933	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
934	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
935	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
936	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
937	}
938	else
939	{
940	/* The offset is too large, so we must load it into a register and use
941	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
942	off = iemNativeEmitLoadGprImmEx(pReNative, off, idxTmp2, offVCpu);
943	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
944	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
945	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
946	}
947
948	#else
949	# error "port me"
950	#endif
951	return off;
952	}
953
954
955	/**
956	* Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
957	*
958	* @note The two temp registers are not required for AMD64. ARM64 always
959	* requires the first, and the 2nd is needed if the offset cannot be
960	* encoded as an immediate.
961	*/
962	DECL_FORCE_INLINE(uint32_t)
963	iemNativeEmitIncStamCounterInVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
964	{
965	#ifdef RT_ARCH_AMD64
966	off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, idxTmp1, idxTmp2, offVCpu);
967	#elif defined(RT_ARCH_ARM64)
968	off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 4+3), off, idxTmp1, idxTmp2, offVCpu);
969	#else
970	# error "port me"
971	#endif
972	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
973	return off;
974	}
975
976
977	/**
978	* Emits a gprdst = gprsrc load.
979	*/
980	DECL_FORCE_INLINE(uint32_t)
981	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
982	{
983	#ifdef RT_ARCH_AMD64
984	/* mov gprdst, gprsrc */
985	if ((iGprDst \| iGprSrc) >= 8)
986	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
987	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
988	: X86_OP_REX_W \| X86_OP_REX_R;
989	else
990	pCodeBuf[off++] = X86_OP_REX_W;
991	pCodeBuf[off++] = 0x8b;
992	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
993
994	#elif defined(RT_ARCH_ARM64)
995	/* mov dst, src; alias for: orr dst, xzr, src */
996	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
997
998	#else
999	# error "port me"
1000	#endif
1001	return off;
1002	}
1003
1004
1005	/**
1006	* Emits a gprdst = gprsrc load.
1007	*/
1008	DECL_INLINE_THROW(uint32_t)
1009	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1010	{
1011	#ifdef RT_ARCH_AMD64
1012	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
1013	#elif defined(RT_ARCH_ARM64)
1014	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1015	#else
1016	# error "port me"
1017	#endif
1018	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1019	return off;
1020	}
1021
1022
1023	/**
1024	* Emits a gprdst = gprsrc[31:0] load.
1025	* @note Bits 63 thru 32 are cleared.
1026	*/
1027	DECL_FORCE_INLINE(uint32_t)
1028	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1029	{
1030	#ifdef RT_ARCH_AMD64
1031	/* mov gprdst, gprsrc */
1032	if ((iGprDst \| iGprSrc) >= 8)
1033	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1034	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1035	: X86_OP_REX_R;
1036	pCodeBuf[off++] = 0x8b;
1037	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1038
1039	#elif defined(RT_ARCH_ARM64)
1040	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
1041	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
1042
1043	#else
1044	# error "port me"
1045	#endif
1046	return off;
1047	}
1048
1049
1050	/**
1051	* Emits a gprdst = gprsrc[31:0] load.
1052	* @note Bits 63 thru 32 are cleared.
1053	*/
1054	DECL_INLINE_THROW(uint32_t)
1055	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1056	{
1057	#ifdef RT_ARCH_AMD64
1058	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
1059	#elif defined(RT_ARCH_ARM64)
1060	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1061	#else
1062	# error "port me"
1063	#endif
1064	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1065	return off;
1066	}
1067
1068
1069	/**
1070	* Emits a gprdst = gprsrc[15:0] load.
1071	* @note Bits 63 thru 15 are cleared.
1072	*/
1073	DECL_INLINE_THROW(uint32_t)
1074	iemNativeEmitLoadGprFromGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1075	{
1076	#ifdef RT_ARCH_AMD64
1077	/* movzx Gv,Ew */
1078	if ((iGprDst \| iGprSrc) >= 8)
1079	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1080	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1081	: X86_OP_REX_R;
1082	pCodeBuf[off++] = 0x0f;
1083	pCodeBuf[off++] = 0xb7;
1084	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1085
1086	#elif defined(RT_ARCH_ARM64)
1087	/* and gprdst, gprsrc, #0xffff */
1088	# if 1
1089	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
1090	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
1091	# else
1092	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
1093	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1094	# endif
1095
1096	#else
1097	# error "port me"
1098	#endif
1099	return off;
1100	}
1101
1102
1103	/**
1104	* Emits a gprdst = gprsrc[15:0] load.
1105	* @note Bits 63 thru 15 are cleared.
1106	*/
1107	DECL_INLINE_THROW(uint32_t)
1108	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1109	{
1110	#ifdef RT_ARCH_AMD64
1111	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1112	#elif defined(RT_ARCH_ARM64)
1113	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1114	#else
1115	# error "port me"
1116	#endif
1117	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1118	return off;
1119	}
1120
1121
1122	/**
1123	* Emits a gprdst = gprsrc[7:0] load.
1124	* @note Bits 63 thru 8 are cleared.
1125	*/
1126	DECL_FORCE_INLINE(uint32_t)
1127	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1128	{
1129	#ifdef RT_ARCH_AMD64
1130	/* movzx Gv,Eb */
1131	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1132	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1133	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1134	: X86_OP_REX_R;
1135	else if (iGprSrc >= 4)
1136	pCodeBuf[off++] = X86_OP_REX;
1137	pCodeBuf[off++] = 0x0f;
1138	pCodeBuf[off++] = 0xb6;
1139	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1140
1141	#elif defined(RT_ARCH_ARM64)
1142	/* and gprdst, gprsrc, #0xff */
1143	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1144	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
1145
1146	#else
1147	# error "port me"
1148	#endif
1149	return off;
1150	}
1151
1152
1153	/**
1154	* Emits a gprdst = gprsrc[7:0] load.
1155	* @note Bits 63 thru 8 are cleared.
1156	*/
1157	DECL_INLINE_THROW(uint32_t)
1158	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1159	{
1160	#ifdef RT_ARCH_AMD64
1161	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1162	#elif defined(RT_ARCH_ARM64)
1163	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1164	#else
1165	# error "port me"
1166	#endif
1167	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1168	return off;
1169	}
1170
1171
1172	/**
1173	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1174	* @note Bits 63 thru 8 are cleared.
1175	*/
1176	DECL_INLINE_THROW(uint32_t)
1177	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1178	{
1179	#ifdef RT_ARCH_AMD64
1180	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1181
1182	/* movzx Gv,Ew */
1183	if ((iGprDst \| iGprSrc) >= 8)
1184	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1185	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1186	: X86_OP_REX_R;
1187	pbCodeBuf[off++] = 0x0f;
1188	pbCodeBuf[off++] = 0xb7;
1189	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1190
1191	/* shr Ev,8 */
1192	if (iGprDst >= 8)
1193	pbCodeBuf[off++] = X86_OP_REX_B;
1194	pbCodeBuf[off++] = 0xc1;
1195	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1196	pbCodeBuf[off++] = 8;
1197
1198	#elif defined(RT_ARCH_ARM64)
1199	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1200	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1201	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1202
1203	#else
1204	# error "port me"
1205	#endif
1206	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1207	return off;
1208	}
1209
1210
1211	/**
1212	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1213	*/
1214	DECL_INLINE_THROW(uint32_t)
1215	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1216	{
1217	#ifdef RT_ARCH_AMD64
1218	/* movsxd r64, r/m32 */
1219	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1220	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1221	pbCodeBuf[off++] = 0x63;
1222	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1223
1224	#elif defined(RT_ARCH_ARM64)
1225	/* sxtw dst, src */
1226	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1227	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1228
1229	#else
1230	# error "port me"
1231	#endif
1232	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1233	return off;
1234	}
1235
1236
1237	/**
1238	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1239	*/
1240	DECL_INLINE_THROW(uint32_t)
1241	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1242	{
1243	#ifdef RT_ARCH_AMD64
1244	/* movsx r64, r/m16 */
1245	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1246	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1247	pbCodeBuf[off++] = 0x0f;
1248	pbCodeBuf[off++] = 0xbf;
1249	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1250
1251	#elif defined(RT_ARCH_ARM64)
1252	/* sxth dst, src */
1253	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1254	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1255
1256	#else
1257	# error "port me"
1258	#endif
1259	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1260	return off;
1261	}
1262
1263
1264	/**
1265	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1266	*/
1267	DECL_INLINE_THROW(uint32_t)
1268	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1269	{
1270	#ifdef RT_ARCH_AMD64
1271	/* movsx r64, r/m16 */
1272	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1273	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1274	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1275	pbCodeBuf[off++] = 0x0f;
1276	pbCodeBuf[off++] = 0xbf;
1277	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1278
1279	#elif defined(RT_ARCH_ARM64)
1280	/* sxth dst32, src */
1281	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1282	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1283
1284	#else
1285	# error "port me"
1286	#endif
1287	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1288	return off;
1289	}
1290
1291
1292	/**
1293	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1294	*/
1295	DECL_INLINE_THROW(uint32_t)
1296	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1297	{
1298	#ifdef RT_ARCH_AMD64
1299	/* movsx r64, r/m8 */
1300	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1301	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1302	pbCodeBuf[off++] = 0x0f;
1303	pbCodeBuf[off++] = 0xbe;
1304	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1305
1306	#elif defined(RT_ARCH_ARM64)
1307	/* sxtb dst, src */
1308	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1309	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1310
1311	#else
1312	# error "port me"
1313	#endif
1314	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1315	return off;
1316	}
1317
1318
1319	/**
1320	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1321	* @note Bits 63 thru 32 are cleared.
1322	*/
1323	DECL_INLINE_THROW(uint32_t)
1324	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1325	{
1326	#ifdef RT_ARCH_AMD64
1327	/* movsx r32, r/m8 */
1328	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1329	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1330	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1331	else if (iGprSrc >= 4)
1332	pbCodeBuf[off++] = X86_OP_REX;
1333	pbCodeBuf[off++] = 0x0f;
1334	pbCodeBuf[off++] = 0xbe;
1335	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1336
1337	#elif defined(RT_ARCH_ARM64)
1338	/* sxtb dst32, src32 */
1339	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1340	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1341
1342	#else
1343	# error "port me"
1344	#endif
1345	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1346	return off;
1347	}
1348
1349
1350	/**
1351	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1352	* @note Bits 63 thru 16 are cleared.
1353	*/
1354	DECL_INLINE_THROW(uint32_t)
1355	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1356	{
1357	#ifdef RT_ARCH_AMD64
1358	/* movsx r16, r/m8 */
1359	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1360	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1361	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1362	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1363	else if (iGprSrc >= 4)
1364	pbCodeBuf[off++] = X86_OP_REX;
1365	pbCodeBuf[off++] = 0x0f;
1366	pbCodeBuf[off++] = 0xbe;
1367	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1368
1369	/* movzx r32, r/m16 */
1370	if (iGprDst >= 8)
1371	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1372	pbCodeBuf[off++] = 0x0f;
1373	pbCodeBuf[off++] = 0xb7;
1374	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1375
1376	#elif defined(RT_ARCH_ARM64)
1377	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1378	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1379	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1380	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1381	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1382
1383	#else
1384	# error "port me"
1385	#endif
1386	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1387	return off;
1388	}
1389
1390
1391	/**
1392	* Emits a gprdst = gprsrc + addend load.
1393	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1394	*/
1395	#ifdef RT_ARCH_AMD64
1396	DECL_INLINE_THROW(uint32_t)
1397	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1398	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1399	{
1400	Assert(iAddend != 0);
1401
1402	/* lea gprdst, [gprsrc + iAddend] */
1403	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1404	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1405	pbCodeBuf[off++] = 0x8d;
1406	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1407	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1408	return off;
1409	}
1410
1411	#elif defined(RT_ARCH_ARM64)
1412	DECL_INLINE_THROW(uint32_t)
1413	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1414	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1415	{
1416	if ((uint32_t)iAddend < 4096)
1417	{
1418	/* add dst, src, uimm12 */
1419	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1420	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1421	}
1422	else if ((uint32_t)-iAddend < 4096)
1423	{
1424	/* sub dst, src, uimm12 */
1425	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1426	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1427	}
1428	else
1429	{
1430	Assert(iGprSrc != iGprDst);
1431	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1432	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1433	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1434	}
1435	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1436	return off;
1437	}
1438	#else
1439	# error "port me"
1440	#endif
1441
1442	/**
1443	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1444	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1445	*/
1446	#ifdef RT_ARCH_AMD64
1447	DECL_INLINE_THROW(uint32_t)
1448	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1449	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1450	#else
1451	DECL_INLINE_THROW(uint32_t)
1452	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1453	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1454	#endif
1455	{
1456	if (iAddend != 0)
1457	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1458	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1459	}
1460
1461
1462	/**
1463	* Emits a gprdst = gprsrc32 + addend load.
1464	* @note Bits 63 thru 32 are cleared.
1465	*/
1466	DECL_INLINE_THROW(uint32_t)
1467	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1468	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1469	{
1470	Assert(iAddend != 0);
1471
1472	#ifdef RT_ARCH_AMD64
1473	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1474	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1475	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1476	if ((iGprDst \| iGprSrc) >= 8)
1477	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1478	pbCodeBuf[off++] = 0x8d;
1479	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1480
1481	#elif defined(RT_ARCH_ARM64)
1482	if ((uint32_t)iAddend < 4096)
1483	{
1484	/* add dst, src, uimm12 */
1485	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1486	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1487	}
1488	else if ((uint32_t)-iAddend < 4096)
1489	{
1490	/* sub dst, src, uimm12 */
1491	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1492	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1493	}
1494	else
1495	{
1496	Assert(iGprSrc != iGprDst);
1497	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1498	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1499	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1500	}
1501
1502	#else
1503	# error "port me"
1504	#endif
1505	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1506	return off;
1507	}
1508
1509
1510	/**
1511	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1512	*/
1513	DECL_INLINE_THROW(uint32_t)
1514	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1515	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1516	{
1517	if (iAddend != 0)
1518	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1519	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1520	}
1521
1522
1523	/**
1524	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1525	* destination.
1526	*/
1527	DECL_FORCE_INLINE(uint32_t)
1528	iemNativeEmitGprMergeInGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1529	{
1530	#ifdef RT_ARCH_AMD64
1531	/* mov reg16, r/m16 */
1532	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1533	if (idxDst >= 8 \|\| idxSrc >= 8)
1534	pCodeBuf[off++] = (idxDst < 8 ? 0 : X86_OP_REX_R) \| (idxSrc < 8 ? 0 : X86_OP_REX_B);
1535	pCodeBuf[off++] = 0x8b;
1536	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxDst & 7, idxSrc & 7);
1537
1538	#elif defined(RT_ARCH_ARM64)
1539	/* bfi w1, w2, 0, 16 - moves bits 15:0 from idxSrc to idxDst bits 15:0. */
1540	pCodeBuf[off++] = Armv8A64MkInstrBfi(idxDst, idxSrc, 0, 16);
1541
1542	#else
1543	# error "Port me!"
1544	#endif
1545	return off;
1546	}
1547
1548
1549	/**
1550	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1551	* destination.
1552	*/
1553	DECL_INLINE_THROW(uint32_t)
1554	iemNativeEmitGprMergeInGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1555	{
1556	#ifdef RT_ARCH_AMD64
1557	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, idxDst, idxSrc);
1558	#elif defined(RT_ARCH_ARM64)
1559	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, idxDst, idxSrc);
1560	#else
1561	# error "Port me!"
1562	#endif
1563	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1564	return off;
1565	}
1566
1567
1568	#ifdef RT_ARCH_AMD64
1569	/**
1570	* Common bit of iemNativeEmitLoadGprByBp and friends.
1571	*/
1572	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1573	PIEMRECOMPILERSTATE pReNativeAssert)
1574	{
1575	if (offDisp < 128 && offDisp >= -128)
1576	{
1577	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1578	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1579	}
1580	else
1581	{
1582	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1583	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1584	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1585	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1586	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1587	}
1588	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1589	return off;
1590	}
1591	#elif defined(RT_ARCH_ARM64)
1592	/**
1593	* Common bit of iemNativeEmitLoadGprByBp and friends.
1594	*/
1595	DECL_FORCE_INLINE_THROW(uint32_t)
1596	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1597	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1598	{
1599	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1600	{
1601	/* str w/ unsigned imm12 (scaled) */
1602	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1603	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1604	}
1605	else if (offDisp >= -256 && offDisp <= 256)
1606	{
1607	/* stur w/ signed imm9 (unscaled) */
1608	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1609	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1610	}
1611	else
1612	{
1613	/* Use temporary indexing register. */
1614	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1615	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1616	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1617	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1618	}
1619	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1620	return off;
1621	}
1622	#endif
1623
1624
1625	/**
1626	* Emits a 64-bit GRP load instruction with an BP relative source address.
1627	*/
1628	DECL_INLINE_THROW(uint32_t)
1629	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1630	{
1631	#ifdef RT_ARCH_AMD64
1632	/* mov gprdst, qword [rbp + offDisp] */
1633	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1634	if (iGprDst < 8)
1635	pbCodeBuf[off++] = X86_OP_REX_W;
1636	else
1637	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1638	pbCodeBuf[off++] = 0x8b;
1639	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1640
1641	#elif defined(RT_ARCH_ARM64)
1642	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1643
1644	#else
1645	# error "port me"
1646	#endif
1647	}
1648
1649
1650	/**
1651	* Emits a 32-bit GRP load instruction with an BP relative source address.
1652	* @note Bits 63 thru 32 of the GPR will be cleared.
1653	*/
1654	DECL_INLINE_THROW(uint32_t)
1655	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1656	{
1657	#ifdef RT_ARCH_AMD64
1658	/* mov gprdst, dword [rbp + offDisp] */
1659	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1660	if (iGprDst >= 8)
1661	pbCodeBuf[off++] = X86_OP_REX_R;
1662	pbCodeBuf[off++] = 0x8b;
1663	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1664
1665	#elif defined(RT_ARCH_ARM64)
1666	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1667
1668	#else
1669	# error "port me"
1670	#endif
1671	}
1672
1673
1674	/**
1675	* Emits a 16-bit GRP load instruction with an BP relative source address.
1676	* @note Bits 63 thru 16 of the GPR will be cleared.
1677	*/
1678	DECL_INLINE_THROW(uint32_t)
1679	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1680	{
1681	#ifdef RT_ARCH_AMD64
1682	/* movzx gprdst, word [rbp + offDisp] */
1683	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1684	if (iGprDst >= 8)
1685	pbCodeBuf[off++] = X86_OP_REX_R;
1686	pbCodeBuf[off++] = 0x0f;
1687	pbCodeBuf[off++] = 0xb7;
1688	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1689
1690	#elif defined(RT_ARCH_ARM64)
1691	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1692
1693	#else
1694	# error "port me"
1695	#endif
1696	}
1697
1698
1699	/**
1700	* Emits a 8-bit GRP load instruction with an BP relative source address.
1701	* @note Bits 63 thru 8 of the GPR will be cleared.
1702	*/
1703	DECL_INLINE_THROW(uint32_t)
1704	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1705	{
1706	#ifdef RT_ARCH_AMD64
1707	/* movzx gprdst, byte [rbp + offDisp] */
1708	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1709	if (iGprDst >= 8)
1710	pbCodeBuf[off++] = X86_OP_REX_R;
1711	pbCodeBuf[off++] = 0x0f;
1712	pbCodeBuf[off++] = 0xb6;
1713	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1714
1715	#elif defined(RT_ARCH_ARM64)
1716	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1717
1718	#else
1719	# error "port me"
1720	#endif
1721	}
1722
1723
1724	/**
1725	* Emits a load effective address to a GRP with an BP relative source address.
1726	*/
1727	DECL_INLINE_THROW(uint32_t)
1728	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1729	{
1730	#ifdef RT_ARCH_AMD64
1731	/* lea gprdst, [rbp + offDisp] */
1732	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1733	if (iGprDst < 8)
1734	pbCodeBuf[off++] = X86_OP_REX_W;
1735	else
1736	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1737	pbCodeBuf[off++] = 0x8d;
1738	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1739
1740	#elif defined(RT_ARCH_ARM64)
1741	if ((uint32_t)offDisp < (unsigned)_4K)
1742	{
1743	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1744	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1745	}
1746	else if ((uint32_t)-offDisp < (unsigned)_4K)
1747	{
1748	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1749	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1750	}
1751	else
1752	{
1753	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1754	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1755	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1756	if (offDisp >= 0)
1757	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1758	else
1759	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1760	}
1761
1762	#else
1763	# error "port me"
1764	#endif
1765
1766	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1767	return off;
1768	}
1769
1770
1771	/**
1772	* Emits a 64-bit GPR store with an BP relative destination address.
1773	*
1774	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1775	*/
1776	DECL_INLINE_THROW(uint32_t)
1777	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1778	{
1779	#ifdef RT_ARCH_AMD64
1780	/* mov qword [rbp + offDisp], gprdst */
1781	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1782	if (iGprSrc < 8)
1783	pbCodeBuf[off++] = X86_OP_REX_W;
1784	else
1785	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1786	pbCodeBuf[off++] = 0x89;
1787	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1788
1789	#elif defined(RT_ARCH_ARM64)
1790	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1791	{
1792	/* str w/ unsigned imm12 (scaled) */
1793	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1794	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1795	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1796	}
1797	else if (offDisp >= -256 && offDisp <= 256)
1798	{
1799	/* stur w/ signed imm9 (unscaled) */
1800	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1801	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1802	}
1803	else if ((uint32_t)-offDisp < (unsigned)_4K)
1804	{
1805	/* Use temporary indexing register w/ sub uimm12. */
1806	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1807	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1808	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1809	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1810	}
1811	else
1812	{
1813	/* Use temporary indexing register. */
1814	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1815	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1816	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1817	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1818	}
1819	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1820	return off;
1821
1822	#else
1823	# error "Port me!"
1824	#endif
1825	}
1826
1827
1828	/**
1829	* Emits a 64-bit immediate store with an BP relative destination address.
1830	*
1831	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1832	*/
1833	DECL_INLINE_THROW(uint32_t)
1834	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1835	{
1836	#ifdef RT_ARCH_AMD64
1837	if ((int64_t)uImm64 == (int32_t)uImm64)
1838	{
1839	/* mov qword [rbp + offDisp], imm32 - sign extended */
1840	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1841	pbCodeBuf[off++] = X86_OP_REX_W;
1842	pbCodeBuf[off++] = 0xc7;
1843	if (offDisp < 128 && offDisp >= -128)
1844	{
1845	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1846	pbCodeBuf[off++] = (uint8_t)offDisp;
1847	}
1848	else
1849	{
1850	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1851	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1852	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1853	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1854	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1855	}
1856	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1857	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1858	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1859	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1860	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1861	return off;
1862	}
1863	#endif
1864
1865	/* Load tmp0, imm64; Store tmp to bp+disp. */
1866	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1867	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1868	}
1869
1870	#if defined(RT_ARCH_ARM64)
1871
1872	/**
1873	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1874	*
1875	* @note Odd and large @a offDisp values requires a temporary, unless it's a
1876	* load and @a iGprReg differs from @a iGprBase. Will assert / throw if
1877	* caller does not heed this.
1878	*
1879	* @note DON'T try this with prefetch.
1880	*/
1881	DECL_FORCE_INLINE_THROW(uint32_t)
1882	iemNativeEmitGprByGprLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp,
1883	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
1884	{
1885	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1886	{
1887	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1888	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1889	}
1890	else if ( ( !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation)
1891	&& iGprReg != iGprBase)
1892	\|\| iGprTmp != UINT8_MAX)
1893	{
1894	/* The offset is too large, so we must load it into a register and use
1895	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1896	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1897	if (iGprTmp == UINT8_MAX)
1898	iGprTmp = iGprReg;
1899	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (int64_t)offDisp);
1900	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, iGprTmp);
1901	}
1902	else
1903	# ifdef IEM_WITH_THROW_CATCH
1904	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
1905	# else
1906	AssertReleaseFailedStmt(off = UINT32_MAX);
1907	# endif
1908	return off;
1909	}
1910
1911	/**
1912	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1913	*/
1914	DECL_FORCE_INLINE_THROW(uint32_t)
1915	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1916	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1917	{
1918	/*
1919	* There are a couple of ldr variants that takes an immediate offset, so
1920	* try use those if we can, otherwise we have to use the temporary register
1921	* help with the addressing.
1922	*/
1923	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1924	{
1925	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1926	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1927	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1928	}
1929	else
1930	{
1931	/* The offset is too large, so we must load it into a register and use
1932	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1933	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1934	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (int64_t)offDisp);
1935
1936	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1937	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1938
1939	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1940	}
1941	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1942	return off;
1943	}
1944
1945	#endif /* RT_ARCH_ARM64 */
1946
1947	/**
1948	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1949	*
1950	* @note ARM64: Misaligned @a offDisp values and values not in the
1951	* -0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
1952	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
1953	* does not heed this.
1954	*/
1955	DECL_FORCE_INLINE_THROW(uint32_t)
1956	iemNativeEmitLoadGprByGprU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1957	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1958	{
1959	#ifdef RT_ARCH_AMD64
1960	/* mov reg64, mem64 */
1961	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1962	pCodeBuf[off++] = 0x8b;
1963	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1964	RT_NOREF(iGprTmp);
1965
1966	#elif defined(RT_ARCH_ARM64)
1967	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1968	kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t), iGprTmp);
1969
1970	#else
1971	# error "port me"
1972	#endif
1973	return off;
1974	}
1975
1976
1977	/**
1978	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1979	*/
1980	DECL_INLINE_THROW(uint32_t)
1981	iemNativeEmitLoadGprByGprU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1982	{
1983	#ifdef RT_ARCH_AMD64
1984	off = iemNativeEmitLoadGprByGprU64Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
1985	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1986
1987	#elif defined(RT_ARCH_ARM64)
1988	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1989
1990	#else
1991	# error "port me"
1992	#endif
1993	return off;
1994	}
1995
1996
1997	/**
1998	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1999	*
2000	* @note ARM64: Misaligned @a offDisp values and values not in the
2001	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
2002	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2003	* caller does not heed this.
2004	*
2005	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2006	*/
2007	DECL_FORCE_INLINE_THROW(uint32_t)
2008	iemNativeEmitLoadGprByGprU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2009	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2010	{
2011	#ifdef RT_ARCH_AMD64
2012	/* mov reg32, mem32 */
2013	if (iGprDst >= 8 \|\| iGprBase >= 8)
2014	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2015	pCodeBuf[off++] = 0x8b;
2016	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2017	RT_NOREF(iGprTmp);
2018
2019	#elif defined(RT_ARCH_ARM64)
2020	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2021	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t), iGprTmp);
2022
2023	#else
2024	# error "port me"
2025	#endif
2026	return off;
2027	}
2028
2029
2030	/**
2031	* Emits a 32-bit GPR load via a GPR base address with a displacement.
2032	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2033	*/
2034	DECL_INLINE_THROW(uint32_t)
2035	iemNativeEmitLoadGprByGprU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
2036	{
2037	#ifdef RT_ARCH_AMD64
2038	off = iemNativeEmitLoadGprByGprU32Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
2039	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2040
2041	#elif defined(RT_ARCH_ARM64)
2042	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
2043
2044	#else
2045	# error "port me"
2046	#endif
2047	return off;
2048	}
2049
2050
2051	/**
2052	* Emits a 32-bit GPR load via a GPR base address with a displacement,
2053	* sign-extending the value to 64 bits.
2054	*
2055	* @note ARM64: Misaligned @a offDisp values and values not in the
2056	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
2057	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2058	* caller does not heed this.
2059	*/
2060	DECL_FORCE_INLINE_THROW(uint32_t)
2061	iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2062	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2063	{
2064	#ifdef RT_ARCH_AMD64
2065	/* movsxd reg64, mem32 */
2066	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2067	pCodeBuf[off++] = 0x63;
2068	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2069	RT_NOREF(iGprTmp);
2070
2071	#elif defined(RT_ARCH_ARM64)
2072	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2073	kArmv8A64InstrLdStType_Ld_SignWord64, sizeof(uint32_t), iGprTmp);
2074
2075	#else
2076	# error "port me"
2077	#endif
2078	return off;
2079	}
2080
2081
2082	/**
2083	* Emits a 16-bit GPR load via a GPR base address with a displacement.
2084	*
2085	* @note ARM64: Misaligned @a offDisp values and values not in the
2086	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2087	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2088	* caller does not heed this.
2089	*
2090	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2091	*/
2092	DECL_FORCE_INLINE_THROW(uint32_t)
2093	iemNativeEmitLoadGprByGprU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2094	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2095	{
2096	#ifdef RT_ARCH_AMD64
2097	/* movzx reg32, mem16 */
2098	if (iGprDst >= 8 \|\| iGprBase >= 8)
2099	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2100	pCodeBuf[off++] = 0x0f;
2101	pCodeBuf[off++] = 0xb7;
2102	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2103	RT_NOREF(iGprTmp);
2104
2105	#elif defined(RT_ARCH_ARM64)
2106	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2107	kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t), iGprTmp);
2108
2109	#else
2110	# error "port me"
2111	#endif
2112	return off;
2113	}
2114
2115
2116	/**
2117	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2118	* sign-extending the value to 64 bits.
2119	*
2120	* @note ARM64: Misaligned @a offDisp values and values not in the
2121	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2122	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2123	* caller does not heed this.
2124	*/
2125	DECL_FORCE_INLINE_THROW(uint32_t)
2126	iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2127	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2128	{
2129	#ifdef RT_ARCH_AMD64
2130	/* movsx reg64, mem16 */
2131	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2132	pCodeBuf[off++] = 0x0f;
2133	pCodeBuf[off++] = 0xbf;
2134	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2135	RT_NOREF(iGprTmp);
2136
2137	#elif defined(RT_ARCH_ARM64)
2138	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2139	kArmv8A64InstrLdStType_Ld_SignHalf64, sizeof(uint16_t), iGprTmp);
2140
2141	#else
2142	# error "port me"
2143	#endif
2144	return off;
2145	}
2146
2147
2148	/**
2149	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2150	* sign-extending the value to 32 bits.
2151	*
2152	* @note ARM64: Misaligned @a offDisp values and values not in the
2153	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2154	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2155	* caller does not heed this.
2156	*
2157	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2158	*/
2159	DECL_FORCE_INLINE_THROW(uint32_t)
2160	iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2161	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2162	{
2163	#ifdef RT_ARCH_AMD64
2164	/* movsx reg32, mem16 */
2165	if (iGprDst >= 8 \|\| iGprBase >= 8)
2166	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2167	pCodeBuf[off++] = 0x0f;
2168	pCodeBuf[off++] = 0xbf;
2169	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2170	RT_NOREF(iGprTmp);
2171
2172	#elif defined(RT_ARCH_ARM64)
2173	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2174	kArmv8A64InstrLdStType_Ld_SignHalf32, sizeof(uint16_t), iGprTmp);
2175
2176	#else
2177	# error "port me"
2178	#endif
2179	return off;
2180	}
2181
2182
2183	/**
2184	* Emits a 8-bit GPR load via a GPR base address with a displacement.
2185	*
2186	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2187	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2188	* same. Will assert / throw if caller does not heed this.
2189	*
2190	* @note Bits 63 thru 8 in @a iGprDst will be cleared.
2191	*/
2192	DECL_FORCE_INLINE_THROW(uint32_t)
2193	iemNativeEmitLoadGprByGprU8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2194	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2195	{
2196	#ifdef RT_ARCH_AMD64
2197	/* movzx reg32, mem8 */
2198	if (iGprDst >= 8 \|\| iGprBase >= 8)
2199	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2200	pCodeBuf[off++] = 0x0f;
2201	pCodeBuf[off++] = 0xb6;
2202	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2203	RT_NOREF(iGprTmp);
2204
2205	#elif defined(RT_ARCH_ARM64)
2206	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2207	kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t), iGprTmp);
2208
2209	#else
2210	# error "port me"
2211	#endif
2212	return off;
2213	}
2214
2215
2216	/**
2217	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2218	* sign-extending the value to 64 bits.
2219	*
2220	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2221	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2222	* same. Will assert / throw if caller does not heed this.
2223	*/
2224	DECL_FORCE_INLINE_THROW(uint32_t)
2225	iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2226	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2227	{
2228	#ifdef RT_ARCH_AMD64
2229	/* movsx reg64, mem8 */
2230	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2231	pCodeBuf[off++] = 0x0f;
2232	pCodeBuf[off++] = 0xbe;
2233	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2234	RT_NOREF(iGprTmp);
2235
2236	#elif defined(RT_ARCH_ARM64)
2237	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2238	kArmv8A64InstrLdStType_Ld_SignByte64, sizeof(uint8_t), iGprTmp);
2239
2240	#else
2241	# error "port me"
2242	#endif
2243	return off;
2244	}
2245
2246
2247	/**
2248	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2249	* sign-extending the value to 32 bits.
2250	*
2251	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2252	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2253	* same. Will assert / throw if caller does not heed this.
2254	*
2255	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2256	*/
2257	DECL_FORCE_INLINE_THROW(uint32_t)
2258	iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2259	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2260	{
2261	#ifdef RT_ARCH_AMD64
2262	/* movsx reg32, mem8 */
2263	if (iGprDst >= 8 \|\| iGprBase >= 8)
2264	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2265	pCodeBuf[off++] = 0x0f;
2266	pCodeBuf[off++] = 0xbe;
2267	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2268	RT_NOREF(iGprTmp);
2269
2270	#elif defined(RT_ARCH_ARM64)
2271	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2272	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2273
2274	#else
2275	# error "port me"
2276	#endif
2277	return off;
2278	}
2279
2280
2281	/**
2282	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2283	* sign-extending the value to 16 bits.
2284	*
2285	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2286	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2287	* same. Will assert / throw if caller does not heed this.
2288	*
2289	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2290	*/
2291	DECL_FORCE_INLINE_THROW(uint32_t)
2292	iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2293	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2294	{
2295	#ifdef RT_ARCH_AMD64
2296	/* movsx reg32, mem8 */
2297	if (iGprDst >= 8 \|\| iGprBase >= 8)
2298	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2299	pCodeBuf[off++] = 0x0f;
2300	pCodeBuf[off++] = 0xbe;
2301	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2302	# if 1 /** @todo use 'movzx reg32, reg16' instead of 'and reg32, 0ffffh' ? */
2303	/* and reg32, 0xffffh */
2304	if (iGprDst >= 8)
2305	pCodeBuf[off++] = X86_OP_REX_B;
2306	pCodeBuf[off++] = 0x81;
2307	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2308	pCodeBuf[off++] = 0xff;
2309	pCodeBuf[off++] = 0xff;
2310	pCodeBuf[off++] = 0;
2311	pCodeBuf[off++] = 0;
2312	# else
2313	/* movzx reg32, reg16 */
2314	if (iGprDst >= 8)
2315	pCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2316	pCodeBuf[off++] = 0x0f;
2317	pCodeBuf[off++] = 0xb7;
2318	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2319	# endif
2320	RT_NOREF(iGprTmp);
2321
2322	#elif defined(RT_ARCH_ARM64)
2323	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2324	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2325	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
2326	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /64Bit/);
2327
2328	#else
2329	# error "port me"
2330	#endif
2331	return off;
2332	}
2333
2334
2335	/**
2336	* Emits a 64-bit GPR store via a GPR base address with a displacement.
2337	*
2338	* @note ARM64: Misaligned @a offDisp values and values not in the
2339	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2340	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2341	* does not heed this.
2342	*/
2343	DECL_FORCE_INLINE_THROW(uint32_t)
2344	iemNativeEmitStoreGpr64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2345	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2346	{
2347	#ifdef RT_ARCH_AMD64
2348	/* mov mem64, reg64 */
2349	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2350	pCodeBuf[off++] = 0x89;
2351	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2352	RT_NOREF(iGprTmp);
2353
2354	#elif defined(RT_ARCH_ARM64)
2355	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2356	kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
2357
2358	#else
2359	# error "port me"
2360	#endif
2361	return off;
2362	}
2363
2364
2365	/**
2366	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2367	*
2368	* @note ARM64: Misaligned @a offDisp values and values not in the
2369	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2370	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2371	* does not heed this.
2372	*/
2373	DECL_FORCE_INLINE_THROW(uint32_t)
2374	iemNativeEmitStoreGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2375	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2376	{
2377	#ifdef RT_ARCH_AMD64
2378	/* mov mem32, reg32 */
2379	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2380	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2381	pCodeBuf[off++] = 0x89;
2382	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2383	RT_NOREF(iGprTmp);
2384
2385	#elif defined(RT_ARCH_ARM64)
2386	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2387	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2388
2389	#else
2390	# error "port me"
2391	#endif
2392	return off;
2393	}
2394
2395
2396	/**
2397	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2398	*
2399	* @note ARM64: Misaligned @a offDisp values and values not in the
2400	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2401	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2402	* does not heed this.
2403	*/
2404	DECL_FORCE_INLINE_THROW(uint32_t)
2405	iemNativeEmitStoreGpr16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2406	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2407	{
2408	#ifdef RT_ARCH_AMD64
2409	/* mov mem16, reg16 */
2410	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2411	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2412	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2413	pCodeBuf[off++] = 0x89;
2414	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2415	RT_NOREF(iGprTmp);
2416
2417	#elif defined(RT_ARCH_ARM64)
2418	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2419	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2420
2421	#else
2422	# error "port me"
2423	#endif
2424	return off;
2425	}
2426
2427
2428	/**
2429	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2430	*
2431	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2432	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2433	* same. Will assert / throw if caller does not heed this.
2434	*/
2435	DECL_FORCE_INLINE_THROW(uint32_t)
2436	iemNativeEmitStoreGpr8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2437	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2438	{
2439	#ifdef RT_ARCH_AMD64
2440	/* mov mem8, reg8 */
2441	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2442	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2443	else if (iGprSrc >= 4)
2444	pCodeBuf[off++] = X86_OP_REX;
2445	pCodeBuf[off++] = 0x88;
2446	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2447	RT_NOREF(iGprTmp);
2448
2449	#elif defined(RT_ARCH_ARM64)
2450	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2451	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2452
2453	#else
2454	# error "port me"
2455	#endif
2456	return off;
2457	}
2458
2459
2460	/**
2461	* Emits a 64-bit immediate store via a GPR base address with a displacement.
2462	*
2463	* @note This will always require @a iGprTmpImm on ARM (except for uImm=0), on
2464	* AMD64 it depends on the immediate value.
2465	*
2466	* @note ARM64: Misaligned @a offDisp values and values not in the
2467	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2468	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2469	* does not heed this.
2470	*/
2471	DECL_FORCE_INLINE_THROW(uint32_t)
2472	iemNativeEmitStoreImm64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint64_t uImm, uint8_t iGprBase,
2473	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2474	{
2475	#ifdef RT_ARCH_AMD64
2476	if ((int32_t)uImm == (int64_t)uImm)
2477	{
2478	/* mov mem64, imm32 (sign-extended) */
2479	pCodeBuf[off++] = X86_OP_REX_W \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2480	pCodeBuf[off++] = 0xc7;
2481	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2482	pCodeBuf[off++] = RT_BYTE1(uImm);
2483	pCodeBuf[off++] = RT_BYTE2(uImm);
2484	pCodeBuf[off++] = RT_BYTE3(uImm);
2485	pCodeBuf[off++] = RT_BYTE4(uImm);
2486	}
2487	else if (iGprImmTmp != UINT8_MAX \|\| iGprTmp != UINT8_MAX)
2488	{
2489	/* require temporary register. */
2490	if (iGprImmTmp == UINT8_MAX)
2491	iGprImmTmp = iGprTmp;
2492	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2493	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp);
2494	}
2495	else
2496	# ifdef IEM_WITH_THROW_CATCH
2497	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2498	# else
2499	AssertReleaseFailedStmt(off = UINT32_MAX);
2500	# endif
2501
2502	#elif defined(RT_ARCH_ARM64)
2503	if (uImm == 0)
2504	iGprImmTmp = ARMV8_A64_REG_XZR;
2505	else
2506	{
2507	Assert(iGprImmTmp < 31);
2508	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2509	}
2510	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp, iGprTmp);
2511
2512	#else
2513	# error "port me"
2514	#endif
2515	return off;
2516	}
2517
2518
2519	/**
2520	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2521	*
2522	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2523	*
2524	* @note ARM64: Misaligned @a offDisp values and values not in the
2525	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2526	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2527	* does not heed this.
2528	*/
2529	DECL_FORCE_INLINE_THROW(uint32_t)
2530	iemNativeEmitStoreImm32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uImm, uint8_t iGprBase,
2531	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2532	{
2533	#ifdef RT_ARCH_AMD64
2534	/* mov mem32, imm32 */
2535	if (iGprBase >= 8)
2536	pCodeBuf[off++] = X86_OP_REX_B;
2537	pCodeBuf[off++] = 0xc7;
2538	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2539	pCodeBuf[off++] = RT_BYTE1(uImm);
2540	pCodeBuf[off++] = RT_BYTE2(uImm);
2541	pCodeBuf[off++] = RT_BYTE3(uImm);
2542	pCodeBuf[off++] = RT_BYTE4(uImm);
2543	RT_NOREF(iGprImmTmp, iGprTmp);
2544
2545	#elif defined(RT_ARCH_ARM64)
2546	Assert(iGprImmTmp < 31);
2547	if (uImm == 0)
2548	iGprImmTmp = ARMV8_A64_REG_XZR;
2549	else
2550	{
2551	Assert(iGprImmTmp < 31);
2552	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2553	}
2554	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2555	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2556
2557	#else
2558	# error "port me"
2559	#endif
2560	return off;
2561	}
2562
2563
2564	/**
2565	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2566	*
2567	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2568	*
2569	* @note ARM64: Misaligned @a offDisp values and values not in the
2570	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2571	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2572	* does not heed this.
2573	*/
2574	DECL_FORCE_INLINE_THROW(uint32_t)
2575	iemNativeEmitStoreImm16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint8_t iGprBase,
2576	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2577	{
2578	#ifdef RT_ARCH_AMD64
2579	/* mov mem16, imm16 */
2580	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2581	if (iGprBase >= 8)
2582	pCodeBuf[off++] = X86_OP_REX_B;
2583	pCodeBuf[off++] = 0xc7;
2584	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2585	pCodeBuf[off++] = RT_BYTE1(uImm);
2586	pCodeBuf[off++] = RT_BYTE2(uImm);
2587	RT_NOREF(iGprImmTmp, iGprTmp);
2588
2589	#elif defined(RT_ARCH_ARM64)
2590	if (uImm == 0)
2591	iGprImmTmp = ARMV8_A64_REG_XZR;
2592	else
2593	{
2594	Assert(iGprImmTmp < 31);
2595	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2596	}
2597	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2598	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2599
2600	#else
2601	# error "port me"
2602	#endif
2603	return off;
2604	}
2605
2606
2607	/**
2608	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2609	*
2610	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2611	*
2612	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2613	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2614	* same. Will assert / throw if caller does not heed this.
2615	*/
2616	DECL_FORCE_INLINE_THROW(uint32_t)
2617	iemNativeEmitStoreImm8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t uImm, uint8_t iGprBase,
2618	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2619	{
2620	#ifdef RT_ARCH_AMD64
2621	/* mov mem8, imm8 */
2622	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2623	if (iGprBase >= 8)
2624	pCodeBuf[off++] = X86_OP_REX_B;
2625	pCodeBuf[off++] = 0xc6;
2626	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2627	pCodeBuf[off++] = uImm;
2628	RT_NOREF(iGprImmTmp, iGprTmp);
2629
2630	#elif defined(RT_ARCH_ARM64)
2631	if (uImm == 0)
2632	iGprImmTmp = ARMV8_A64_REG_XZR;
2633	else
2634	{
2635	Assert(iGprImmTmp < 31);
2636	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2637	}
2638	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2639	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2640
2641	#else
2642	# error "port me"
2643	#endif
2644	return off;
2645	}
2646
2647
2648
2649	/*********************************************************************************************************************************
2650	* Subtraction and Additions *
2651	*********************************************************************************************************************************/
2652
2653	/**
2654	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
2655	* @note The AMD64 version sets flags.
2656	*/
2657	DECL_INLINE_THROW(uint32_t)
2658	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2659	{
2660	#if defined(RT_ARCH_AMD64)
2661	/* sub Gv,Ev */
2662	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2663	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2664	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2665	pbCodeBuf[off++] = 0x2b;
2666	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2667
2668	#elif defined(RT_ARCH_ARM64)
2669	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2670	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
2671
2672	#else
2673	# error "Port me"
2674	#endif
2675	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2676	return off;
2677	}
2678
2679
2680	/**
2681	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2682	* @note The AMD64 version sets flags.
2683	*/
2684	DECL_FORCE_INLINE(uint32_t)
2685	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2686	{
2687	#if defined(RT_ARCH_AMD64)
2688	/* sub Gv,Ev */
2689	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
2690	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
2691	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2692	pCodeBuf[off++] = 0x2b;
2693	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2694
2695	#elif defined(RT_ARCH_ARM64)
2696	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
2697
2698	#else
2699	# error "Port me"
2700	#endif
2701	return off;
2702	}
2703
2704
2705	/**
2706	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2707	* @note The AMD64 version sets flags.
2708	*/
2709	DECL_INLINE_THROW(uint32_t)
2710	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2711	{
2712	#if defined(RT_ARCH_AMD64)
2713	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
2714	#elif defined(RT_ARCH_ARM64)
2715	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
2716	#else
2717	# error "Port me"
2718	#endif
2719	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2720	return off;
2721	}
2722
2723
2724	/**
2725	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2726	*
2727	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2728	*
2729	* @note Larger constants will require a temporary register. Failing to specify
2730	* one when needed will trigger fatal assertion / throw.
2731	*/
2732	DECL_FORCE_INLINE_THROW(uint32_t)
2733	iemNativeEmitSubGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2734	uint8_t iGprTmp = UINT8_MAX)
2735	{
2736	#ifdef RT_ARCH_AMD64
2737	pCodeBuf[off++] = iGprDst >= 8 ? X86_OP_REX_W \| X86_OP_REX_B : X86_OP_REX_W;
2738	if (iSubtrahend == 1)
2739	{
2740	/* dec r/m64 */
2741	pCodeBuf[off++] = 0xff;
2742	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2743	}
2744	else if (iSubtrahend == -1)
2745	{
2746	/* inc r/m64 */
2747	pCodeBuf[off++] = 0xff;
2748	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2749	}
2750	else if ((int8_t)iSubtrahend == iSubtrahend)
2751	{
2752	/* sub r/m64, imm8 */
2753	pCodeBuf[off++] = 0x83;
2754	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2755	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2756	}
2757	else if ((int32_t)iSubtrahend == iSubtrahend)
2758	{
2759	/* sub r/m64, imm32 */
2760	pCodeBuf[off++] = 0x81;
2761	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2762	pCodeBuf[off++] = RT_BYTE1((uint64_t)iSubtrahend);
2763	pCodeBuf[off++] = RT_BYTE2((uint64_t)iSubtrahend);
2764	pCodeBuf[off++] = RT_BYTE3((uint64_t)iSubtrahend);
2765	pCodeBuf[off++] = RT_BYTE4((uint64_t)iSubtrahend);
2766	}
2767	else if (iGprTmp != UINT8_MAX)
2768	{
2769	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off - 1, iGprTmp, (uint64_t)iSubtrahend);
2770	/* sub r/m64, r64 */
2771	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B) \| (iGprTmp < 8 ? 0 : X86_OP_REX_R);
2772	pCodeBuf[off++] = 0x29;
2773	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprTmp & 7, iGprDst & 7);
2774	}
2775	else
2776	# ifdef IEM_WITH_THROW_CATCH
2777	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2778	# else
2779	AssertReleaseFailedStmt(off = UINT32_MAX);
2780	# endif
2781
2782	#elif defined(RT_ARCH_ARM64)
2783	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2784	if (uAbsSubtrahend < 4096)
2785	{
2786	if (iSubtrahend >= 0)
2787	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2788	else
2789	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2790	}
2791	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2792	{
2793	if (iSubtrahend >= 0)
2794	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2795	true /f64Bit/, false /fSetFlags/, true /fShift/);
2796	else
2797	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2798	true /f64Bit/, false /fSetFlags/, true /fShift/);
2799	}
2800	else if (iGprTmp != UINT8_MAX)
2801	{
2802	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (uint64_t)iSubtrahend);
2803	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp);
2804	}
2805	else
2806	# ifdef IEM_WITH_THROW_CATCH
2807	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2808	# else
2809	AssertReleaseFailedStmt(off = UINT32_MAX);
2810	# endif
2811
2812	#else
2813	# error "Port me"
2814	#endif
2815	return off;
2816	}
2817
2818
2819	/**
2820	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2821	*
2822	* @note Larger constants will require a temporary register. Failing to specify
2823	* one when needed will trigger fatal assertion / throw.
2824	*/
2825	DECL_INLINE_THROW(uint32_t)
2826	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2827	uint8_t iGprTmp = UINT8_MAX)
2828
2829	{
2830	#ifdef RT_ARCH_AMD64
2831	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 13), off, iGprDst, iSubtrahend, iGprTmp);
2832	#elif defined(RT_ARCH_ARM64)
2833	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGprDst, iSubtrahend, iGprTmp);
2834	#else
2835	# error "Port me"
2836	#endif
2837	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2838	return off;
2839	}
2840
2841
2842	/**
2843	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2844	*
2845	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2846	*
2847	* @note ARM64: Larger constants will require a temporary register. Failing to
2848	* specify one when needed will trigger fatal assertion / throw.
2849	*/
2850	DECL_FORCE_INLINE_THROW(uint32_t)
2851	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2852	uint8_t iGprTmp = UINT8_MAX)
2853	{
2854	#ifdef RT_ARCH_AMD64
2855	if (iGprDst >= 8)
2856	pCodeBuf[off++] = X86_OP_REX_B;
2857	if (iSubtrahend == 1)
2858	{
2859	/* dec r/m32 */
2860	pCodeBuf[off++] = 0xff;
2861	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2862	}
2863	else if (iSubtrahend == -1)
2864	{
2865	/* inc r/m32 */
2866	pCodeBuf[off++] = 0xff;
2867	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2868	}
2869	else if (iSubtrahend < 128 && iSubtrahend >= -128)
2870	{
2871	/* sub r/m32, imm8 */
2872	pCodeBuf[off++] = 0x83;
2873	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2874	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2875	}
2876	else
2877	{
2878	/* sub r/m32, imm32 */
2879	pCodeBuf[off++] = 0x81;
2880	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2881	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
2882	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
2883	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
2884	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
2885	}
2886	RT_NOREF(iGprTmp);
2887
2888	#elif defined(RT_ARCH_ARM64)
2889	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2890	if (uAbsSubtrahend < 4096)
2891	{
2892	if (iSubtrahend >= 0)
2893	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2894	else
2895	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2896	}
2897	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2898	{
2899	if (iSubtrahend >= 0)
2900	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2901	false /f64Bit/, false /fSetFlags/, true /fShift/);
2902	else
2903	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2904	false /f64Bit/, false /fSetFlags/, true /fShift/);
2905	}
2906	else if (iGprTmp != UINT8_MAX)
2907	{
2908	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2909	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2910	}
2911	else
2912	# ifdef IEM_WITH_THROW_CATCH
2913	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2914	# else
2915	AssertReleaseFailedStmt(off = UINT32_MAX);
2916	# endif
2917
2918	#else
2919	# error "Port me"
2920	#endif
2921	return off;
2922	}
2923
2924
2925	/**
2926	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2927	*
2928	* @note ARM64: Larger constants will require a temporary register. Failing to
2929	* specify one when needed will trigger fatal assertion / throw.
2930	*/
2931	DECL_INLINE_THROW(uint32_t)
2932	iemNativeEmitSubGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2933	uint8_t iGprTmp = UINT8_MAX)
2934
2935	{
2936	#ifdef RT_ARCH_AMD64
2937	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iSubtrahend, iGprTmp);
2938	#elif defined(RT_ARCH_ARM64)
2939	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iSubtrahend, iGprTmp);
2940	#else
2941	# error "Port me"
2942	#endif
2943	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2944	return off;
2945	}
2946
2947
2948	/**
2949	* Emits a 16-bit GPR subtract with a signed immediate subtrahend.
2950	*
2951	* This will optimize using DEC/INC/whatever and ARM64 will not set flags,
2952	* so not suitable as a base for conditional jumps.
2953	*
2954	* @note AMD64: Will only update the lower 16 bits of the register.
2955	* @note ARM64: Will update the entire register.
2956	* @note ARM64: Larger constants will require a temporary register. Failing to
2957	* specify one when needed will trigger fatal assertion / throw.
2958	*/
2959	DECL_FORCE_INLINE_THROW(uint32_t)
2960	iemNativeEmitSubGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iSubtrahend,
2961	uint8_t iGprTmp = UINT8_MAX)
2962	{
2963	#ifdef RT_ARCH_AMD64
2964	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2965	if (iGprDst >= 8)
2966	pCodeBuf[off++] = X86_OP_REX_B;
2967	if (iSubtrahend == 1)
2968	{
2969	/* dec r/m16 */
2970	pCodeBuf[off++] = 0xff;
2971	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2972	}
2973	else if (iSubtrahend == -1)
2974	{
2975	/* inc r/m16 */
2976	pCodeBuf[off++] = 0xff;
2977	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2978	}
2979	else if ((int8_t)iSubtrahend == iSubtrahend)
2980	{
2981	/* sub r/m16, imm8 */
2982	pCodeBuf[off++] = 0x83;
2983	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2984	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2985	}
2986	else
2987	{
2988	/* sub r/m16, imm16 */
2989	pCodeBuf[off++] = 0x81;
2990	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2991	pCodeBuf[off++] = RT_BYTE1((uint16_t)iSubtrahend);
2992	pCodeBuf[off++] = RT_BYTE2((uint16_t)iSubtrahend);
2993	}
2994	RT_NOREF(iGprTmp);
2995
2996	#elif defined(RT_ARCH_ARM64)
2997	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2998	if (uAbsSubtrahend < 4096)
2999	{
3000	if (iSubtrahend >= 0)
3001	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
3002	else
3003	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
3004	}
3005	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
3006	{
3007	if (iSubtrahend >= 0)
3008	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
3009	false /f64Bit/, false /fSetFlags/, true /fShift/);
3010	else
3011	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
3012	false /f64Bit/, false /fSetFlags/, true /fShift/);
3013	}
3014	else if (iGprTmp != UINT8_MAX)
3015	{
3016	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
3017	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
3018	}
3019	else
3020	# ifdef IEM_WITH_THROW_CATCH
3021	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3022	# else
3023	AssertReleaseFailedStmt(off = UINT32_MAX);
3024	# endif
3025	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
3026
3027	#else
3028	# error "Port me"
3029	#endif
3030	return off;
3031	}
3032
3033
3034	/**
3035	* Emits adding a 64-bit GPR to another, storing the result in the first.
3036	* @note The AMD64 version sets flags.
3037	*/
3038	DECL_FORCE_INLINE(uint32_t)
3039	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3040	{
3041	#if defined(RT_ARCH_AMD64)
3042	/* add Gv,Ev */
3043	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3044	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
3045	pCodeBuf[off++] = 0x03;
3046	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
3047
3048	#elif defined(RT_ARCH_ARM64)
3049	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
3050
3051	#else
3052	# error "Port me"
3053	#endif
3054	return off;
3055	}
3056
3057
3058	/**
3059	* Emits adding a 64-bit GPR to another, storing the result in the first.
3060	* @note The AMD64 version sets flags.
3061	*/
3062	DECL_INLINE_THROW(uint32_t)
3063	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3064	{
3065	#if defined(RT_ARCH_AMD64)
3066	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
3067	#elif defined(RT_ARCH_ARM64)
3068	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
3069	#else
3070	# error "Port me"
3071	#endif
3072	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3073	return off;
3074	}
3075
3076
3077	/**
3078	* Emits adding a 64-bit GPR to another, storing the result in the first.
3079	* @note The AMD64 version sets flags.
3080	*/
3081	DECL_FORCE_INLINE(uint32_t)
3082	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3083	{
3084	#if defined(RT_ARCH_AMD64)
3085	/* add Gv,Ev */
3086	if (iGprDst >= 8 \|\| iGprAddend >= 8)
3087	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
3088	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
3089	pCodeBuf[off++] = 0x03;
3090	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
3091
3092	#elif defined(RT_ARCH_ARM64)
3093	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
3094
3095	#else
3096	# error "Port me"
3097	#endif
3098	return off;
3099	}
3100
3101
3102	/**
3103	* Emits adding a 64-bit GPR to another, storing the result in the first.
3104	* @note The AMD64 version sets flags.
3105	*/
3106	DECL_INLINE_THROW(uint32_t)
3107	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3108	{
3109	#if defined(RT_ARCH_AMD64)
3110	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
3111	#elif defined(RT_ARCH_ARM64)
3112	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
3113	#else
3114	# error "Port me"
3115	#endif
3116	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3117	return off;
3118	}
3119
3120
3121	/**
3122	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3123	*/
3124	DECL_INLINE_THROW(uint32_t)
3125	iemNativeEmitAddGprImm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3126	{
3127	#if defined(RT_ARCH_AMD64)
3128	/* add or inc */
3129	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3130	if (iImm8 != 1)
3131	{
3132	pCodeBuf[off++] = 0x83;
3133	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3134	pCodeBuf[off++] = (uint8_t)iImm8;
3135	}
3136	else
3137	{
3138	pCodeBuf[off++] = 0xff;
3139	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3140	}
3141
3142	#elif defined(RT_ARCH_ARM64)
3143	if (iImm8 >= 0)
3144	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint8_t)iImm8);
3145	else
3146	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint8_t)-iImm8);
3147
3148	#else
3149	# error "Port me"
3150	#endif
3151	return off;
3152	}
3153
3154
3155	/**
3156	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3157	*/
3158	DECL_INLINE_THROW(uint32_t)
3159	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3160	{
3161	#if defined(RT_ARCH_AMD64)
3162	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3163	#elif defined(RT_ARCH_ARM64)
3164	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3165	#else
3166	# error "Port me"
3167	#endif
3168	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3169	return off;
3170	}
3171
3172
3173	/**
3174	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3175	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3176	*/
3177	DECL_FORCE_INLINE(uint32_t)
3178	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3179	{
3180	#if defined(RT_ARCH_AMD64)
3181	/* add or inc */
3182	if (iGprDst >= 8)
3183	pCodeBuf[off++] = X86_OP_REX_B;
3184	if (iImm8 != 1)
3185	{
3186	pCodeBuf[off++] = 0x83;
3187	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3188	pCodeBuf[off++] = (uint8_t)iImm8;
3189	}
3190	else
3191	{
3192	pCodeBuf[off++] = 0xff;
3193	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3194	}
3195
3196	#elif defined(RT_ARCH_ARM64)
3197	if (iImm8 >= 0)
3198	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
3199	else
3200	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
3201
3202	#else
3203	# error "Port me"
3204	#endif
3205	return off;
3206	}
3207
3208
3209	/**
3210	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3211	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3212	*/
3213	DECL_INLINE_THROW(uint32_t)
3214	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3215	{
3216	#if defined(RT_ARCH_AMD64)
3217	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3218	#elif defined(RT_ARCH_ARM64)
3219	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3220	#else
3221	# error "Port me"
3222	#endif
3223	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3224	return off;
3225	}
3226
3227
3228	/**
3229	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3230	*
3231	* @note Will assert / throw if @a iGprTmp is not specified when needed.
3232	*/
3233	DECL_FORCE_INLINE_THROW(uint32_t)
3234	iemNativeEmitAddGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iAddend, uint8_t iGprTmp = UINT8_MAX)
3235	{
3236	#if defined(RT_ARCH_AMD64)
3237	if ((int8_t)iAddend == iAddend)
3238	return iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3239
3240	if ((int32_t)iAddend == iAddend)
3241	{
3242	/* add grp, imm32 */
3243	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3244	pCodeBuf[off++] = 0x81;
3245	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3246	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3247	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3248	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3249	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3250	}
3251	else if (iGprTmp != UINT8_MAX)
3252	{
3253	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3254
3255	/* add dst, tmpreg */
3256	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3257	\| (iGprTmp < 8 ? 0 : X86_OP_REX_B);
3258	pCodeBuf[off++] = 0x03;
3259	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprTmp & 7);
3260	}
3261	else
3262	# ifdef IEM_WITH_THROW_CATCH
3263	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3264	# else
3265	AssertReleaseFailedStmt(off = UINT32_MAX);
3266	# endif
3267
3268	#elif defined(RT_ARCH_ARM64)
3269	uint64_t const uAbsAddend = (uint64_t)RT_ABS(iAddend);
3270	if (uAbsAddend < 4096)
3271	{
3272	if (iAddend >= 0)
3273	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3274	else
3275	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3276	}
3277	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3278	{
3279	if (iAddend >= 0)
3280	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3281	true /f64Bit/, true /fShift12/);
3282	else
3283	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3284	true /f64Bit/, true /fShift12/);
3285	}
3286	else if (iGprTmp != UINT8_MAX)
3287	{
3288	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3289	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp);
3290	}
3291	else
3292	# ifdef IEM_WITH_THROW_CATCH
3293	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3294	# else
3295	AssertReleaseFailedStmt(off = UINT32_MAX);
3296	# endif
3297
3298	#else
3299	# error "Port me"
3300	#endif
3301	return off;
3302	}
3303
3304
3305	/**
3306	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3307	*/
3308	DECL_INLINE_THROW(uint32_t)
3309	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
3310	{
3311	#if defined(RT_ARCH_AMD64)
3312	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3313	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
3314
3315	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
3316	{
3317	/* add grp, imm32 */
3318	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3319	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3320	pbCodeBuf[off++] = 0x81;
3321	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3322	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3323	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3324	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3325	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3326	}
3327	else
3328	{
3329	/* Best to use a temporary register to deal with this in the simplest way: */
3330	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3331
3332	/* add dst, tmpreg */
3333	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3334	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3335	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3336	pbCodeBuf[off++] = 0x03;
3337	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
3338
3339	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3340	}
3341
3342	#elif defined(RT_ARCH_ARM64)
3343	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3344	{
3345	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3346	if (iAddend >= 0)
3347	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
3348	else
3349	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
3350	}
3351	else
3352	{
3353	/* Use temporary register for the immediate. */
3354	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3355
3356	/* add gprdst, gprdst, tmpreg */
3357	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3358	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
3359
3360	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3361	}
3362
3363	#else
3364	# error "Port me"
3365	#endif
3366	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3367	return off;
3368	}
3369
3370
3371	/**
3372	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3373	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3374	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
3375	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
3376	* the lower 12 bits always zero). The negative ranges are also allowed,
3377	* making it behave like a subtraction. If the constant does not conform,
3378	* bad stuff will happen.
3379	*/
3380	DECL_FORCE_INLINE_THROW(uint32_t)
3381	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3382	{
3383	#if defined(RT_ARCH_AMD64)
3384	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3385	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3386
3387	/* add grp, imm32 */
3388	if (iGprDst >= 8)
3389	pCodeBuf[off++] = X86_OP_REX_B;
3390	pCodeBuf[off++] = 0x81;
3391	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3392	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3393	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3394	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3395	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3396
3397	#elif defined(RT_ARCH_ARM64)
3398	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
3399	if (uAbsAddend <= 0xfff)
3400	{
3401	if (iAddend >= 0)
3402	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3403	else
3404	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3405	}
3406	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3407	{
3408	if (iAddend >= 0)
3409	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3410	false /f64Bit/, true /fShift12/);
3411	else
3412	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3413	false /f64Bit/, true /fShift12/);
3414	}
3415	else
3416	# ifdef IEM_WITH_THROW_CATCH
3417	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3418	# else
3419	AssertReleaseFailedStmt(off = UINT32_MAX);
3420	# endif
3421
3422	#else
3423	# error "Port me"
3424	#endif
3425	return off;
3426	}
3427
3428
3429	/**
3430	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3431	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3432	*/
3433	DECL_INLINE_THROW(uint32_t)
3434	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3435	{
3436	#if defined(RT_ARCH_AMD64)
3437	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
3438
3439	#elif defined(RT_ARCH_ARM64)
3440	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3441	{
3442	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3443	if (iAddend >= 0)
3444	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
3445	else
3446	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
3447	}
3448	else
3449	{
3450	/* Use temporary register for the immediate. */
3451	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
3452
3453	/* add gprdst, gprdst, tmpreg */
3454	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3455	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
3456
3457	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3458	}
3459
3460	#else
3461	# error "Port me"
3462	#endif
3463	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3464	return off;
3465	}
3466
3467
3468	/**
3469	* Emits a 16-bit GPR add with a signed immediate addend.
3470	*
3471	* This will optimize using INC/DEC/whatever and ARM64 will not set flags,
3472	* so not suitable as a base for conditional jumps.
3473	*
3474	* @note AMD64: Will only update the lower 16 bits of the register.
3475	* @note ARM64: Will update the entire register.
3476	* @note ARM64: Larger constants will require a temporary register. Failing to
3477	* specify one when needed will trigger fatal assertion / throw.
3478	* @sa iemNativeEmitSubGpr16ImmEx
3479	*/
3480	DECL_FORCE_INLINE_THROW(uint32_t)
3481	iemNativeEmitAddGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iAddend,
3482	uint8_t iGprTmp = UINT8_MAX)
3483	{
3484	#ifdef RT_ARCH_AMD64
3485	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
3486	if (iGprDst >= 8)
3487	pCodeBuf[off++] = X86_OP_REX_B;
3488	if (iAddend == 1)
3489	{
3490	/* inc r/m16 */
3491	pCodeBuf[off++] = 0xff;
3492	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3493	}
3494	else if (iAddend == -1)
3495	{
3496	/* dec r/m16 */
3497	pCodeBuf[off++] = 0xff;
3498	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
3499	}
3500	else if ((int8_t)iAddend == iAddend)
3501	{
3502	/* add r/m16, imm8 */
3503	pCodeBuf[off++] = 0x83;
3504	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3505	pCodeBuf[off++] = (uint8_t)iAddend;
3506	}
3507	else
3508	{
3509	/* add r/m16, imm16 */
3510	pCodeBuf[off++] = 0x81;
3511	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3512	pCodeBuf[off++] = RT_BYTE1((uint16_t)iAddend);
3513	pCodeBuf[off++] = RT_BYTE2((uint16_t)iAddend);
3514	}
3515	RT_NOREF(iGprTmp);
3516
3517	#elif defined(RT_ARCH_ARM64)
3518	uint32_t uAbsAddend = RT_ABS(iAddend);
3519	if (uAbsAddend < 4096)
3520	{
3521	if (iAddend >= 0)
3522	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3523	else
3524	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3525	}
3526	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3527	{
3528	if (iAddend >= 0)
3529	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3530	false /f64Bit/, false /fSetFlags/, true /fShift/);
3531	else
3532	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3533	false /f64Bit/, false /fSetFlags/, true /fShift/);
3534	}
3535	else if (iGprTmp != UINT8_MAX)
3536	{
3537	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iAddend);
3538	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
3539	}
3540	else
3541	# ifdef IEM_WITH_THROW_CATCH
3542	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3543	# else
3544	AssertReleaseFailedStmt(off = UINT32_MAX);
3545	# endif
3546	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
3547
3548	#else
3549	# error "Port me"
3550	#endif
3551	return off;
3552	}
3553
3554
3555
3556	/**
3557	* Adds two 64-bit GPRs together, storing the result in a third register.
3558	*/
3559	DECL_FORCE_INLINE(uint32_t)
3560	iemNativeEmitGprEqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3561	{
3562	#ifdef RT_ARCH_AMD64
3563	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3564	{
3565	/** @todo consider LEA */
3566	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend1);
3567	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend2);
3568	}
3569	else
3570	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3571
3572	#elif defined(RT_ARCH_ARM64)
3573	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2);
3574
3575	#else
3576	# error "Port me!"
3577	#endif
3578	return off;
3579	}
3580
3581
3582
3583	/**
3584	* Adds two 32-bit GPRs together, storing the result in a third register.
3585	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3586	*/
3587	DECL_FORCE_INLINE(uint32_t)
3588	iemNativeEmitGpr32EqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3589	{
3590	#ifdef RT_ARCH_AMD64
3591	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3592	{
3593	/** @todo consider LEA */
3594	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend1);
3595	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend2);
3596	}
3597	else
3598	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3599
3600	#elif defined(RT_ARCH_ARM64)
3601	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2, false /f64Bit/);
3602
3603	#else
3604	# error "Port me!"
3605	#endif
3606	return off;
3607	}
3608
3609
3610	/**
3611	* Adds a 64-bit GPR and a 64-bit unsigned constant, storing the result in a
3612	* third register.
3613	*
3614	* @note The ARM64 version does not work for non-trivial constants if the
3615	* two registers are the same. Will assert / throw exception.
3616	*/
3617	DECL_FORCE_INLINE_THROW(uint32_t)
3618	iemNativeEmitGprEqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int64_t iImmAddend)
3619	{
3620	#ifdef RT_ARCH_AMD64
3621	/** @todo consider LEA */
3622	if ((int8_t)iImmAddend == iImmAddend)
3623	{
3624	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend);
3625	off = iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3626	}
3627	else
3628	{
3629	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3630	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3631	}
3632
3633	#elif defined(RT_ARCH_ARM64)
3634	uint64_t const uAbsImmAddend = RT_ABS(iImmAddend);
3635	if (uAbsImmAddend < 4096)
3636	{
3637	if (iImmAddend >= 0)
3638	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3639	else
3640	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3641	}
3642	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3643	{
3644	if (iImmAddend >= 0)
3645	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3646	else
3647	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3648	}
3649	else if (iGprDst != iGprAddend)
3650	{
3651	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, (uint64_t)iImmAddend);
3652	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3653	}
3654	else
3655	# ifdef IEM_WITH_THROW_CATCH
3656	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3657	# else
3658	AssertReleaseFailedStmt(off = UINT32_MAX);
3659	# endif
3660
3661	#else
3662	# error "Port me!"
3663	#endif
3664	return off;
3665	}
3666
3667
3668	/**
3669	* Adds a 32-bit GPR and a 32-bit unsigned constant, storing the result in a
3670	* third register.
3671	*
3672	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3673	*
3674	* @note The ARM64 version does not work for non-trivial constants if the
3675	* two registers are the same. Will assert / throw exception.
3676	*/
3677	DECL_FORCE_INLINE_THROW(uint32_t)
3678	iemNativeEmitGpr32EqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
3679	{
3680	#ifdef RT_ARCH_AMD64
3681	/** @todo consider LEA */
3682	if ((int8_t)iImmAddend == iImmAddend)
3683	{
3684	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3685	off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3686	}
3687	else
3688	{
3689	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3690	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3691	}
3692
3693	#elif defined(RT_ARCH_ARM64)
3694	uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
3695	if (uAbsImmAddend < 4096)
3696	{
3697	if (iImmAddend >= 0)
3698	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3699	else
3700	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3701	}
3702	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3703	{
3704	if (iImmAddend >= 0)
3705	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3706	else
3707	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3708	}
3709	else if (iGprDst != iGprAddend)
3710	{
3711	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
3712	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3713	}
3714	else
3715	# ifdef IEM_WITH_THROW_CATCH
3716	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3717	# else
3718	AssertReleaseFailedStmt(off = UINT32_MAX);
3719	# endif
3720
3721	#else
3722	# error "Port me!"
3723	#endif
3724	return off;
3725	}
3726
3727
3728	/*********************************************************************************************************************************
3729	* Unary Operations *
3730	*********************************************************************************************************************************/
3731
3732	/**
3733	* Emits code for two complement negation of a 64-bit GPR.
3734	*/
3735	DECL_FORCE_INLINE_THROW(uint32_t)
3736	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3737	{
3738	#if defined(RT_ARCH_AMD64)
3739	/* neg Ev */
3740	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3741	pCodeBuf[off++] = 0xf7;
3742	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3743
3744	#elif defined(RT_ARCH_ARM64)
3745	/* sub dst, xzr, dst */
3746	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
3747
3748	#else
3749	# error "Port me"
3750	#endif
3751	return off;
3752	}
3753
3754
3755	/**
3756	* Emits code for two complement negation of a 64-bit GPR.
3757	*/
3758	DECL_INLINE_THROW(uint32_t)
3759	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3760	{
3761	#if defined(RT_ARCH_AMD64)
3762	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3763	#elif defined(RT_ARCH_ARM64)
3764	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3765	#else
3766	# error "Port me"
3767	#endif
3768	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3769	return off;
3770	}
3771
3772
3773	/**
3774	* Emits code for two complement negation of a 32-bit GPR.
3775	* @note bit 32 thru 63 are set to zero.
3776	*/
3777	DECL_FORCE_INLINE_THROW(uint32_t)
3778	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3779	{
3780	#if defined(RT_ARCH_AMD64)
3781	/* neg Ev */
3782	if (iGprDst >= 8)
3783	pCodeBuf[off++] = X86_OP_REX_B;
3784	pCodeBuf[off++] = 0xf7;
3785	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3786
3787	#elif defined(RT_ARCH_ARM64)
3788	/* sub dst, xzr, dst */
3789	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
3790
3791	#else
3792	# error "Port me"
3793	#endif
3794	return off;
3795	}
3796
3797
3798	/**
3799	* Emits code for two complement negation of a 32-bit GPR.
3800	* @note bit 32 thru 63 are set to zero.
3801	*/
3802	DECL_INLINE_THROW(uint32_t)
3803	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3804	{
3805	#if defined(RT_ARCH_AMD64)
3806	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3807	#elif defined(RT_ARCH_ARM64)
3808	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3809	#else
3810	# error "Port me"
3811	#endif
3812	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3813	return off;
3814	}
3815
3816
3817
3818	/*********************************************************************************************************************************
3819	* Bit Operations *
3820	*********************************************************************************************************************************/
3821
3822	/**
3823	* Emits code for clearing bits 16 thru 63 in the GPR.
3824	*/
3825	DECL_INLINE_THROW(uint32_t)
3826	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3827	{
3828	#if defined(RT_ARCH_AMD64)
3829	/* movzx Gv,Ew */
3830	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3831	if (iGprDst >= 8)
3832	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
3833	pbCodeBuf[off++] = 0x0f;
3834	pbCodeBuf[off++] = 0xb7;
3835	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
3836
3837	#elif defined(RT_ARCH_ARM64)
3838	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3839	# if 1
3840	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
3841	# else
3842	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
3843	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
3844	# endif
3845	#else
3846	# error "Port me"
3847	#endif
3848	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3849	return off;
3850	}
3851
3852
3853	/**
3854	* Emits code for AND'ing two 64-bit GPRs.
3855	*
3856	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3857	* and ARM64 hosts.
3858	*/
3859	DECL_INLINE_THROW(uint32_t)
3860	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3861	{
3862	#if defined(RT_ARCH_AMD64)
3863	/* and Gv, Ev */
3864	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3865	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3866	pbCodeBuf[off++] = 0x23;
3867	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3868	RT_NOREF(fSetFlags);
3869
3870	#elif defined(RT_ARCH_ARM64)
3871	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3872	if (!fSetFlags)
3873	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
3874	else
3875	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
3876
3877	#else
3878	# error "Port me"
3879	#endif
3880	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3881	return off;
3882	}
3883
3884
3885	/**
3886	* Emits code for AND'ing two 32-bit GPRs.
3887	*/
3888	DECL_FORCE_INLINE(uint32_t)
3889	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3890	{
3891	#if defined(RT_ARCH_AMD64)
3892	/* and Gv, Ev */
3893	if (iGprDst >= 8 \|\| iGprSrc >= 8)
3894	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3895	pCodeBuf[off++] = 0x23;
3896	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3897	RT_NOREF(fSetFlags);
3898
3899	#elif defined(RT_ARCH_ARM64)
3900	if (!fSetFlags)
3901	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3902	else
3903	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3904
3905	#else
3906	# error "Port me"
3907	#endif
3908	return off;
3909	}
3910
3911
3912	/**
3913	* Emits code for AND'ing two 32-bit GPRs.
3914	*/
3915	DECL_INLINE_THROW(uint32_t)
3916	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3917	{
3918	#if defined(RT_ARCH_AMD64)
3919	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
3920	#elif defined(RT_ARCH_ARM64)
3921	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
3922	#else
3923	# error "Port me"
3924	#endif
3925	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3926	return off;
3927	}
3928
3929
3930	/**
3931	* Emits code for AND'ing a 64-bit GPRs with a constant.
3932	*
3933	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3934	* and ARM64 hosts.
3935	*/
3936	DECL_INLINE_THROW(uint32_t)
3937	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
3938	{
3939	#if defined(RT_ARCH_AMD64)
3940	if ((int64_t)uImm == (int8_t)uImm)
3941	{
3942	/* and Ev, imm8 */
3943	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3944	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3945	pbCodeBuf[off++] = 0x83;
3946	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3947	pbCodeBuf[off++] = (uint8_t)uImm;
3948	}
3949	else if ((int64_t)uImm == (int32_t)uImm)
3950	{
3951	/* and Ev, imm32 */
3952	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3953	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3954	pbCodeBuf[off++] = 0x81;
3955	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3956	pbCodeBuf[off++] = RT_BYTE1(uImm);
3957	pbCodeBuf[off++] = RT_BYTE2(uImm);
3958	pbCodeBuf[off++] = RT_BYTE3(uImm);
3959	pbCodeBuf[off++] = RT_BYTE4(uImm);
3960	}
3961	else
3962	{
3963	/* Use temporary register for the 64-bit immediate. */
3964	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3965	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
3966	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3967	}
3968	RT_NOREF(fSetFlags);
3969
3970	#elif defined(RT_ARCH_ARM64)
3971	uint32_t uImmR = 0;
3972	uint32_t uImmNandS = 0;
3973	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
3974	{
3975	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3976	if (!fSetFlags)
3977	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
3978	else
3979	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
3980	}
3981	else
3982	{
3983	/* Use temporary register for the 64-bit immediate. */
3984	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3985	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
3986	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3987	}
3988
3989	#else
3990	# error "Port me"
3991	#endif
3992	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3993	return off;
3994	}
3995
3996
3997	/**
3998	* Emits code for AND'ing an 32-bit GPRs with a constant.
3999	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4000	* @note For ARM64 this only supports @a uImm values that can be expressed using
4001	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
4002	* make sure this is possible!
4003	*/
4004	DECL_FORCE_INLINE_THROW(uint32_t)
4005	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
4006	{
4007	#if defined(RT_ARCH_AMD64)
4008	/* and Ev, imm */
4009	if (iGprDst >= 8)
4010	pCodeBuf[off++] = X86_OP_REX_B;
4011	if ((int32_t)uImm == (int8_t)uImm)
4012	{
4013	pCodeBuf[off++] = 0x83;
4014	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4015	pCodeBuf[off++] = (uint8_t)uImm;
4016	}
4017	else
4018	{
4019	pCodeBuf[off++] = 0x81;
4020	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4021	pCodeBuf[off++] = RT_BYTE1(uImm);
4022	pCodeBuf[off++] = RT_BYTE2(uImm);
4023	pCodeBuf[off++] = RT_BYTE3(uImm);
4024	pCodeBuf[off++] = RT_BYTE4(uImm);
4025	}
4026	RT_NOREF(fSetFlags);
4027
4028	#elif defined(RT_ARCH_ARM64)
4029	uint32_t uImmR = 0;
4030	uint32_t uImmNandS = 0;
4031	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4032	{
4033	if (!fSetFlags)
4034	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4035	else
4036	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4037	}
4038	else
4039	# ifdef IEM_WITH_THROW_CATCH
4040	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4041	# else
4042	AssertReleaseFailedStmt(off = UINT32_MAX);
4043	# endif
4044
4045	#else
4046	# error "Port me"
4047	#endif
4048	return off;
4049	}
4050
4051
4052	/**
4053	* Emits code for AND'ing an 32-bit GPRs with a constant.
4054	*
4055	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4056	*/
4057	DECL_INLINE_THROW(uint32_t)
4058	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
4059	{
4060	#if defined(RT_ARCH_AMD64)
4061	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
4062
4063	#elif defined(RT_ARCH_ARM64)
4064	uint32_t uImmR = 0;
4065	uint32_t uImmNandS = 0;
4066	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4067	{
4068	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4069	if (!fSetFlags)
4070	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4071	else
4072	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4073	}
4074	else
4075	{
4076	/* Use temporary register for the 64-bit immediate. */
4077	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4078	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
4079	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4080	}
4081
4082	#else
4083	# error "Port me"
4084	#endif
4085	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4086	return off;
4087	}
4088
4089
4090	/**
4091	* Emits code for AND'ing an 32-bit GPRs with a constant.
4092	*
4093	* @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
4094	* encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
4095	* the same.
4096	*
4097	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4098	*/
4099	DECL_FORCE_INLINE_THROW(uint32_t)
4100	iemNativeEmitGpr32EqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint32_t uImm,
4101	bool fSetFlags = false)
4102	{
4103	#if defined(RT_ARCH_AMD64)
4104	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4105	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4106	RT_NOREF(fSetFlags);
4107
4108	#elif defined(RT_ARCH_ARM64)
4109	uint32_t uImmR = 0;
4110	uint32_t uImmNandS = 0;
4111	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4112	{
4113	if (!fSetFlags)
4114	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4115	else
4116	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4117	}
4118	else if (iGprDst != iGprSrc)
4119	{
4120	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4121	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
4122	}
4123	else
4124	# ifdef IEM_WITH_THROW_CATCH
4125	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4126	# else
4127	AssertReleaseFailedStmt(off = UINT32_MAX);
4128	# endif
4129
4130	#else
4131	# error "Port me"
4132	#endif
4133	return off;
4134	}
4135
4136
4137	/**
4138	* Emits code for OR'ing two 64-bit GPRs.
4139	*/
4140	DECL_FORCE_INLINE(uint32_t)
4141	iemNativeEmitOrGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4142	{
4143	#if defined(RT_ARCH_AMD64)
4144	/* or Gv, Ev */
4145	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4146	pCodeBuf[off++] = 0x0b;
4147	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4148
4149	#elif defined(RT_ARCH_ARM64)
4150	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc);
4151
4152	#else
4153	# error "Port me"
4154	#endif
4155	return off;
4156	}
4157
4158
4159	/**
4160	* Emits code for OR'ing two 32-bit GPRs.
4161	* @note Bits 63:32 of the destination GPR will be cleared.
4162	*/
4163	DECL_FORCE_INLINE(uint32_t)
4164	iemNativeEmitOrGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4165	{
4166	#if defined(RT_ARCH_AMD64)
4167	/* or Gv, Ev */
4168	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4169	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4170	pCodeBuf[off++] = 0x0b;
4171	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4172
4173	#elif defined(RT_ARCH_ARM64)
4174	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4175
4176	#else
4177	# error "Port me"
4178	#endif
4179	return off;
4180	}
4181
4182
4183	/**
4184	* Emits code for OR'ing two 32-bit GPRs.
4185	* @note Bits 63:32 of the destination GPR will be cleared.
4186	*/
4187	DECL_INLINE_THROW(uint32_t)
4188	iemNativeEmitOrGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4189	{
4190	#if defined(RT_ARCH_AMD64)
4191	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
4192	#elif defined(RT_ARCH_ARM64)
4193	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
4194	#else
4195	# error "Port me"
4196	#endif
4197	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4198	return off;
4199	}
4200
4201
4202
4203	/**
4204	* Emits code for XOR'ing two 64-bit GPRs.
4205	*/
4206	DECL_INLINE_THROW(uint32_t)
4207	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4208	{
4209	#if defined(RT_ARCH_AMD64)
4210	/* and Gv, Ev */
4211	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4212	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4213	pbCodeBuf[off++] = 0x33;
4214	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4215
4216	#elif defined(RT_ARCH_ARM64)
4217	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4218	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
4219
4220	#else
4221	# error "Port me"
4222	#endif
4223	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4224	return off;
4225	}
4226
4227
4228	/**
4229	* Emits code for XOR'ing two 32-bit GPRs.
4230	*/
4231	DECL_INLINE_THROW(uint32_t)
4232	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4233	{
4234	#if defined(RT_ARCH_AMD64)
4235	/* and Gv, Ev */
4236	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4237	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4238	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4239	pbCodeBuf[off++] = 0x33;
4240	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4241
4242	#elif defined(RT_ARCH_ARM64)
4243	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4244	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4245
4246	#else
4247	# error "Port me"
4248	#endif
4249	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4250	return off;
4251	}
4252
4253
4254	/**
4255	* Emits code for XOR'ing an 32-bit GPRs with a constant.
4256	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4257	* @note For ARM64 this only supports @a uImm values that can be expressed using
4258	* the two 6-bit immediates of the EOR instructions. The caller must make
4259	* sure this is possible!
4260	*/
4261	DECL_FORCE_INLINE_THROW(uint32_t)
4262	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
4263	{
4264	#if defined(RT_ARCH_AMD64)
4265	/* and Ev, imm */
4266	if (iGprDst >= 8)
4267	pCodeBuf[off++] = X86_OP_REX_B;
4268	if ((int32_t)uImm == (int8_t)uImm)
4269	{
4270	pCodeBuf[off++] = 0x83;
4271	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4272	pCodeBuf[off++] = (uint8_t)uImm;
4273	}
4274	else
4275	{
4276	pCodeBuf[off++] = 0x81;
4277	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4278	pCodeBuf[off++] = RT_BYTE1(uImm);
4279	pCodeBuf[off++] = RT_BYTE2(uImm);
4280	pCodeBuf[off++] = RT_BYTE3(uImm);
4281	pCodeBuf[off++] = RT_BYTE4(uImm);
4282	}
4283
4284	#elif defined(RT_ARCH_ARM64)
4285	uint32_t uImmR = 0;
4286	uint32_t uImmNandS = 0;
4287	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4288	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4289	else
4290	# ifdef IEM_WITH_THROW_CATCH
4291	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4292	# else
4293	AssertReleaseFailedStmt(off = UINT32_MAX);
4294	# endif
4295
4296	#else
4297	# error "Port me"
4298	#endif
4299	return off;
4300	}
4301
4302
4303	/*********************************************************************************************************************************
4304	* Shifting *
4305	*********************************************************************************************************************************/
4306
4307	/**
4308	* Emits code for shifting a GPR a fixed number of bits to the left.
4309	*/
4310	DECL_FORCE_INLINE(uint32_t)
4311	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4312	{
4313	Assert(cShift > 0 && cShift < 64);
4314
4315	#if defined(RT_ARCH_AMD64)
4316	/* shl dst, cShift */
4317	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4318	if (cShift != 1)
4319	{
4320	pCodeBuf[off++] = 0xc1;
4321	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4322	pCodeBuf[off++] = cShift;
4323	}
4324	else
4325	{
4326	pCodeBuf[off++] = 0xd1;
4327	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4328	}
4329
4330	#elif defined(RT_ARCH_ARM64)
4331	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
4332
4333	#else
4334	# error "Port me"
4335	#endif
4336	return off;
4337	}
4338
4339
4340	/**
4341	* Emits code for shifting a GPR a fixed number of bits to the left.
4342	*/
4343	DECL_INLINE_THROW(uint32_t)
4344	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4345	{
4346	#if defined(RT_ARCH_AMD64)
4347	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4348	#elif defined(RT_ARCH_ARM64)
4349	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4350	#else
4351	# error "Port me"
4352	#endif
4353	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4354	return off;
4355	}
4356
4357
4358	/**
4359	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4360	*/
4361	DECL_FORCE_INLINE(uint32_t)
4362	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4363	{
4364	Assert(cShift > 0 && cShift < 32);
4365
4366	#if defined(RT_ARCH_AMD64)
4367	/* shl dst, cShift */
4368	if (iGprDst >= 8)
4369	pCodeBuf[off++] = X86_OP_REX_B;
4370	if (cShift != 1)
4371	{
4372	pCodeBuf[off++] = 0xc1;
4373	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4374	pCodeBuf[off++] = cShift;
4375	}
4376	else
4377	{
4378	pCodeBuf[off++] = 0xd1;
4379	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4380	}
4381
4382	#elif defined(RT_ARCH_ARM64)
4383	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
4384
4385	#else
4386	# error "Port me"
4387	#endif
4388	return off;
4389	}
4390
4391
4392	/**
4393	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4394	*/
4395	DECL_INLINE_THROW(uint32_t)
4396	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4397	{
4398	#if defined(RT_ARCH_AMD64)
4399	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4400	#elif defined(RT_ARCH_ARM64)
4401	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4402	#else
4403	# error "Port me"
4404	#endif
4405	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4406	return off;
4407	}
4408
4409
4410	/**
4411	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4412	*/
4413	DECL_FORCE_INLINE(uint32_t)
4414	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4415	{
4416	Assert(cShift > 0 && cShift < 64);
4417
4418	#if defined(RT_ARCH_AMD64)
4419	/* shr dst, cShift */
4420	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4421	if (cShift != 1)
4422	{
4423	pCodeBuf[off++] = 0xc1;
4424	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4425	pCodeBuf[off++] = cShift;
4426	}
4427	else
4428	{
4429	pCodeBuf[off++] = 0xd1;
4430	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4431	}
4432
4433	#elif defined(RT_ARCH_ARM64)
4434	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
4435
4436	#else
4437	# error "Port me"
4438	#endif
4439	return off;
4440	}
4441
4442
4443	/**
4444	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4445	*/
4446	DECL_INLINE_THROW(uint32_t)
4447	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4448	{
4449	#if defined(RT_ARCH_AMD64)
4450	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4451	#elif defined(RT_ARCH_ARM64)
4452	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4453	#else
4454	# error "Port me"
4455	#endif
4456	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4457	return off;
4458	}
4459
4460
4461	/**
4462	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4463	* right.
4464	*/
4465	DECL_FORCE_INLINE(uint32_t)
4466	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4467	{
4468	Assert(cShift > 0 && cShift < 32);
4469
4470	#if defined(RT_ARCH_AMD64)
4471	/* shr dst, cShift */
4472	if (iGprDst >= 8)
4473	pCodeBuf[off++] = X86_OP_REX_B;
4474	if (cShift != 1)
4475	{
4476	pCodeBuf[off++] = 0xc1;
4477	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4478	pCodeBuf[off++] = cShift;
4479	}
4480	else
4481	{
4482	pCodeBuf[off++] = 0xd1;
4483	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4484	}
4485
4486	#elif defined(RT_ARCH_ARM64)
4487	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
4488
4489	#else
4490	# error "Port me"
4491	#endif
4492	return off;
4493	}
4494
4495
4496	/**
4497	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4498	* right.
4499	*/
4500	DECL_INLINE_THROW(uint32_t)
4501	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4502	{
4503	#if defined(RT_ARCH_AMD64)
4504	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4505	#elif defined(RT_ARCH_ARM64)
4506	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4507	#else
4508	# error "Port me"
4509	#endif
4510	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4511	return off;
4512	}
4513
4514
4515	/**
4516	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4517	* right and assigning it to a different GPR.
4518	*/
4519	DECL_INLINE_THROW(uint32_t)
4520	iemNativeEmitGpr32EqGprShiftRightImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint8_t cShift)
4521	{
4522	Assert(cShift > 0); Assert(cShift < 32);
4523	#if defined(RT_ARCH_AMD64)
4524	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4525	off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, iGprDst, cShift);
4526
4527	#elif defined(RT_ARCH_ARM64)
4528	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprSrc, cShift, false /64Bit/);
4529
4530	#else
4531	# error "Port me"
4532	#endif
4533	return off;
4534	}
4535
4536
4537	/**
4538	* Emits code for rotating a GPR a fixed number of bits to the left.
4539	*/
4540	DECL_FORCE_INLINE(uint32_t)
4541	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4542	{
4543	Assert(cShift > 0 && cShift < 64);
4544
4545	#if defined(RT_ARCH_AMD64)
4546	/* rol dst, cShift */
4547	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4548	if (cShift != 1)
4549	{
4550	pCodeBuf[off++] = 0xc1;
4551	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4552	pCodeBuf[off++] = cShift;
4553	}
4554	else
4555	{
4556	pCodeBuf[off++] = 0xd1;
4557	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4558	}
4559
4560	#elif defined(RT_ARCH_ARM64)
4561	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
4562
4563	#else
4564	# error "Port me"
4565	#endif
4566	return off;
4567	}
4568
4569
4570
4571	/*********************************************************************************************************************************
4572	* Compare and Testing *
4573	*********************************************************************************************************************************/
4574
4575
4576	#ifdef RT_ARCH_ARM64
4577	/**
4578	* Emits an ARM64 compare instruction.
4579	*/
4580	DECL_INLINE_THROW(uint32_t)
4581	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
4582	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
4583	{
4584	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4585	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
4586	f64Bit, true /fSetFlags/, cShift, enmShift);
4587	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4588	return off;
4589	}
4590	#endif
4591
4592
4593	/**
4594	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4595	* with conditional instruction.
4596	*/
4597	DECL_FORCE_INLINE(uint32_t)
4598	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4599	{
4600	#ifdef RT_ARCH_AMD64
4601	/* cmp Gv, Ev */
4602	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4603	pCodeBuf[off++] = 0x3b;
4604	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4605
4606	#elif defined(RT_ARCH_ARM64)
4607	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
4608
4609	#else
4610	# error "Port me!"
4611	#endif
4612	return off;
4613	}
4614
4615
4616	/**
4617	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4618	* with conditional instruction.
4619	*/
4620	DECL_INLINE_THROW(uint32_t)
4621	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4622	{
4623	#ifdef RT_ARCH_AMD64
4624	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4625	#elif defined(RT_ARCH_ARM64)
4626	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4627	#else
4628	# error "Port me!"
4629	#endif
4630	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4631	return off;
4632	}
4633
4634
4635	/**
4636	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4637	* with conditional instruction.
4638	*/
4639	DECL_FORCE_INLINE(uint32_t)
4640	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4641	{
4642	#ifdef RT_ARCH_AMD64
4643	/* cmp Gv, Ev */
4644	if (iGprLeft >= 8 \|\| iGprRight >= 8)
4645	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4646	pCodeBuf[off++] = 0x3b;
4647	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4648
4649	#elif defined(RT_ARCH_ARM64)
4650	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
4651
4652	#else
4653	# error "Port me!"
4654	#endif
4655	return off;
4656	}
4657
4658
4659	/**
4660	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4661	* with conditional instruction.
4662	*/
4663	DECL_INLINE_THROW(uint32_t)
4664	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4665	{
4666	#ifdef RT_ARCH_AMD64
4667	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4668	#elif defined(RT_ARCH_ARM64)
4669	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4670	#else
4671	# error "Port me!"
4672	#endif
4673	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4674	return off;
4675	}
4676
4677
4678	/**
4679	* Emits a compare of a 64-bit GPR with a constant value, settings status
4680	* flags/whatever for use with conditional instruction.
4681	*/
4682	DECL_INLINE_THROW(uint32_t)
4683	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
4684	{
4685	#ifdef RT_ARCH_AMD64
4686	if (uImm <= UINT32_C(0xff))
4687	{
4688	/* cmp Ev, Ib */
4689	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
4690	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4691	pbCodeBuf[off++] = 0x83;
4692	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4693	pbCodeBuf[off++] = (uint8_t)uImm;
4694	}
4695	else if ((int64_t)uImm == (int32_t)uImm)
4696	{
4697	/* cmp Ev, imm */
4698	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
4699	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4700	pbCodeBuf[off++] = 0x81;
4701	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4702	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4703	pbCodeBuf[off++] = RT_BYTE1(uImm);
4704	pbCodeBuf[off++] = RT_BYTE2(uImm);
4705	pbCodeBuf[off++] = RT_BYTE3(uImm);
4706	pbCodeBuf[off++] = RT_BYTE4(uImm);
4707	}
4708	else
4709	{
4710	/* Use temporary register for the immediate. */
4711	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4712	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4713	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4714	}
4715
4716	#elif defined(RT_ARCH_ARM64)
4717	/** @todo guess there are clevere things we can do here... */
4718	if (uImm < _4K)
4719	{
4720	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4721	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4722	true /64Bit/, true /fSetFlags/);
4723	}
4724	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4725	{
4726	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4727	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
4728	true /64Bit/, true /fSetFlags/, true /fShift12/);
4729	}
4730	else
4731	{
4732	/* Use temporary register for the immediate. */
4733	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4734	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4735	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4736	}
4737
4738	#else
4739	# error "Port me!"
4740	#endif
4741
4742	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4743	return off;
4744	}
4745
4746
4747	/**
4748	* Emits a compare of a 32-bit GPR with a constant value, settings status
4749	* flags/whatever for use with conditional instruction.
4750	*
4751	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
4752	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
4753	* bits all zero). Will release assert or throw exception if the caller
4754	* violates this restriction.
4755	*/
4756	DECL_FORCE_INLINE_THROW(uint32_t)
4757	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4758	{
4759	#ifdef RT_ARCH_AMD64
4760	if (iGprLeft >= 8)
4761	pCodeBuf[off++] = X86_OP_REX_B;
4762	if (uImm <= UINT32_C(0x7f))
4763	{
4764	/* cmp Ev, Ib */
4765	pCodeBuf[off++] = 0x83;
4766	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4767	pCodeBuf[off++] = (uint8_t)uImm;
4768	}
4769	else
4770	{
4771	/* cmp Ev, imm */
4772	pCodeBuf[off++] = 0x81;
4773	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4774	pCodeBuf[off++] = RT_BYTE1(uImm);
4775	pCodeBuf[off++] = RT_BYTE2(uImm);
4776	pCodeBuf[off++] = RT_BYTE3(uImm);
4777	pCodeBuf[off++] = RT_BYTE4(uImm);
4778	}
4779
4780	#elif defined(RT_ARCH_ARM64)
4781	/** @todo guess there are clevere things we can do here... */
4782	if (uImm < _4K)
4783	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4784	false /64Bit/, true /fSetFlags/);
4785	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4786	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4787	false /64Bit/, true /fSetFlags/, true /fShift12/);
4788	else
4789	# ifdef IEM_WITH_THROW_CATCH
4790	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4791	# else
4792	AssertReleaseFailedStmt(off = UINT32_MAX);
4793	# endif
4794
4795	#else
4796	# error "Port me!"
4797	#endif
4798	return off;
4799	}
4800
4801
4802	/**
4803	* Emits a compare of a 32-bit GPR with a constant value, settings status
4804	* flags/whatever for use with conditional instruction.
4805	*/
4806	DECL_INLINE_THROW(uint32_t)
4807	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4808	{
4809	#ifdef RT_ARCH_AMD64
4810	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
4811
4812	#elif defined(RT_ARCH_ARM64)
4813	/** @todo guess there are clevere things we can do here... */
4814	if (uImm < _4K)
4815	{
4816	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4817	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4818	false /64Bit/, true /fSetFlags/);
4819	}
4820	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4821	{
4822	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4823	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4824	false /64Bit/, true /fSetFlags/, true /fShift12/);
4825	}
4826	else
4827	{
4828	/* Use temporary register for the immediate. */
4829	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4830	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
4831	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4832	}
4833
4834	#else
4835	# error "Port me!"
4836	#endif
4837
4838	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4839	return off;
4840	}
4841
4842
4843
4844	/*********************************************************************************************************************************
4845	* Branching *
4846	*********************************************************************************************************************************/
4847
4848	/**
4849	* Emits a JMP rel32 / B imm19 to the given label.
4850	*/
4851	DECL_FORCE_INLINE_THROW(uint32_t)
4852	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
4853	{
4854	Assert(idxLabel < pReNative->cLabels);
4855
4856	#ifdef RT_ARCH_AMD64
4857	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4858	{
4859	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
4860	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
4861	{
4862	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
4863	pCodeBuf[off++] = (uint8_t)offRel;
4864	}
4865	else
4866	{
4867	offRel -= 3;
4868	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4869	pCodeBuf[off++] = RT_BYTE1(offRel);
4870	pCodeBuf[off++] = RT_BYTE2(offRel);
4871	pCodeBuf[off++] = RT_BYTE3(offRel);
4872	pCodeBuf[off++] = RT_BYTE4(offRel);
4873	}
4874	}
4875	else
4876	{
4877	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4878	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4879	pCodeBuf[off++] = 0xfe;
4880	pCodeBuf[off++] = 0xff;
4881	pCodeBuf[off++] = 0xff;
4882	pCodeBuf[off++] = 0xff;
4883	}
4884	pCodeBuf[off++] = 0xcc; /* int3 poison */
4885
4886	#elif defined(RT_ARCH_ARM64)
4887	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4888	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
4889	else
4890	{
4891	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
4892	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
4893	}
4894
4895	#else
4896	# error "Port me!"
4897	#endif
4898	return off;
4899	}
4900
4901
4902	/**
4903	* Emits a JMP rel32 / B imm19 to the given label.
4904	*/
4905	DECL_INLINE_THROW(uint32_t)
4906	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4907	{
4908	#ifdef RT_ARCH_AMD64
4909	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
4910	#elif defined(RT_ARCH_ARM64)
4911	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
4912	#else
4913	# error "Port me!"
4914	#endif
4915	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4916	return off;
4917	}
4918
4919
4920	/**
4921	* Emits a JMP rel32 / B imm19 to a new undefined label.
4922	*/
4923	DECL_INLINE_THROW(uint32_t)
4924	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4925	{
4926	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4927	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
4928	}
4929
4930	/** Condition type. */
4931	#ifdef RT_ARCH_AMD64
4932	typedef enum IEMNATIVEINSTRCOND : uint8_t
4933	{
4934	kIemNativeInstrCond_o = 0,
4935	kIemNativeInstrCond_no,
4936	kIemNativeInstrCond_c,
4937	kIemNativeInstrCond_nc,
4938	kIemNativeInstrCond_e,
4939	kIemNativeInstrCond_ne,
4940	kIemNativeInstrCond_be,
4941	kIemNativeInstrCond_nbe,
4942	kIemNativeInstrCond_s,
4943	kIemNativeInstrCond_ns,
4944	kIemNativeInstrCond_p,
4945	kIemNativeInstrCond_np,
4946	kIemNativeInstrCond_l,
4947	kIemNativeInstrCond_nl,
4948	kIemNativeInstrCond_le,
4949	kIemNativeInstrCond_nle
4950	} IEMNATIVEINSTRCOND;
4951	#elif defined(RT_ARCH_ARM64)
4952	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
4953	# define kIemNativeInstrCond_o todo_conditional_codes
4954	# define kIemNativeInstrCond_no todo_conditional_codes
4955	# define kIemNativeInstrCond_c todo_conditional_codes
4956	# define kIemNativeInstrCond_nc todo_conditional_codes
4957	# define kIemNativeInstrCond_e kArmv8InstrCond_Eq
4958	# define kIemNativeInstrCond_ne kArmv8InstrCond_Ne
4959	# define kIemNativeInstrCond_be kArmv8InstrCond_Ls
4960	# define kIemNativeInstrCond_nbe kArmv8InstrCond_Hi
4961	# define kIemNativeInstrCond_s todo_conditional_codes
4962	# define kIemNativeInstrCond_ns todo_conditional_codes
4963	# define kIemNativeInstrCond_p todo_conditional_codes
4964	# define kIemNativeInstrCond_np todo_conditional_codes
4965	# define kIemNativeInstrCond_l kArmv8InstrCond_Lt
4966	# define kIemNativeInstrCond_nl kArmv8InstrCond_Ge
4967	# define kIemNativeInstrCond_le kArmv8InstrCond_Le
4968	# define kIemNativeInstrCond_nle kArmv8InstrCond_Gt
4969	#else
4970	# error "Port me!"
4971	#endif
4972
4973
4974	/**
4975	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
4976	*/
4977	DECL_FORCE_INLINE_THROW(uint32_t)
4978	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
4979	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
4980	{
4981	Assert(idxLabel < pReNative->cLabels);
4982
4983	uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
4984	#ifdef RT_ARCH_AMD64
4985	if (offLabel >= off)
4986	{
4987	/* jcc rel32 */
4988	pCodeBuf[off++] = 0x0f;
4989	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4990	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4991	pCodeBuf[off++] = 0x00;
4992	pCodeBuf[off++] = 0x00;
4993	pCodeBuf[off++] = 0x00;
4994	pCodeBuf[off++] = 0x00;
4995	}
4996	else
4997	{
4998	int32_t offDisp = offLabel - (off + 2);
4999	if ((int8_t)offDisp == offDisp)
5000	{
5001	/* jcc rel8 */
5002	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
5003	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5004	}
5005	else
5006	{
5007	/* jcc rel32 */
5008	offDisp -= 4;
5009	pCodeBuf[off++] = 0x0f;
5010	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5011	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5012	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5013	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5014	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5015	}
5016	}
5017
5018	#elif defined(RT_ARCH_ARM64)
5019	if (offLabel >= off)
5020	{
5021	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5022	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
5023	}
5024	else
5025	{
5026	Assert(off - offLabel <= 0x3ffffU);
5027	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
5028	}
5029
5030	#else
5031	# error "Port me!"
5032	#endif
5033	return off;
5034	}
5035
5036
5037	/**
5038	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
5039	*/
5040	DECL_INLINE_THROW(uint32_t)
5041	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
5042	{
5043	#ifdef RT_ARCH_AMD64
5044	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
5045	#elif defined(RT_ARCH_ARM64)
5046	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
5047	#else
5048	# error "Port me!"
5049	#endif
5050	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5051	return off;
5052	}
5053
5054
5055	/**
5056	* Emits a Jcc rel32 / B.cc imm19 to a new label.
5057	*/
5058	DECL_INLINE_THROW(uint32_t)
5059	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5060	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
5061	{
5062	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5063	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
5064	}
5065
5066
5067	/**
5068	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
5069	*/
5070	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5071	{
5072	#ifdef RT_ARCH_AMD64
5073	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
5074	#elif defined(RT_ARCH_ARM64)
5075	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
5076	#else
5077	# error "Port me!"
5078	#endif
5079	}
5080
5081	/**
5082	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
5083	*/
5084	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5085	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5086	{
5087	#ifdef RT_ARCH_AMD64
5088	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
5089	#elif defined(RT_ARCH_ARM64)
5090	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
5091	#else
5092	# error "Port me!"
5093	#endif
5094	}
5095
5096
5097	/**
5098	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
5099	*/
5100	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5101	{
5102	#ifdef RT_ARCH_AMD64
5103	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
5104	#elif defined(RT_ARCH_ARM64)
5105	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
5106	#else
5107	# error "Port me!"
5108	#endif
5109	}
5110
5111	/**
5112	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
5113	*/
5114	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5115	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5116	{
5117	#ifdef RT_ARCH_AMD64
5118	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
5119	#elif defined(RT_ARCH_ARM64)
5120	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
5121	#else
5122	# error "Port me!"
5123	#endif
5124	}
5125
5126
5127	/**
5128	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
5129	*/
5130	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5131	{
5132	#ifdef RT_ARCH_AMD64
5133	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
5134	#elif defined(RT_ARCH_ARM64)
5135	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
5136	#else
5137	# error "Port me!"
5138	#endif
5139	}
5140
5141	/**
5142	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
5143	*/
5144	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5145	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5146	{
5147	#ifdef RT_ARCH_AMD64
5148	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
5149	#elif defined(RT_ARCH_ARM64)
5150	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
5151	#else
5152	# error "Port me!"
5153	#endif
5154	}
5155
5156
5157	/**
5158	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
5159	*/
5160	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5161	{
5162	#ifdef RT_ARCH_AMD64
5163	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
5164	#elif defined(RT_ARCH_ARM64)
5165	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
5166	#else
5167	# error "Port me!"
5168	#endif
5169	}
5170
5171	/**
5172	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
5173	*/
5174	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5175	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5176	{
5177	#ifdef RT_ARCH_AMD64
5178	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
5179	#elif defined(RT_ARCH_ARM64)
5180	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
5181	#else
5182	# error "Port me!"
5183	#endif
5184	}
5185
5186
5187	/**
5188	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
5189	*/
5190	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5191	{
5192	#ifdef RT_ARCH_AMD64
5193	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
5194	#elif defined(RT_ARCH_ARM64)
5195	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
5196	#else
5197	# error "Port me!"
5198	#endif
5199	}
5200
5201	/**
5202	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
5203	*/
5204	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5205	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5206	{
5207	#ifdef RT_ARCH_AMD64
5208	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
5209	#elif defined(RT_ARCH_ARM64)
5210	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
5211	#else
5212	# error "Port me!"
5213	#endif
5214	}
5215
5216
5217	/**
5218	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5219	*
5220	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5221	*
5222	* Only use hardcoded jumps forward when emitting for exactly one
5223	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5224	* the right target address on all platforms!
5225	*
5226	* Please also note that on x86 it is necessary pass off + 256 or higher
5227	* for @a offTarget one believe the intervening code is more than 127
5228	* bytes long.
5229	*/
5230	DECL_FORCE_INLINE(uint32_t)
5231	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5232	{
5233	#ifdef RT_ARCH_AMD64
5234	/* jcc rel8 / rel32 */
5235	int32_t offDisp = (int32_t)(offTarget - (off + 2));
5236	if (offDisp < 128 && offDisp >= -128)
5237	{
5238	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
5239	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5240	}
5241	else
5242	{
5243	offDisp -= 4;
5244	pCodeBuf[off++] = 0x0f;
5245	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5246	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5247	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5248	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5249	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5250	}
5251
5252	#elif defined(RT_ARCH_ARM64)
5253	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
5254
5255	#else
5256	# error "Port me!"
5257	#endif
5258	return off;
5259	}
5260
5261
5262	/**
5263	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5264	*
5265	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5266	*
5267	* Only use hardcoded jumps forward when emitting for exactly one
5268	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5269	* the right target address on all platforms!
5270	*
5271	* Please also note that on x86 it is necessary pass off + 256 or higher
5272	* for @a offTarget one believe the intervening code is more than 127
5273	* bytes long.
5274	*/
5275	DECL_INLINE_THROW(uint32_t)
5276	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5277	{
5278	#ifdef RT_ARCH_AMD64
5279	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
5280	#elif defined(RT_ARCH_ARM64)
5281	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
5282	#else
5283	# error "Port me!"
5284	#endif
5285	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5286	return off;
5287	}
5288
5289
5290	/**
5291	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
5292	*
5293	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5294	*/
5295	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5296	{
5297	#ifdef RT_ARCH_AMD64
5298	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
5299	#elif defined(RT_ARCH_ARM64)
5300	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
5301	#else
5302	# error "Port me!"
5303	#endif
5304	}
5305
5306
5307	/**
5308	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
5309	*
5310	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5311	*/
5312	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5313	{
5314	#ifdef RT_ARCH_AMD64
5315	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
5316	#elif defined(RT_ARCH_ARM64)
5317	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
5318	#else
5319	# error "Port me!"
5320	#endif
5321	}
5322
5323
5324	/**
5325	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
5326	*
5327	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5328	*/
5329	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5330	{
5331	#ifdef RT_ARCH_AMD64
5332	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
5333	#elif defined(RT_ARCH_ARM64)
5334	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
5335	#else
5336	# error "Port me!"
5337	#endif
5338	}
5339
5340
5341	/**
5342	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
5343	*
5344	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5345	*/
5346	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5347	{
5348	#ifdef RT_ARCH_AMD64
5349	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
5350	#elif defined(RT_ARCH_ARM64)
5351	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
5352	#else
5353	# error "Port me!"
5354	#endif
5355	}
5356
5357
5358	/**
5359	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5360	*
5361	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5362	*/
5363	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
5364	{
5365	#ifdef RT_ARCH_AMD64
5366	/* jmp rel8 or rel32 */
5367	int32_t offDisp = offTarget - (off + 2);
5368	if (offDisp < 128 && offDisp >= -128)
5369	{
5370	pCodeBuf[off++] = 0xeb;
5371	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5372	}
5373	else
5374	{
5375	offDisp -= 3;
5376	pCodeBuf[off++] = 0xe9;
5377	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5378	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5379	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5380	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5381	}
5382
5383	#elif defined(RT_ARCH_ARM64)
5384	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
5385
5386	#else
5387	# error "Port me!"
5388	#endif
5389	return off;
5390	}
5391
5392
5393	/**
5394	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5395	*
5396	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5397	*/
5398	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5399	{
5400	#ifdef RT_ARCH_AMD64
5401	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
5402	#elif defined(RT_ARCH_ARM64)
5403	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
5404	#else
5405	# error "Port me!"
5406	#endif
5407	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5408	return off;
5409	}
5410
5411
5412	/**
5413	* Fixes up a conditional jump to a fixed label.
5414	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
5415	* iemNativeEmitJzToFixed, ...
5416	*/
5417	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
5418	{
5419	#ifdef RT_ARCH_AMD64
5420	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
5421	uint8_t const bOpcode = pbCodeBuf[offFixup];
5422	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
5423	{
5424	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
5425	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
5426	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
5427	}
5428	else
5429	{
5430	if (bOpcode != 0x0f)
5431	Assert(bOpcode == 0xe9);
5432	else
5433	{
5434	offFixup += 1;
5435	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
5436	}
5437	uint32_t const offRel32 = offTarget - (offFixup + 5);
5438	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
5439	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
5440	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
5441	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
5442	}
5443
5444	#elif defined(RT_ARCH_ARM64)
5445	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
5446	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
5447	{
5448	/* B.COND + BC.COND */
5449	int32_t const offDisp = offTarget - offFixup;
5450	Assert(offDisp >= -262144 && offDisp < 262144);
5451	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
5452	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
5453	}
5454	else
5455	{
5456	/* B imm26 */
5457	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
5458	int32_t const offDisp = offTarget - offFixup;
5459	Assert(offDisp >= -33554432 && offDisp < 33554432);
5460	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
5461	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
5462	}
5463
5464	#else
5465	# error "Port me!"
5466	#endif
5467	}
5468
5469
5470	/**
5471	* Internal helper, don't call directly.
5472	*/
5473	DECL_INLINE_THROW(uint32_t)
5474	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5475	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
5476	{
5477	Assert(iBitNo < 64);
5478	#ifdef RT_ARCH_AMD64
5479	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5480	if (iBitNo < 8)
5481	{
5482	/* test Eb, imm8 */
5483	if (iGprSrc >= 4)
5484	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5485	pbCodeBuf[off++] = 0xf6;
5486	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5487	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
5488	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5489	}
5490	else
5491	{
5492	/* bt Ev, imm8 */
5493	if (iBitNo >= 32)
5494	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
5495	else if (iGprSrc >= 8)
5496	pbCodeBuf[off++] = X86_OP_REX_B;
5497	pbCodeBuf[off++] = 0x0f;
5498	pbCodeBuf[off++] = 0xba;
5499	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
5500	pbCodeBuf[off++] = iBitNo;
5501	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
5502	}
5503
5504	#elif defined(RT_ARCH_ARM64)
5505	/* Use the TBNZ instruction here. */
5506	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5507	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
5508	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
5509
5510	#else
5511	# error "Port me!"
5512	#endif
5513	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5514	return off;
5515	}
5516
5517
5518	/**
5519	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
5520	* @a iGprSrc.
5521	*
5522	* @note On ARM64 the range is only +/-8191 instructions.
5523	*/
5524	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5525	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5526	{
5527	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
5528	}
5529
5530
5531	/**
5532	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
5533	* _set_ in @a iGprSrc.
5534	*
5535	* @note On ARM64 the range is only +/-8191 instructions.
5536	*/
5537	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5538	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5539	{
5540	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
5541	}
5542
5543
5544	/**
5545	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
5546	* flags accordingly.
5547	*/
5548	DECL_INLINE_THROW(uint32_t)
5549	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
5550	{
5551	Assert(fBits != 0);
5552	#ifdef RT_ARCH_AMD64
5553
5554	if (fBits >= UINT32_MAX)
5555	{
5556	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5557
5558	/* test Ev,Gv */
5559	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5560	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
5561	pbCodeBuf[off++] = 0x85;
5562	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
5563
5564	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5565	}
5566	else if (fBits <= UINT32_MAX)
5567	{
5568	/* test Eb, imm8 or test Ev, imm32 */
5569	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
5570	if (fBits <= UINT8_MAX)
5571	{
5572	if (iGprSrc >= 4)
5573	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5574	pbCodeBuf[off++] = 0xf6;
5575	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5576	pbCodeBuf[off++] = (uint8_t)fBits;
5577	}
5578	else
5579	{
5580	if (iGprSrc >= 8)
5581	pbCodeBuf[off++] = X86_OP_REX_B;
5582	pbCodeBuf[off++] = 0xf7;
5583	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5584	pbCodeBuf[off++] = RT_BYTE1(fBits);
5585	pbCodeBuf[off++] = RT_BYTE2(fBits);
5586	pbCodeBuf[off++] = RT_BYTE3(fBits);
5587	pbCodeBuf[off++] = RT_BYTE4(fBits);
5588	}
5589	}
5590	/** @todo implement me. */
5591	else
5592	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
5593
5594	#elif defined(RT_ARCH_ARM64)
5595	uint32_t uImmR = 0;
5596	uint32_t uImmNandS = 0;
5597	if (Armv8A64ConvertMask64ToImmRImmS(fBits, &uImmNandS, &uImmR))
5598	{
5599	/* ands xzr, iGprSrc, #fBits */
5600	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5601	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR);
5602	}
5603	else
5604	{
5605	/* ands xzr, iGprSrc, iTmpReg */
5606	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5607	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5608	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
5609	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5610	}
5611
5612	#else
5613	# error "Port me!"
5614	#endif
5615	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5616	return off;
5617	}
5618
5619
5620	/**
5621	* Emits a test for any of the bits from @a fBits in the lower 32 bits of
5622	* @a iGprSrc, setting CPU flags accordingly.
5623	*
5624	* @note For ARM64 this only supports @a fBits values that can be expressed
5625	* using the two 6-bit immediates of the ANDS instruction. The caller
5626	* must make sure this is possible!
5627	*/
5628	DECL_FORCE_INLINE_THROW(uint32_t)
5629	iemNativeEmitTestAnyBitsInGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint32_t fBits)
5630	{
5631	Assert(fBits != 0);
5632
5633	#ifdef RT_ARCH_AMD64
5634	if (fBits <= UINT8_MAX)
5635	{
5636	/* test Eb, imm8 */
5637	if (iGprSrc >= 4)
5638	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5639	pCodeBuf[off++] = 0xf6;
5640	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5641	pCodeBuf[off++] = (uint8_t)fBits;
5642	}
5643	else
5644	{
5645	/* test Ev, imm32 */
5646	if (iGprSrc >= 8)
5647	pCodeBuf[off++] = X86_OP_REX_B;
5648	pCodeBuf[off++] = 0xf7;
5649	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5650	pCodeBuf[off++] = RT_BYTE1(fBits);
5651	pCodeBuf[off++] = RT_BYTE2(fBits);
5652	pCodeBuf[off++] = RT_BYTE3(fBits);
5653	pCodeBuf[off++] = RT_BYTE4(fBits);
5654	}
5655
5656	#elif defined(RT_ARCH_ARM64)
5657	/* ands xzr, src, #fBits */
5658	uint32_t uImmR = 0;
5659	uint32_t uImmNandS = 0;
5660	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5661	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5662	else
5663	# ifdef IEM_WITH_THROW_CATCH
5664	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5665	# else
5666	AssertReleaseFailedStmt(off = UINT32_MAX);
5667	# endif
5668
5669	#else
5670	# error "Port me!"
5671	#endif
5672	return off;
5673	}
5674
5675
5676
5677	/**
5678	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5679	* @a iGprSrc, setting CPU flags accordingly.
5680	*
5681	* @note For ARM64 this only supports @a fBits values that can be expressed
5682	* using the two 6-bit immediates of the ANDS instruction. The caller
5683	* must make sure this is possible!
5684	*/
5685	DECL_FORCE_INLINE_THROW(uint32_t)
5686	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5687	{
5688	Assert(fBits != 0);
5689
5690	#ifdef RT_ARCH_AMD64
5691	/* test Eb, imm8 */
5692	if (iGprSrc >= 4)
5693	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5694	pCodeBuf[off++] = 0xf6;
5695	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5696	pCodeBuf[off++] = fBits;
5697
5698	#elif defined(RT_ARCH_ARM64)
5699	/* ands xzr, src, #fBits */
5700	uint32_t uImmR = 0;
5701	uint32_t uImmNandS = 0;
5702	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5703	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5704	else
5705	# ifdef IEM_WITH_THROW_CATCH
5706	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5707	# else
5708	AssertReleaseFailedStmt(off = UINT32_MAX);
5709	# endif
5710
5711	#else
5712	# error "Port me!"
5713	#endif
5714	return off;
5715	}
5716
5717
5718	/**
5719	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5720	* @a iGprSrc, setting CPU flags accordingly.
5721	*/
5722	DECL_INLINE_THROW(uint32_t)
5723	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5724	{
5725	Assert(fBits != 0);
5726
5727	#ifdef RT_ARCH_AMD64
5728	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
5729
5730	#elif defined(RT_ARCH_ARM64)
5731	/* ands xzr, src, [tmp\|#imm] */
5732	uint32_t uImmR = 0;
5733	uint32_t uImmNandS = 0;
5734	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5735	{
5736	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5737	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5738	}
5739	else
5740	{
5741	/* Use temporary register for the 64-bit immediate. */
5742	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5743	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5744	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
5745	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5746	}
5747
5748	#else
5749	# error "Port me!"
5750	#endif
5751	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5752	return off;
5753	}
5754
5755
5756	/**
5757	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
5758	* are set in @a iGprSrc.
5759	*/
5760	DECL_INLINE_THROW(uint32_t)
5761	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5762	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5763	{
5764	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5765
5766	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5767	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5768
5769	return off;
5770	}
5771
5772
5773	/**
5774	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
5775	* are set in @a iGprSrc.
5776	*/
5777	DECL_INLINE_THROW(uint32_t)
5778	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5779	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5780	{
5781	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5782
5783	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5784	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
5785
5786	return off;
5787	}
5788
5789
5790	/**
5791	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5792	*
5793	* The operand size is given by @a f64Bit.
5794	*/
5795	DECL_FORCE_INLINE_THROW(uint32_t)
5796	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5797	uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5798	{
5799	Assert(idxLabel < pReNative->cLabels);
5800
5801	#ifdef RT_ARCH_AMD64
5802	/* test reg32,reg32 / test reg64,reg64 */
5803	if (f64Bit)
5804	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
5805	else if (iGprSrc >= 8)
5806	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
5807	pCodeBuf[off++] = 0x85;
5808	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
5809
5810	/* jnz idxLabel */
5811	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
5812	fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5813
5814	#elif defined(RT_ARCH_ARM64)
5815	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
5816	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(pReNative->paLabels[idxLabel].off - off),
5817	iGprSrc, f64Bit);
5818	else
5819	{
5820	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5821	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, 0, iGprSrc, f64Bit);
5822	}
5823
5824	#else
5825	# error "Port me!"
5826	#endif
5827	return off;
5828	}
5829
5830
5831	/**
5832	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5833	*
5834	* The operand size is given by @a f64Bit.
5835	*/
5836	DECL_FORCE_INLINE_THROW(uint32_t)
5837	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5838	bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5839	{
5840	#ifdef RT_ARCH_AMD64
5841	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
5842	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5843	#elif defined(RT_ARCH_ARM64)
5844	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1),
5845	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5846	#else
5847	# error "Port me!"
5848	#endif
5849	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5850	return off;
5851	}
5852
5853
5854	/**
5855	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5856	*
5857	* The operand size is given by @a f64Bit.
5858	*/
5859	DECL_FORCE_INLINE_THROW(uint32_t)
5860	iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5861	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5862	{
5863	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5864	f64Bit, false /fJmpIfNotZero/, idxLabel);
5865	}
5866
5867
5868	/**
5869	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5870	*
5871	* The operand size is given by @a f64Bit.
5872	*/
5873	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5874	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5875	{
5876	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /fJmpIfNotZero/, idxLabel);
5877	}
5878
5879
5880	/**
5881	* Emits code that jumps to a new label if @a iGprSrc is zero.
5882	*
5883	* The operand size is given by @a f64Bit.
5884	*/
5885	DECL_INLINE_THROW(uint32_t)
5886	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5887	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5888	{
5889	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5890	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5891	}
5892
5893
5894	/**
5895	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5896	*
5897	* The operand size is given by @a f64Bit.
5898	*/
5899	DECL_FORCE_INLINE_THROW(uint32_t)
5900	iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5901	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5902	{
5903	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5904	f64Bit, true /fJmpIfNotZero/, idxLabel);
5905	}
5906
5907
5908	/**
5909	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5910	*
5911	* The operand size is given by @a f64Bit.
5912	*/
5913	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5914	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5915	{
5916	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /fJmpIfNotZero/, idxLabel);
5917	}
5918
5919
5920	/**
5921	* Emits code that jumps to a new label if @a iGprSrc is not zero.
5922	*
5923	* The operand size is given by @a f64Bit.
5924	*/
5925	DECL_INLINE_THROW(uint32_t)
5926	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5927	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5928	{
5929	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5930	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5931	}
5932
5933
5934	/**
5935	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
5936	* differs.
5937	*/
5938	DECL_INLINE_THROW(uint32_t)
5939	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5940	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
5941	{
5942	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
5943	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5944	return off;
5945	}
5946
5947
5948	/**
5949	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
5950	*/
5951	DECL_INLINE_THROW(uint32_t)
5952	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5953	uint8_t iGprLeft, uint8_t iGprRight,
5954	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5955	{
5956	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5957	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
5958	}
5959
5960
5961	/**
5962	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
5963	*/
5964	DECL_INLINE_THROW(uint32_t)
5965	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5966	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
5967	{
5968	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
5969	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5970	return off;
5971	}
5972
5973
5974	/**
5975	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
5976	*/
5977	DECL_INLINE_THROW(uint32_t)
5978	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5979	uint8_t iGprSrc, uint64_t uImm,
5980	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5981	{
5982	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5983	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
5984	}
5985
5986
5987	/**
5988	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
5989	* @a uImm.
5990	*/
5991	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5992	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
5993	{
5994	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
5995	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5996	return off;
5997	}
5998
5999
6000	/**
6001	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
6002	* @a uImm.
6003	*/
6004	DECL_INLINE_THROW(uint32_t)
6005	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6006	uint8_t iGprSrc, uint32_t uImm,
6007	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6008	{
6009	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6010	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
6011	}
6012
6013
6014	/*********************************************************************************************************************************
6015	* Calls. *
6016	*********************************************************************************************************************************/
6017
6018	/**
6019	* Emits a call to a 64-bit address.
6020	*/
6021	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
6022	{
6023	#ifdef RT_ARCH_AMD64
6024	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
6025
6026	/* call rax */
6027	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
6028	pbCodeBuf[off++] = 0xff;
6029	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
6030
6031	#elif defined(RT_ARCH_ARM64)
6032	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
6033
6034	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
6035	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
6036
6037	#else
6038	# error "port me"
6039	#endif
6040	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
6041	return off;
6042	}
6043
6044
6045	/**
6046	* Emits code to load a stack variable into an argument GPR.
6047	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
6048	*/
6049	DECL_FORCE_INLINE_THROW(uint32_t)
6050	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6051	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = UINT32_MAX,
6052	bool fSpilledVarsInVolatileRegs = false)
6053	{
6054	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6055	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
6056	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6057
6058	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
6059	if ( idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs)
6060	&& ( (RT_BIT_32(idxRegVar) & (~IEMNATIVE_CALL_VOLATILE_GREG_MASK \| fHstVolatileRegsAllowed))
6061	\|\| !fSpilledVarsInVolatileRegs ))
6062	{
6063	AssertStmt( !(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK)
6064	\|\| (RT_BIT_32(idxRegVar) & fHstVolatileRegsAllowed),
6065	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_13));
6066	if (!offAddend)
6067	{
6068	if (idxRegArg != idxRegVar)
6069	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
6070	}
6071	else
6072	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
6073	}
6074	else
6075	{
6076	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
6077	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
6078	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
6079	if (offAddend)
6080	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
6081	}
6082	return off;
6083	}
6084
6085
6086	/**
6087	* Emits code to load a stack or immediate variable value into an argument GPR,
6088	* optional with a addend.
6089	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
6090	*/
6091	DECL_FORCE_INLINE_THROW(uint32_t)
6092	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6093	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = 0,
6094	bool fSpilledVarsInVolatileRegs = false)
6095	{
6096	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6097	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
6098	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
6099	else
6100	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend,
6101	fHstVolatileRegsAllowed, fSpilledVarsInVolatileRegs);
6102	return off;
6103	}
6104
6105
6106	/**
6107	* Emits code to load the variable address into an argument GRP.
6108	*
6109	* This only works for uninitialized and stack variables.
6110	*/
6111	DECL_FORCE_INLINE_THROW(uint32_t)
6112	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6113	bool fFlushShadows)
6114	{
6115	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6116	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
6117	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
6118	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6119
6120	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
6121	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
6122
6123	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
6124	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
6125	{
6126	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
6127	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
6128	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6129	}
6130	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
6131	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6132
6133	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
6134	}
6135
6136
6137	/** @} */
6138
6139	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
6140

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102801

Download in other formats: