IEMN8veRecompilerEmit.h@ 102765

Last change on this file since 102765 was 102765, checked in by vboxsync, 13 months ago
VMM/IEM: Reworking native translation of IEM_MC_PUSH in prep for doing TLB lookups. bugreg:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 200.9 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102765 2024-01-04 19:01:46Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	pCodeBuf[off++] = 0xcc;
88	RT_NOREF(uInfo); /** @todo use multibyte nop for info? */
89
90	#elif defined(RT_ARCH_ARM64)
91	pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
92
93	#else
94	# error "error"
95	#endif
96	return off;
97	}
98
99
100	/**
101	* Emit a breakpoint instruction.
102	*/
103	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
104	{
105	#ifdef RT_ARCH_AMD64
106	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
107	#elif defined(RT_ARCH_ARM64)
108	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
109	#else
110	# error "error"
111	#endif
112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
113	return off;
114	}
115
116
117	/*********************************************************************************************************************************
118	* Loads, Stores and Related Stuff. *
119	*********************************************************************************************************************************/
120
121	#ifdef RT_ARCH_AMD64
122	/**
123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
124	*/
125	DECL_FORCE_INLINE(uint32_t)
126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
127	{
128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
129	{
130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
133	}
134	else if (offDisp == (int8_t)offDisp)
135	{
136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
139	pbCodeBuf[off++] = (uint8_t)offDisp;
140	}
141	else
142	{
143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
150	}
151	return off;
152	}
153	#endif /* RT_ARCH_AMD64 */
154
155	/**
156	* Emits setting a GPR to zero.
157	*/
158	DECL_INLINE_THROW(uint32_t)
159	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
160	{
161	#ifdef RT_ARCH_AMD64
162	/* xor gpr32, gpr32 */
163	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
164	if (iGpr >= 8)
165	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
166	pbCodeBuf[off++] = 0x33;
167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
168
169	#elif defined(RT_ARCH_ARM64)
170	/* mov gpr, #0x0 */
171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
172	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
173
174	#else
175	# error "port me"
176	#endif
177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
178	return off;
179	}
180
181
182	/**
183	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
184	* buffer space.
185	*
186	* Max buffer consumption:
187	* - AMD64: 10 instruction bytes.
188	* - ARM64: 4 instruction words (16 bytes).
189	*/
190	DECL_FORCE_INLINE(uint32_t)
191	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
192	{
193	#ifdef RT_ARCH_AMD64
194	if (uImm64 == 0)
195	{
196	/* xor gpr, gpr */
197	if (iGpr >= 8)
198	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
199	pCodeBuf[off++] = 0x33;
200	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
201	}
202	else if (uImm64 <= UINT32_MAX)
203	{
204	/* mov gpr, imm32 */
205	if (iGpr >= 8)
206	pCodeBuf[off++] = X86_OP_REX_B;
207	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else if (uImm64 == (uint64_t)(int32_t)uImm64)
214	{
215	/* mov gpr, sx(imm32) */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xc7;
221	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
222	pCodeBuf[off++] = RT_BYTE1(uImm64);
223	pCodeBuf[off++] = RT_BYTE2(uImm64);
224	pCodeBuf[off++] = RT_BYTE3(uImm64);
225	pCodeBuf[off++] = RT_BYTE4(uImm64);
226	}
227	else
228	{
229	/* mov gpr, imm64 */
230	if (iGpr < 8)
231	pCodeBuf[off++] = X86_OP_REX_W;
232	else
233	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
234	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
235	pCodeBuf[off++] = RT_BYTE1(uImm64);
236	pCodeBuf[off++] = RT_BYTE2(uImm64);
237	pCodeBuf[off++] = RT_BYTE3(uImm64);
238	pCodeBuf[off++] = RT_BYTE4(uImm64);
239	pCodeBuf[off++] = RT_BYTE5(uImm64);
240	pCodeBuf[off++] = RT_BYTE6(uImm64);
241	pCodeBuf[off++] = RT_BYTE7(uImm64);
242	pCodeBuf[off++] = RT_BYTE8(uImm64);
243	}
244
245	#elif defined(RT_ARCH_ARM64)
246	/*
247	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
248	* supply remaining bits using 'movk grp, imm16, lsl #x'.
249	*
250	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
251	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
252	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
253	* after the first non-zero immediate component so we switch to movk for
254	* the remainder.
255	*/
256	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
257	+ !((uImm64 >> 16) & UINT16_MAX)
258	+ !((uImm64 >> 32) & UINT16_MAX)
259	+ !((uImm64 >> 48) & UINT16_MAX);
260	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
261	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
262	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
263	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
264	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
265	if (cFfffHalfWords <= cZeroHalfWords)
266	{
267	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
268
269	/* movz gpr, imm16 */
270	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
271	if (uImmPart \|\| cZeroHalfWords == 4)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #16 */
277	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
278	if (uImmPart)
279	{
280	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
281	fMovBase \|= RT_BIT_32(29);
282	}
283	/* mov[z/k] gpr, imm16, lsl #32 */
284	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
285	if (uImmPart)
286	{
287	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
288	fMovBase \|= RT_BIT_32(29);
289	}
290	/* mov[z/k] gpr, imm16, lsl #48 */
291	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
292	if (uImmPart)
293	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
294	}
295	else
296	{
297	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
298
299	/* find the first half-word that isn't UINT16_MAX. */
300	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
301	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
302	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
303
304	/* movn gpr, imm16, lsl #iHwNotFfff16 /
305	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
306	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
307	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
308	/* movk gpr, imm16 */
309	if (iHwNotFfff != 0)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #16 */
316	if (iHwNotFfff != 1)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
321	}
322	/* movk gpr, imm16, lsl #32 */
323	if (iHwNotFfff != 2)
324	{
325	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
326	if (uImmPart != UINT32_C(0xffff))
327	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
328	}
329	/* movk gpr, imm16, lsl #48 */
330	if (iHwNotFfff != 3)
331	{
332	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
333	if (uImmPart != UINT32_C(0xffff))
334	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
335	}
336	}
337
338	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
339	* clang 12.x does that, only to use the 'x' version for the
340	* addressing in the following ldr). */
341
342	#else
343	# error "port me"
344	#endif
345	return off;
346	}
347
348
349	/**
350	* Emits loading a constant into a 64-bit GPR
351	*/
352	DECL_INLINE_THROW(uint32_t)
353	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
354	{
355	#ifdef RT_ARCH_AMD64
356	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
357	#elif defined(RT_ARCH_ARM64)
358	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
359	#else
360	# error "port me"
361	#endif
362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
363	return off;
364	}
365
366
367	/**
368	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
369	* buffer space.
370	*
371	* Max buffer consumption:
372	* - AMD64: 6 instruction bytes.
373	* - ARM64: 2 instruction words (8 bytes).
374	*
375	* @note The top 32 bits will be cleared.
376	*/
377	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
378	{
379	#ifdef RT_ARCH_AMD64
380	if (uImm32 == 0)
381	{
382	/* xor gpr, gpr */
383	if (iGpr >= 8)
384	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
385	pCodeBuf[off++] = 0x33;
386	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
387	}
388	else
389	{
390	/* mov gpr, imm32 */
391	if (iGpr >= 8)
392	pCodeBuf[off++] = X86_OP_REX_B;
393	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
394	pCodeBuf[off++] = RT_BYTE1(uImm32);
395	pCodeBuf[off++] = RT_BYTE2(uImm32);
396	pCodeBuf[off++] = RT_BYTE3(uImm32);
397	pCodeBuf[off++] = RT_BYTE4(uImm32);
398	}
399
400	#elif defined(RT_ARCH_ARM64)
401	if ((uImm32 >> 16) == 0)
402	/* movz gpr, imm16 */
403	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
404	else if ((uImm32 & UINT32_C(0xffff)) == 0)
405	/* movz gpr, imm16, lsl #16 */
406	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
407	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
408	/* movn gpr, imm16, lsl #16 */
409	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
410	else if ((uImm32 >> 16) == UINT32_C(0xffff))
411	/* movn gpr, imm16 */
412	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
413	else
414	{
415	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
416	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
417	}
418
419	#else
420	# error "port me"
421	#endif
422	return off;
423	}
424
425
426	/**
427	* Emits loading a constant into a 32-bit GPR.
428	* @note The top 32 bits will be cleared.
429	*/
430	DECL_INLINE_THROW(uint32_t)
431	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
432	{
433	#ifdef RT_ARCH_AMD64
434	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
435	#elif defined(RT_ARCH_ARM64)
436	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
437	#else
438	# error "port me"
439	#endif
440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
441	return off;
442	}
443
444
445	/**
446	* Emits loading a constant into a 8-bit GPR
447	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
448	* only the ARM64 version does that.
449	*/
450	DECL_INLINE_THROW(uint32_t)
451	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
452	{
453	#ifdef RT_ARCH_AMD64
454	/* mov gpr, imm8 */
455	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
456	if (iGpr >= 8)
457	pbCodeBuf[off++] = X86_OP_REX_B;
458	else if (iGpr >= 4)
459	pbCodeBuf[off++] = X86_OP_REX;
460	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
461	pbCodeBuf[off++] = RT_BYTE1(uImm8);
462
463	#elif defined(RT_ARCH_ARM64)
464	/* movz gpr, imm16, lsl #0 */
465	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
466	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
467
468	#else
469	# error "port me"
470	#endif
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472	return off;
473	}
474
475
476	#ifdef RT_ARCH_AMD64
477	/**
478	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
479	*/
480	DECL_FORCE_INLINE(uint32_t)
481	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
482	{
483	if (offVCpu < 128)
484	{
485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
486	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
487	}
488	else
489	{
490	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
491	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
492	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
493	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
494	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
495	}
496	return off;
497	}
498
499	#elif defined(RT_ARCH_ARM64)
500
501	/**
502	* Common bit of iemNativeEmitLoadGprFromVCpuU64Ex and friends.
503	*
504	* @note Loads can use @a iGprReg for large offsets, stores requires a temporary
505	* registers (@a iGprTmp).
506	* @note DON'T try this with prefetch.
507	*/
508	DECL_FORCE_INLINE_THROW(uint32_t)
509	iemNativeEmitGprByVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu,
510	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
511	{
512	/*
513	* There are a couple of ldr variants that takes an immediate offset, so
514	* try use those if we can, otherwise we have to use the temporary register
515	* help with the addressing.
516	*/
517	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
518	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
519	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
520	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
521	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
522	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
523	else if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) \|\| iGprTmp != UINT8_MAX)
524	{
525	/* The offset is too large, so we must load it into a register and use
526	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
527	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
528	if (iGprTmp == UINT8_MAX)
529	iGprTmp = iGprReg;
530	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, offVCpu);
531	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, iGprTmp);
532	}
533	else
534	# ifdef IEM_WITH_THROW_CATCH
535	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
536	# else
537	AssertReleaseFailedStmt(off = UINT32_MAX);
538	# endif
539
540	return off;
541	}
542
543	/**
544	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
545	*/
546	DECL_FORCE_INLINE_THROW(uint32_t)
547	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
548	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
549	{
550	/*
551	* There are a couple of ldr variants that takes an immediate offset, so
552	* try use those if we can, otherwise we have to use the temporary register
553	* help with the addressing.
554	*/
555	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
556	{
557	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
558	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
559	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
560	}
561	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
562	{
563	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
564	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
565	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
566	}
567	else
568	{
569	/* The offset is too large, so we must load it into a register and use
570	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
571	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
572	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
574	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
575	IEMNATIVE_REG_FIXED_TMP0);
576	}
577	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
578	return off;
579	}
580
581	#endif /* RT_ARCH_ARM64 */
582
583
584	/**
585	* Emits a 64-bit GPR load of a VCpu value.
586	*/
587	DECL_FORCE_INLINE_THROW(uint32_t)
588	iemNativeEmitLoadGprFromVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
589	{
590	#ifdef RT_ARCH_AMD64
591	/* mov reg64, mem64 */
592	if (iGpr < 8)
593	pCodeBuf[off++] = X86_OP_REX_W;
594	else
595	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
596	pCodeBuf[off++] = 0x8b;
597	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
598
599	#elif defined(RT_ARCH_ARM64)
600	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
601
602	#else
603	# error "port me"
604	#endif
605	return off;
606	}
607
608
609	/**
610	* Emits a 64-bit GPR load of a VCpu value.
611	*/
612	DECL_INLINE_THROW(uint32_t)
613	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
614	{
615	#ifdef RT_ARCH_AMD64
616	off = iemNativeEmitLoadGprFromVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
617	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
618
619	#elif defined(RT_ARCH_ARM64)
620	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
621
622	#else
623	# error "port me"
624	#endif
625	return off;
626	}
627
628
629	/**
630	* Emits a 32-bit GPR load of a VCpu value.
631	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
632	*/
633	DECL_INLINE_THROW(uint32_t)
634	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
635	{
636	#ifdef RT_ARCH_AMD64
637	/* mov reg32, mem32 */
638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
639	if (iGpr >= 8)
640	pbCodeBuf[off++] = X86_OP_REX_R;
641	pbCodeBuf[off++] = 0x8b;
642	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
644
645	#elif defined(RT_ARCH_ARM64)
646	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
647
648	#else
649	# error "port me"
650	#endif
651	return off;
652	}
653
654
655	/**
656	* Emits a 16-bit GPR load of a VCpu value.
657	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
658	*/
659	DECL_INLINE_THROW(uint32_t)
660	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
661	{
662	#ifdef RT_ARCH_AMD64
663	/* movzx reg32, mem16 */
664	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
665	if (iGpr >= 8)
666	pbCodeBuf[off++] = X86_OP_REX_R;
667	pbCodeBuf[off++] = 0x0f;
668	pbCodeBuf[off++] = 0xb7;
669	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
670	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
671
672	#elif defined(RT_ARCH_ARM64)
673	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
674
675	#else
676	# error "port me"
677	#endif
678	return off;
679	}
680
681
682	/**
683	* Emits a 8-bit GPR load of a VCpu value.
684	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
685	*/
686	DECL_INLINE_THROW(uint32_t)
687	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
688	{
689	#ifdef RT_ARCH_AMD64
690	/* movzx reg32, mem8 */
691	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
692	if (iGpr >= 8)
693	pbCodeBuf[off++] = X86_OP_REX_R;
694	pbCodeBuf[off++] = 0x0f;
695	pbCodeBuf[off++] = 0xb6;
696	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
697	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
698
699	#elif defined(RT_ARCH_ARM64)
700	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
701
702	#else
703	# error "port me"
704	#endif
705	return off;
706	}
707
708
709	/**
710	* Emits a store of a GPR value to a 64-bit VCpu field.
711	*/
712	DECL_INLINE_THROW(uint32_t)
713	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
714	{
715	#ifdef RT_ARCH_AMD64
716	/* mov mem64, reg64 */
717	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
718	if (iGpr < 8)
719	pbCodeBuf[off++] = X86_OP_REX_W;
720	else
721	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
722	pbCodeBuf[off++] = 0x89;
723	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
724	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
725
726	#elif defined(RT_ARCH_ARM64)
727	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
728
729	#else
730	# error "port me"
731	#endif
732	return off;
733	}
734
735
736	/**
737	* Emits a store of a GPR value to a 32-bit VCpu field.
738	*/
739	DECL_INLINE_THROW(uint32_t)
740	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
741	{
742	#ifdef RT_ARCH_AMD64
743	/* mov mem32, reg32 */
744	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
745	if (iGpr >= 8)
746	pbCodeBuf[off++] = X86_OP_REX_R;
747	pbCodeBuf[off++] = 0x89;
748	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
749	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
750
751	#elif defined(RT_ARCH_ARM64)
752	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
753
754	#else
755	# error "port me"
756	#endif
757	return off;
758	}
759
760
761	/**
762	* Emits a store of a GPR value to a 16-bit VCpu field.
763	*/
764	DECL_INLINE_THROW(uint32_t)
765	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
766	{
767	#ifdef RT_ARCH_AMD64
768	/* mov mem16, reg16 */
769	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
770	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
771	if (iGpr >= 8)
772	pbCodeBuf[off++] = X86_OP_REX_R;
773	pbCodeBuf[off++] = 0x89;
774	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
775	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
776
777	#elif defined(RT_ARCH_ARM64)
778	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
779
780	#else
781	# error "port me"
782	#endif
783	return off;
784	}
785
786
787	/**
788	* Emits a store of a GPR value to a 8-bit VCpu field.
789	*/
790	DECL_INLINE_THROW(uint32_t)
791	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
792	{
793	#ifdef RT_ARCH_AMD64
794	/* mov mem8, reg8 */
795	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
796	if (iGpr >= 8)
797	pbCodeBuf[off++] = X86_OP_REX_R;
798	pbCodeBuf[off++] = 0x88;
799	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
800	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
801
802	#elif defined(RT_ARCH_ARM64)
803	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
804
805	#else
806	# error "port me"
807	#endif
808	return off;
809	}
810
811
812	/**
813	* Emits a store of an immediate value to a 8-bit VCpu field.
814	*/
815	DECL_INLINE_THROW(uint32_t)
816	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
817	{
818	#ifdef RT_ARCH_AMD64
819	/* mov mem8, imm8 */
820	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
821	pbCodeBuf[off++] = 0xc6;
822	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
823	pbCodeBuf[off++] = bImm;
824	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
825
826	#elif defined(RT_ARCH_ARM64)
827	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
828	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
829	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
830	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
831
832	#else
833	# error "port me"
834	#endif
835	return off;
836	}
837
838
839	/**
840	* Emits a load effective address to a GRP of a VCpu field.
841	*/
842	DECL_INLINE_THROW(uint32_t)
843	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
844	{
845	#ifdef RT_ARCH_AMD64
846	/* lea gprdst, [rbx + offDisp] */
847	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
848	if (iGprDst < 8)
849	pbCodeBuf[off++] = X86_OP_REX_W;
850	else
851	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
852	pbCodeBuf[off++] = 0x8d;
853	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
854
855	#elif defined(RT_ARCH_ARM64)
856	if (offVCpu < (unsigned)_4K)
857	{
858	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
859	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
860	}
861	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
862	{
863	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
864	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
865	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
866	}
867	else
868	{
869	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
870	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
871	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
872	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
873	}
874
875	#else
876	# error "port me"
877	#endif
878	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
879	return off;
880	}
881
882
883	/**
884	* Emits a gprdst = gprsrc load.
885	*/
886	DECL_FORCE_INLINE(uint32_t)
887	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
888	{
889	#ifdef RT_ARCH_AMD64
890	/* mov gprdst, gprsrc */
891	if ((iGprDst \| iGprSrc) >= 8)
892	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
893	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
894	: X86_OP_REX_W \| X86_OP_REX_R;
895	else
896	pCodeBuf[off++] = X86_OP_REX_W;
897	pCodeBuf[off++] = 0x8b;
898	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
899
900	#elif defined(RT_ARCH_ARM64)
901	/* mov dst, src; alias for: orr dst, xzr, src */
902	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
903
904	#else
905	# error "port me"
906	#endif
907	return off;
908	}
909
910
911	/**
912	* Emits a gprdst = gprsrc load.
913	*/
914	DECL_INLINE_THROW(uint32_t)
915	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
916	{
917	#ifdef RT_ARCH_AMD64
918	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
919	#elif defined(RT_ARCH_ARM64)
920	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
921	#else
922	# error "port me"
923	#endif
924	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
925	return off;
926	}
927
928
929	/**
930	* Emits a gprdst = gprsrc[31:0] load.
931	* @note Bits 63 thru 32 are cleared.
932	*/
933	DECL_FORCE_INLINE(uint32_t)
934	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
935	{
936	#ifdef RT_ARCH_AMD64
937	/* mov gprdst, gprsrc */
938	if ((iGprDst \| iGprSrc) >= 8)
939	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
940	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
941	: X86_OP_REX_R;
942	pCodeBuf[off++] = 0x8b;
943	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
944
945	#elif defined(RT_ARCH_ARM64)
946	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
947	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
948
949	#else
950	# error "port me"
951	#endif
952	return off;
953	}
954
955
956	/**
957	* Emits a gprdst = gprsrc[31:0] load.
958	* @note Bits 63 thru 32 are cleared.
959	*/
960	DECL_INLINE_THROW(uint32_t)
961	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
962	{
963	#ifdef RT_ARCH_AMD64
964	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
965	#elif defined(RT_ARCH_ARM64)
966	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
967	#else
968	# error "port me"
969	#endif
970	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
971	return off;
972	}
973
974
975	/**
976	* Emits a gprdst = gprsrc[15:0] load.
977	* @note Bits 63 thru 15 are cleared.
978	*/
979	DECL_INLINE_THROW(uint32_t)
980	iemNativeEmitLoadGprFromGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
981	{
982	#ifdef RT_ARCH_AMD64
983	/* movzx Gv,Ew */
984	if ((iGprDst \| iGprSrc) >= 8)
985	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
986	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
987	: X86_OP_REX_R;
988	pCodeBuf[off++] = 0x0f;
989	pCodeBuf[off++] = 0xb7;
990	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
991
992	#elif defined(RT_ARCH_ARM64)
993	/* and gprdst, gprsrc, #0xffff */
994	# if 1
995	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
996	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
997	# else
998	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
999	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1000	# endif
1001
1002	#else
1003	# error "port me"
1004	#endif
1005	return off;
1006	}
1007
1008
1009	/**
1010	* Emits a gprdst = gprsrc[15:0] load.
1011	* @note Bits 63 thru 15 are cleared.
1012	*/
1013	DECL_INLINE_THROW(uint32_t)
1014	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1015	{
1016	#ifdef RT_ARCH_AMD64
1017	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1018	#elif defined(RT_ARCH_ARM64)
1019	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1020	#else
1021	# error "port me"
1022	#endif
1023	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1024	return off;
1025	}
1026
1027
1028	/**
1029	* Emits a gprdst = gprsrc[7:0] load.
1030	* @note Bits 63 thru 8 are cleared.
1031	*/
1032	DECL_FORCE_INLINE(uint32_t)
1033	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1034	{
1035	#ifdef RT_ARCH_AMD64
1036	/* movzx Gv,Eb */
1037	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1038	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1039	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1040	: X86_OP_REX_R;
1041	else if (iGprSrc >= 4)
1042	pCodeBuf[off++] = X86_OP_REX;
1043	pCodeBuf[off++] = 0x0f;
1044	pCodeBuf[off++] = 0xb6;
1045	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1046
1047	#elif defined(RT_ARCH_ARM64)
1048	/* and gprdst, gprsrc, #0xff */
1049	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1050	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
1051
1052	#else
1053	# error "port me"
1054	#endif
1055	return off;
1056	}
1057
1058
1059	/**
1060	* Emits a gprdst = gprsrc[7:0] load.
1061	* @note Bits 63 thru 8 are cleared.
1062	*/
1063	DECL_INLINE_THROW(uint32_t)
1064	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1065	{
1066	#ifdef RT_ARCH_AMD64
1067	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1068	#elif defined(RT_ARCH_ARM64)
1069	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1070	#else
1071	# error "port me"
1072	#endif
1073	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1074	return off;
1075	}
1076
1077
1078	/**
1079	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1080	* @note Bits 63 thru 8 are cleared.
1081	*/
1082	DECL_INLINE_THROW(uint32_t)
1083	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1084	{
1085	#ifdef RT_ARCH_AMD64
1086	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1087
1088	/* movzx Gv,Ew */
1089	if ((iGprDst \| iGprSrc) >= 8)
1090	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1091	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1092	: X86_OP_REX_R;
1093	pbCodeBuf[off++] = 0x0f;
1094	pbCodeBuf[off++] = 0xb7;
1095	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1096
1097	/* shr Ev,8 */
1098	if (iGprDst >= 8)
1099	pbCodeBuf[off++] = X86_OP_REX_B;
1100	pbCodeBuf[off++] = 0xc1;
1101	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1102	pbCodeBuf[off++] = 8;
1103
1104	#elif defined(RT_ARCH_ARM64)
1105	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1106	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1107	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1108
1109	#else
1110	# error "port me"
1111	#endif
1112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1113	return off;
1114	}
1115
1116
1117	/**
1118	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1119	*/
1120	DECL_INLINE_THROW(uint32_t)
1121	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1122	{
1123	#ifdef RT_ARCH_AMD64
1124	/* movsxd r64, r/m32 */
1125	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1126	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1127	pbCodeBuf[off++] = 0x63;
1128	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1129
1130	#elif defined(RT_ARCH_ARM64)
1131	/* sxtw dst, src */
1132	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1133	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1134
1135	#else
1136	# error "port me"
1137	#endif
1138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1139	return off;
1140	}
1141
1142
1143	/**
1144	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1145	*/
1146	DECL_INLINE_THROW(uint32_t)
1147	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1148	{
1149	#ifdef RT_ARCH_AMD64
1150	/* movsx r64, r/m16 */
1151	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1152	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1153	pbCodeBuf[off++] = 0x0f;
1154	pbCodeBuf[off++] = 0xbf;
1155	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1156
1157	#elif defined(RT_ARCH_ARM64)
1158	/* sxth dst, src */
1159	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1160	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1161
1162	#else
1163	# error "port me"
1164	#endif
1165	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1166	return off;
1167	}
1168
1169
1170	/**
1171	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1172	*/
1173	DECL_INLINE_THROW(uint32_t)
1174	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1175	{
1176	#ifdef RT_ARCH_AMD64
1177	/* movsx r64, r/m16 */
1178	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1179	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1180	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1181	pbCodeBuf[off++] = 0x0f;
1182	pbCodeBuf[off++] = 0xbf;
1183	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1184
1185	#elif defined(RT_ARCH_ARM64)
1186	/* sxth dst32, src */
1187	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1188	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1189
1190	#else
1191	# error "port me"
1192	#endif
1193	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1194	return off;
1195	}
1196
1197
1198	/**
1199	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1200	*/
1201	DECL_INLINE_THROW(uint32_t)
1202	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1203	{
1204	#ifdef RT_ARCH_AMD64
1205	/* movsx r64, r/m8 */
1206	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1207	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1208	pbCodeBuf[off++] = 0x0f;
1209	pbCodeBuf[off++] = 0xbe;
1210	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1211
1212	#elif defined(RT_ARCH_ARM64)
1213	/* sxtb dst, src */
1214	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1215	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1216
1217	#else
1218	# error "port me"
1219	#endif
1220	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1221	return off;
1222	}
1223
1224
1225	/**
1226	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1227	* @note Bits 63 thru 32 are cleared.
1228	*/
1229	DECL_INLINE_THROW(uint32_t)
1230	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1231	{
1232	#ifdef RT_ARCH_AMD64
1233	/* movsx r32, r/m8 */
1234	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1235	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1236	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1237	else if (iGprSrc >= 4)
1238	pbCodeBuf[off++] = X86_OP_REX;
1239	pbCodeBuf[off++] = 0x0f;
1240	pbCodeBuf[off++] = 0xbe;
1241	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1242
1243	#elif defined(RT_ARCH_ARM64)
1244	/* sxtb dst32, src32 */
1245	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1246	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1247
1248	#else
1249	# error "port me"
1250	#endif
1251	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1252	return off;
1253	}
1254
1255
1256	/**
1257	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1258	* @note Bits 63 thru 16 are cleared.
1259	*/
1260	DECL_INLINE_THROW(uint32_t)
1261	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1262	{
1263	#ifdef RT_ARCH_AMD64
1264	/* movsx r16, r/m8 */
1265	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1266	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1267	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1268	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1269	else if (iGprSrc >= 4)
1270	pbCodeBuf[off++] = X86_OP_REX;
1271	pbCodeBuf[off++] = 0x0f;
1272	pbCodeBuf[off++] = 0xbe;
1273	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1274
1275	/* movzx r32, r/m16 */
1276	if (iGprDst >= 8)
1277	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1278	pbCodeBuf[off++] = 0x0f;
1279	pbCodeBuf[off++] = 0xb7;
1280	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1281
1282	#elif defined(RT_ARCH_ARM64)
1283	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1284	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1285	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1286	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1287	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1288
1289	#else
1290	# error "port me"
1291	#endif
1292	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1293	return off;
1294	}
1295
1296
1297	/**
1298	* Emits a gprdst = gprsrc + addend load.
1299	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1300	*/
1301	#ifdef RT_ARCH_AMD64
1302	DECL_INLINE_THROW(uint32_t)
1303	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1304	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1305	{
1306	Assert(iAddend != 0);
1307
1308	/* lea gprdst, [gprsrc + iAddend] */
1309	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1310	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1311	pbCodeBuf[off++] = 0x8d;
1312	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1313	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1314	return off;
1315	}
1316
1317	#elif defined(RT_ARCH_ARM64)
1318	DECL_INLINE_THROW(uint32_t)
1319	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1320	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1321	{
1322	if ((uint32_t)iAddend < 4096)
1323	{
1324	/* add dst, src, uimm12 */
1325	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1326	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1327	}
1328	else if ((uint32_t)-iAddend < 4096)
1329	{
1330	/* sub dst, src, uimm12 */
1331	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1332	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1333	}
1334	else
1335	{
1336	Assert(iGprSrc != iGprDst);
1337	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1338	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1339	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1340	}
1341	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1342	return off;
1343	}
1344	#else
1345	# error "port me"
1346	#endif
1347
1348	/**
1349	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1350	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1351	*/
1352	#ifdef RT_ARCH_AMD64
1353	DECL_INLINE_THROW(uint32_t)
1354	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1355	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1356	#else
1357	DECL_INLINE_THROW(uint32_t)
1358	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1359	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1360	#endif
1361	{
1362	if (iAddend != 0)
1363	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1364	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1365	}
1366
1367
1368	/**
1369	* Emits a gprdst = gprsrc32 + addend load.
1370	* @note Bits 63 thru 32 are cleared.
1371	*/
1372	DECL_INLINE_THROW(uint32_t)
1373	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1374	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1375	{
1376	Assert(iAddend != 0);
1377
1378	#ifdef RT_ARCH_AMD64
1379	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1380	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1381	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1382	if ((iGprDst \| iGprSrc) >= 8)
1383	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1384	pbCodeBuf[off++] = 0x8d;
1385	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1386
1387	#elif defined(RT_ARCH_ARM64)
1388	if ((uint32_t)iAddend < 4096)
1389	{
1390	/* add dst, src, uimm12 */
1391	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1392	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1393	}
1394	else if ((uint32_t)-iAddend < 4096)
1395	{
1396	/* sub dst, src, uimm12 */
1397	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1398	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1399	}
1400	else
1401	{
1402	Assert(iGprSrc != iGprDst);
1403	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1404	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1405	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1406	}
1407
1408	#else
1409	# error "port me"
1410	#endif
1411	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1412	return off;
1413	}
1414
1415
1416	/**
1417	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1418	*/
1419	DECL_INLINE_THROW(uint32_t)
1420	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1421	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1422	{
1423	if (iAddend != 0)
1424	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1425	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1426	}
1427
1428
1429
1430	#ifdef RT_ARCH_AMD64
1431	/**
1432	* Common bit of iemNativeEmitLoadGprByBp and friends.
1433	*/
1434	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1435	PIEMRECOMPILERSTATE pReNativeAssert)
1436	{
1437	if (offDisp < 128 && offDisp >= -128)
1438	{
1439	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1440	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1441	}
1442	else
1443	{
1444	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1445	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1446	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1447	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1448	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1449	}
1450	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1451	return off;
1452	}
1453	#elif defined(RT_ARCH_ARM64)
1454	/**
1455	* Common bit of iemNativeEmitLoadGprByBp and friends.
1456	*/
1457	DECL_FORCE_INLINE_THROW(uint32_t)
1458	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1459	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1460	{
1461	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1462	{
1463	/* str w/ unsigned imm12 (scaled) */
1464	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1465	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1466	}
1467	else if (offDisp >= -256 && offDisp <= 256)
1468	{
1469	/* stur w/ signed imm9 (unscaled) */
1470	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1471	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1472	}
1473	else
1474	{
1475	/* Use temporary indexing register. */
1476	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1477	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1478	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1479	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1480	}
1481	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1482	return off;
1483	}
1484	#endif
1485
1486
1487	/**
1488	* Emits a 64-bit GRP load instruction with an BP relative source address.
1489	*/
1490	DECL_INLINE_THROW(uint32_t)
1491	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1492	{
1493	#ifdef RT_ARCH_AMD64
1494	/* mov gprdst, qword [rbp + offDisp] */
1495	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1496	if (iGprDst < 8)
1497	pbCodeBuf[off++] = X86_OP_REX_W;
1498	else
1499	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1500	pbCodeBuf[off++] = 0x8b;
1501	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1502
1503	#elif defined(RT_ARCH_ARM64)
1504	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1505
1506	#else
1507	# error "port me"
1508	#endif
1509	}
1510
1511
1512	/**
1513	* Emits a 32-bit GRP load instruction with an BP relative source address.
1514	* @note Bits 63 thru 32 of the GPR will be cleared.
1515	*/
1516	DECL_INLINE_THROW(uint32_t)
1517	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1518	{
1519	#ifdef RT_ARCH_AMD64
1520	/* mov gprdst, dword [rbp + offDisp] */
1521	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1522	if (iGprDst >= 8)
1523	pbCodeBuf[off++] = X86_OP_REX_R;
1524	pbCodeBuf[off++] = 0x8b;
1525	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1526
1527	#elif defined(RT_ARCH_ARM64)
1528	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1529
1530	#else
1531	# error "port me"
1532	#endif
1533	}
1534
1535
1536	/**
1537	* Emits a 16-bit GRP load instruction with an BP relative source address.
1538	* @note Bits 63 thru 16 of the GPR will be cleared.
1539	*/
1540	DECL_INLINE_THROW(uint32_t)
1541	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1542	{
1543	#ifdef RT_ARCH_AMD64
1544	/* movzx gprdst, word [rbp + offDisp] */
1545	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1546	if (iGprDst >= 8)
1547	pbCodeBuf[off++] = X86_OP_REX_R;
1548	pbCodeBuf[off++] = 0x0f;
1549	pbCodeBuf[off++] = 0xb7;
1550	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1551
1552	#elif defined(RT_ARCH_ARM64)
1553	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1554
1555	#else
1556	# error "port me"
1557	#endif
1558	}
1559
1560
1561	/**
1562	* Emits a 8-bit GRP load instruction with an BP relative source address.
1563	* @note Bits 63 thru 8 of the GPR will be cleared.
1564	*/
1565	DECL_INLINE_THROW(uint32_t)
1566	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1567	{
1568	#ifdef RT_ARCH_AMD64
1569	/* movzx gprdst, byte [rbp + offDisp] */
1570	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1571	if (iGprDst >= 8)
1572	pbCodeBuf[off++] = X86_OP_REX_R;
1573	pbCodeBuf[off++] = 0x0f;
1574	pbCodeBuf[off++] = 0xb6;
1575	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1576
1577	#elif defined(RT_ARCH_ARM64)
1578	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1579
1580	#else
1581	# error "port me"
1582	#endif
1583	}
1584
1585
1586	/**
1587	* Emits a load effective address to a GRP with an BP relative source address.
1588	*/
1589	DECL_INLINE_THROW(uint32_t)
1590	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1591	{
1592	#ifdef RT_ARCH_AMD64
1593	/* lea gprdst, [rbp + offDisp] */
1594	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1595	if (iGprDst < 8)
1596	pbCodeBuf[off++] = X86_OP_REX_W;
1597	else
1598	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1599	pbCodeBuf[off++] = 0x8d;
1600	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1601
1602	#elif defined(RT_ARCH_ARM64)
1603	if ((uint32_t)offDisp < (unsigned)_4K)
1604	{
1605	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1606	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1607	}
1608	else if ((uint32_t)-offDisp < (unsigned)_4K)
1609	{
1610	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1611	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1612	}
1613	else
1614	{
1615	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1616	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1617	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1618	if (offDisp >= 0)
1619	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1620	else
1621	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1622	}
1623
1624	#else
1625	# error "port me"
1626	#endif
1627
1628	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1629	return off;
1630	}
1631
1632
1633	/**
1634	* Emits a 64-bit GPR store with an BP relative destination address.
1635	*
1636	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1637	*/
1638	DECL_INLINE_THROW(uint32_t)
1639	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1640	{
1641	#ifdef RT_ARCH_AMD64
1642	/* mov qword [rbp + offDisp], gprdst */
1643	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1644	if (iGprSrc < 8)
1645	pbCodeBuf[off++] = X86_OP_REX_W;
1646	else
1647	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1648	pbCodeBuf[off++] = 0x89;
1649	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1650
1651	#elif defined(RT_ARCH_ARM64)
1652	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1653	{
1654	/* str w/ unsigned imm12 (scaled) */
1655	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1656	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1657	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1658	}
1659	else if (offDisp >= -256 && offDisp <= 256)
1660	{
1661	/* stur w/ signed imm9 (unscaled) */
1662	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1663	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1664	}
1665	else if ((uint32_t)-offDisp < (unsigned)_4K)
1666	{
1667	/* Use temporary indexing register w/ sub uimm12. */
1668	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1669	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1670	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1671	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1672	}
1673	else
1674	{
1675	/* Use temporary indexing register. */
1676	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1677	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1678	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1679	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1680	}
1681	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1682	return off;
1683
1684	#else
1685	# error "Port me!"
1686	#endif
1687	}
1688
1689
1690	/**
1691	* Emits a 64-bit immediate store with an BP relative destination address.
1692	*
1693	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1694	*/
1695	DECL_INLINE_THROW(uint32_t)
1696	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1697	{
1698	#ifdef RT_ARCH_AMD64
1699	if ((int64_t)uImm64 == (int32_t)uImm64)
1700	{
1701	/* mov qword [rbp + offDisp], imm32 - sign extended */
1702	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1703	pbCodeBuf[off++] = X86_OP_REX_W;
1704	pbCodeBuf[off++] = 0xc7;
1705	if (offDisp < 128 && offDisp >= -128)
1706	{
1707	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1708	pbCodeBuf[off++] = (uint8_t)offDisp;
1709	}
1710	else
1711	{
1712	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1713	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1714	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1715	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1716	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1717	}
1718	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1719	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1720	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1721	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1722	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1723	return off;
1724	}
1725	#endif
1726
1727	/* Load tmp0, imm64; Store tmp to bp+disp. */
1728	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1729	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1730	}
1731
1732	#if defined(RT_ARCH_ARM64)
1733
1734	/**
1735	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1736	*
1737	* @note Odd and large @a offDisp values requires a temporary, unless it's a
1738	* load and @a iGprReg differs from @a iGprBase. Will assert / throw if
1739	* caller does not heed this.
1740	*
1741	* @note DON'T try this with prefetch.
1742	*/
1743	DECL_FORCE_INLINE_THROW(uint32_t)
1744	iemNativeEmitGprByGprLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp,
1745	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
1746	{
1747	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1748	{
1749	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1750	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1751	}
1752	else if ( ( !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation)
1753	&& iGprReg != iGprBase)
1754	\|\| iGprTmp != UINT8_MAX)
1755	{
1756	/* The offset is too large, so we must load it into a register and use
1757	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1758	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1759	if (iGprTmp == UINT8_MAX)
1760	iGprTmp = iGprReg;
1761	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (int64_t)offDisp);
1762	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, iGprTmp);
1763	}
1764	else
1765	# ifdef IEM_WITH_THROW_CATCH
1766	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
1767	# else
1768	AssertReleaseFailedStmt(off = UINT32_MAX);
1769	# endif
1770	return off;
1771	}
1772
1773	/**
1774	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1775	*/
1776	DECL_FORCE_INLINE_THROW(uint32_t)
1777	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1778	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1779	{
1780	/*
1781	* There are a couple of ldr variants that takes an immediate offset, so
1782	* try use those if we can, otherwise we have to use the temporary register
1783	* help with the addressing.
1784	*/
1785	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1786	{
1787	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1788	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1789	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1790	}
1791	else
1792	{
1793	/* The offset is too large, so we must load it into a register and use
1794	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1795	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1796	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (int64_t)offDisp);
1797
1798	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1799	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1800
1801	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1802	}
1803	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1804	return off;
1805	}
1806
1807	#endif /* RT_ARCH_ARM64 */
1808
1809	/**
1810	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1811	*
1812	* @note ARM64: Misaligned @a offDisp values and values not in the
1813	* -0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
1814	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
1815	* does not heed this.
1816	*/
1817	DECL_FORCE_INLINE_THROW(uint32_t)
1818	iemNativeEmitLoadGprByGprU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1819	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1820	{
1821	#ifdef RT_ARCH_AMD64
1822	/* mov reg64, mem64 */
1823	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1824	pCodeBuf[off++] = 0x8b;
1825	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1826	RT_NOREF(iGprTmp);
1827
1828	#elif defined(RT_ARCH_ARM64)
1829	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1830	kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t), iGprTmp);
1831
1832	#else
1833	# error "port me"
1834	#endif
1835	return off;
1836	}
1837
1838
1839	/**
1840	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1841	*/
1842	DECL_INLINE_THROW(uint32_t)
1843	iemNativeEmitLoadGprByGprU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1844	{
1845	#ifdef RT_ARCH_AMD64
1846	off = iemNativeEmitLoadGprByGprU64Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
1847	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1848
1849	#elif defined(RT_ARCH_ARM64)
1850	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1851
1852	#else
1853	# error "port me"
1854	#endif
1855	return off;
1856	}
1857
1858
1859	/**
1860	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1861	*
1862	* @note ARM64: Misaligned @a offDisp values and values not in the
1863	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
1864	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1865	* caller does not heed this.
1866	*
1867	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1868	*/
1869	DECL_FORCE_INLINE_THROW(uint32_t)
1870	iemNativeEmitLoadGprByGprU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1871	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1872	{
1873	#ifdef RT_ARCH_AMD64
1874	/* mov reg32, mem32 */
1875	if (iGprDst >= 8 \|\| iGprBase >= 8)
1876	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1877	pCodeBuf[off++] = 0x8b;
1878	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1879	RT_NOREF(iGprTmp);
1880
1881	#elif defined(RT_ARCH_ARM64)
1882	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1883	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t), iGprTmp);
1884
1885	#else
1886	# error "port me"
1887	#endif
1888	return off;
1889	}
1890
1891
1892	/**
1893	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1894	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1895	*/
1896	DECL_INLINE_THROW(uint32_t)
1897	iemNativeEmitLoadGprByGprU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1898	{
1899	#ifdef RT_ARCH_AMD64
1900	off = iemNativeEmitLoadGprByGprU32Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
1901	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1902
1903	#elif defined(RT_ARCH_ARM64)
1904	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1905
1906	#else
1907	# error "port me"
1908	#endif
1909	return off;
1910	}
1911
1912
1913	/**
1914	* Emits a 32-bit GPR load via a GPR base address with a displacement,
1915	* sign-extending the value to 64 bits.
1916	*
1917	* @note ARM64: Misaligned @a offDisp values and values not in the
1918	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
1919	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1920	* caller does not heed this.
1921	*/
1922	DECL_FORCE_INLINE_THROW(uint32_t)
1923	iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1924	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1925	{
1926	#ifdef RT_ARCH_AMD64
1927	/* movsxd reg64, mem32 */
1928	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1929	pCodeBuf[off++] = 0x63;
1930	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1931	RT_NOREF(iGprTmp);
1932
1933	#elif defined(RT_ARCH_ARM64)
1934	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1935	kArmv8A64InstrLdStType_Ld_SignWord64, sizeof(uint32_t), iGprTmp);
1936
1937	#else
1938	# error "port me"
1939	#endif
1940	return off;
1941	}
1942
1943
1944	/**
1945	* Emits a 16-bit GPR load via a GPR base address with a displacement.
1946	*
1947	* @note ARM64: Misaligned @a offDisp values and values not in the
1948	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
1949	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1950	* caller does not heed this.
1951	*
1952	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
1953	*/
1954	DECL_FORCE_INLINE_THROW(uint32_t)
1955	iemNativeEmitLoadGprByGprU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1956	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1957	{
1958	#ifdef RT_ARCH_AMD64
1959	/* movzx reg32, mem16 */
1960	if (iGprDst >= 8 \|\| iGprBase >= 8)
1961	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1962	pCodeBuf[off++] = 0x0f;
1963	pCodeBuf[off++] = 0xb7;
1964	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1965	RT_NOREF(iGprTmp);
1966
1967	#elif defined(RT_ARCH_ARM64)
1968	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1969	kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t), iGprTmp);
1970
1971	#else
1972	# error "port me"
1973	#endif
1974	return off;
1975	}
1976
1977
1978	/**
1979	* Emits a 16-bit GPR load via a GPR base address with a displacement,
1980	* sign-extending the value to 64 bits.
1981	*
1982	* @note ARM64: Misaligned @a offDisp values and values not in the
1983	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
1984	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1985	* caller does not heed this.
1986	*/
1987	DECL_FORCE_INLINE_THROW(uint32_t)
1988	iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1989	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1990	{
1991	#ifdef RT_ARCH_AMD64
1992	/* movsx reg64, mem16 */
1993	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1994	pCodeBuf[off++] = 0x0f;
1995	pCodeBuf[off++] = 0xbf;
1996	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1997	RT_NOREF(iGprTmp);
1998
1999	#elif defined(RT_ARCH_ARM64)
2000	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2001	kArmv8A64InstrLdStType_Ld_SignHalf64, sizeof(uint16_t), iGprTmp);
2002
2003	#else
2004	# error "port me"
2005	#endif
2006	return off;
2007	}
2008
2009
2010	/**
2011	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2012	* sign-extending the value to 32 bits.
2013	*
2014	* @note ARM64: Misaligned @a offDisp values and values not in the
2015	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2016	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2017	* caller does not heed this.
2018	*
2019	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2020	*/
2021	DECL_FORCE_INLINE_THROW(uint32_t)
2022	iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2023	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2024	{
2025	#ifdef RT_ARCH_AMD64
2026	/* movsx reg32, mem16 */
2027	if (iGprDst >= 8 \|\| iGprBase >= 8)
2028	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2029	pCodeBuf[off++] = 0x0f;
2030	pCodeBuf[off++] = 0xbf;
2031	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2032	RT_NOREF(iGprTmp);
2033
2034	#elif defined(RT_ARCH_ARM64)
2035	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2036	kArmv8A64InstrLdStType_Ld_SignHalf32, sizeof(uint16_t), iGprTmp);
2037
2038	#else
2039	# error "port me"
2040	#endif
2041	return off;
2042	}
2043
2044
2045	/**
2046	* Emits a 8-bit GPR load via a GPR base address with a displacement.
2047	*
2048	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2049	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2050	* same. Will assert / throw if caller does not heed this.
2051	*
2052	* @note Bits 63 thru 8 in @a iGprDst will be cleared.
2053	*/
2054	DECL_FORCE_INLINE_THROW(uint32_t)
2055	iemNativeEmitLoadGprByGprU8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2056	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2057	{
2058	#ifdef RT_ARCH_AMD64
2059	/* movzx reg32, mem8 */
2060	if (iGprDst >= 8 \|\| iGprBase >= 8)
2061	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2062	pCodeBuf[off++] = 0x0f;
2063	pCodeBuf[off++] = 0xb6;
2064	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2065	RT_NOREF(iGprTmp);
2066
2067	#elif defined(RT_ARCH_ARM64)
2068	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2069	kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t), iGprTmp);
2070
2071	#else
2072	# error "port me"
2073	#endif
2074	return off;
2075	}
2076
2077
2078	/**
2079	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2080	* sign-extending the value to 64 bits.
2081	*
2082	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2083	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2084	* same. Will assert / throw if caller does not heed this.
2085	*/
2086	DECL_FORCE_INLINE_THROW(uint32_t)
2087	iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2088	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2089	{
2090	#ifdef RT_ARCH_AMD64
2091	/* movsx reg64, mem8 */
2092	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2093	pCodeBuf[off++] = 0x0f;
2094	pCodeBuf[off++] = 0xbe;
2095	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2096	RT_NOREF(iGprTmp);
2097
2098	#elif defined(RT_ARCH_ARM64)
2099	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2100	kArmv8A64InstrLdStType_Ld_SignByte64, sizeof(uint8_t), iGprTmp);
2101
2102	#else
2103	# error "port me"
2104	#endif
2105	return off;
2106	}
2107
2108
2109	/**
2110	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2111	* sign-extending the value to 32 bits.
2112	*
2113	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2114	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2115	* same. Will assert / throw if caller does not heed this.
2116	*
2117	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2118	*/
2119	DECL_FORCE_INLINE_THROW(uint32_t)
2120	iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2121	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2122	{
2123	#ifdef RT_ARCH_AMD64
2124	/* movsx reg32, mem8 */
2125	if (iGprDst >= 8 \|\| iGprBase >= 8)
2126	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2127	pCodeBuf[off++] = 0x0f;
2128	pCodeBuf[off++] = 0xbe;
2129	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2130	RT_NOREF(iGprTmp);
2131
2132	#elif defined(RT_ARCH_ARM64)
2133	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2134	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2135
2136	#else
2137	# error "port me"
2138	#endif
2139	return off;
2140	}
2141
2142
2143	/**
2144	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2145	* sign-extending the value to 16 bits.
2146	*
2147	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2148	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2149	* same. Will assert / throw if caller does not heed this.
2150	*
2151	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2152	*/
2153	DECL_FORCE_INLINE_THROW(uint32_t)
2154	iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2155	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2156	{
2157	#ifdef RT_ARCH_AMD64
2158	/* movsx reg32, mem8 */
2159	if (iGprDst >= 8 \|\| iGprBase >= 8)
2160	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2161	pCodeBuf[off++] = 0x0f;
2162	pCodeBuf[off++] = 0xbe;
2163	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2164	# if 1 /** @todo use 'movzx reg32, reg16' instead of 'and reg32, 0ffffh' ? */
2165	/* and reg32, 0xffffh */
2166	if (iGprDst >= 8)
2167	pCodeBuf[off++] = X86_OP_REX_B;
2168	pCodeBuf[off++] = 0x81;
2169	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2170	pCodeBuf[off++] = 0xff;
2171	pCodeBuf[off++] = 0xff;
2172	pCodeBuf[off++] = 0;
2173	pCodeBuf[off++] = 0;
2174	# else
2175	/* movzx reg32, reg16 */
2176	if (iGprDst >= 8)
2177	pCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2178	pCodeBuf[off++] = 0x0f;
2179	pCodeBuf[off++] = 0xb7;
2180	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2181	# endif
2182	RT_NOREF(iGprTmp);
2183
2184	#elif defined(RT_ARCH_ARM64)
2185	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2186	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2187	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
2188	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /64Bit/);
2189
2190	#else
2191	# error "port me"
2192	#endif
2193	return off;
2194	}
2195
2196
2197	/**
2198	* Emits a 64-bit GPR store via a GPR base address with a displacement.
2199	*
2200	* @note ARM64: Misaligned @a offDisp values and values not in the
2201	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2202	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2203	* does not heed this.
2204	*/
2205	DECL_FORCE_INLINE_THROW(uint32_t)
2206	iemNativeEmitStoreGpr64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2207	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2208	{
2209	#ifdef RT_ARCH_AMD64
2210	/* mov mem64, reg64 */
2211	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2212	pCodeBuf[off++] = 0x89;
2213	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2214	RT_NOREF(iGprTmp);
2215
2216	#elif defined(RT_ARCH_ARM64)
2217	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2218	kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
2219
2220	#else
2221	# error "port me"
2222	#endif
2223	return off;
2224	}
2225
2226
2227	/**
2228	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2229	*
2230	* @note ARM64: Misaligned @a offDisp values and values not in the
2231	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2232	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2233	* does not heed this.
2234	*/
2235	DECL_FORCE_INLINE_THROW(uint32_t)
2236	iemNativeEmitStoreGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2237	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2238	{
2239	#ifdef RT_ARCH_AMD64
2240	/* mov mem32, reg32 */
2241	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2242	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2243	pCodeBuf[off++] = 0x89;
2244	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2245	RT_NOREF(iGprTmp);
2246
2247	#elif defined(RT_ARCH_ARM64)
2248	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2249	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2250
2251	#else
2252	# error "port me"
2253	#endif
2254	return off;
2255	}
2256
2257
2258	/**
2259	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2260	*
2261	* @note ARM64: Misaligned @a offDisp values and values not in the
2262	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2263	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2264	* does not heed this.
2265	*/
2266	DECL_FORCE_INLINE_THROW(uint32_t)
2267	iemNativeEmitStoreGpr16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2268	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2269	{
2270	#ifdef RT_ARCH_AMD64
2271	/* mov mem16, reg16 */
2272	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2273	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2274	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2275	pCodeBuf[off++] = 0x89;
2276	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2277	RT_NOREF(iGprTmp);
2278
2279	#elif defined(RT_ARCH_ARM64)
2280	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2281	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2282
2283	#else
2284	# error "port me"
2285	#endif
2286	return off;
2287	}
2288
2289
2290	/**
2291	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2292	*
2293	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2294	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2295	* same. Will assert / throw if caller does not heed this.
2296	*/
2297	DECL_FORCE_INLINE_THROW(uint32_t)
2298	iemNativeEmitStoreGpr8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2299	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2300	{
2301	#ifdef RT_ARCH_AMD64
2302	/* mov mem8, reg8 */
2303	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2304	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2305	else if (iGprSrc >= 4)
2306	pCodeBuf[off++] = X86_OP_REX;
2307	pCodeBuf[off++] = 0x88;
2308	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2309	RT_NOREF(iGprTmp);
2310
2311	#elif defined(RT_ARCH_ARM64)
2312	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2313	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2314
2315	#else
2316	# error "port me"
2317	#endif
2318	return off;
2319	}
2320
2321
2322	/**
2323	* Emits a 64-bit immediate store via a GPR base address with a displacement.
2324	*
2325	* @note This will always require @a iGprTmpImm on ARM (except for uImm=0), on
2326	* AMD64 it depends on the immediate value.
2327	*
2328	* @note ARM64: Misaligned @a offDisp values and values not in the
2329	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2330	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2331	* does not heed this.
2332	*/
2333	DECL_FORCE_INLINE_THROW(uint32_t)
2334	iemNativeEmitStoreImm64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint64_t uImm, uint8_t iGprBase,
2335	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2336	{
2337	#ifdef RT_ARCH_AMD64
2338	if ((int32_t)uImm == (int64_t)uImm)
2339	{
2340	/* mov mem64, imm32 (sign-extended) */
2341	pCodeBuf[off++] = X86_OP_REX_W \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2342	pCodeBuf[off++] = 0xc7;
2343	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2344	pCodeBuf[off++] = RT_BYTE1(uImm);
2345	pCodeBuf[off++] = RT_BYTE2(uImm);
2346	pCodeBuf[off++] = RT_BYTE3(uImm);
2347	pCodeBuf[off++] = RT_BYTE4(uImm);
2348	}
2349	else if (iGprImmTmp != UINT8_MAX \|\| iGprTmp != UINT8_MAX)
2350	{
2351	/* require temporary register. */
2352	if (iGprImmTmp == UINT8_MAX)
2353	iGprImmTmp = iGprTmp;
2354	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2355	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp);
2356	}
2357	else
2358	# ifdef IEM_WITH_THROW_CATCH
2359	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2360	# else
2361	AssertReleaseFailedStmt(off = UINT32_MAX);
2362	# endif
2363
2364	#elif defined(RT_ARCH_ARM64)
2365	if (uImm == 0)
2366	iGprImmTmp = ARMV8_A64_REG_XZR;
2367	else
2368	{
2369	Assert(iGprImmTmp < 31);
2370	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2371	}
2372	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp, iGprTmp);
2373
2374	#else
2375	# error "port me"
2376	#endif
2377	return off;
2378	}
2379
2380
2381	/**
2382	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2383	*
2384	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2385	*
2386	* @note ARM64: Misaligned @a offDisp values and values not in the
2387	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2388	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2389	* does not heed this.
2390	*/
2391	DECL_FORCE_INLINE_THROW(uint32_t)
2392	iemNativeEmitStoreImm32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uImm, uint8_t iGprBase,
2393	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2394	{
2395	#ifdef RT_ARCH_AMD64
2396	/* mov mem32, imm32 */
2397	if (iGprBase >= 8)
2398	pCodeBuf[off++] = X86_OP_REX_B;
2399	pCodeBuf[off++] = 0xc7;
2400	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2401	pCodeBuf[off++] = RT_BYTE1(uImm);
2402	pCodeBuf[off++] = RT_BYTE2(uImm);
2403	pCodeBuf[off++] = RT_BYTE3(uImm);
2404	pCodeBuf[off++] = RT_BYTE4(uImm);
2405	RT_NOREF(iGprImmTmp, iGprTmp);
2406
2407	#elif defined(RT_ARCH_ARM64)
2408	Assert(iGprImmTmp < 31);
2409	if (uImm == 0)
2410	iGprImmTmp = ARMV8_A64_REG_XZR;
2411	else
2412	{
2413	Assert(iGprImmTmp < 31);
2414	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2415	}
2416	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2417	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2418
2419	#else
2420	# error "port me"
2421	#endif
2422	return off;
2423	}
2424
2425
2426	/**
2427	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2428	*
2429	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2430	*
2431	* @note ARM64: Misaligned @a offDisp values and values not in the
2432	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2433	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2434	* does not heed this.
2435	*/
2436	DECL_FORCE_INLINE_THROW(uint32_t)
2437	iemNativeEmitStoreImm16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint8_t iGprBase,
2438	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2439	{
2440	#ifdef RT_ARCH_AMD64
2441	/* mov mem16, imm16 */
2442	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2443	if (iGprBase >= 8)
2444	pCodeBuf[off++] = X86_OP_REX_B;
2445	pCodeBuf[off++] = 0xc7;
2446	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2447	pCodeBuf[off++] = RT_BYTE1(uImm);
2448	pCodeBuf[off++] = RT_BYTE2(uImm);
2449	RT_NOREF(iGprImmTmp, iGprTmp);
2450
2451	#elif defined(RT_ARCH_ARM64)
2452	if (uImm == 0)
2453	iGprImmTmp = ARMV8_A64_REG_XZR;
2454	else
2455	{
2456	Assert(iGprImmTmp < 31);
2457	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2458	}
2459	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2460	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2461
2462	#else
2463	# error "port me"
2464	#endif
2465	return off;
2466	}
2467
2468
2469	/**
2470	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2471	*
2472	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2473	*
2474	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2475	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2476	* same. Will assert / throw if caller does not heed this.
2477	*/
2478	DECL_FORCE_INLINE_THROW(uint32_t)
2479	iemNativeEmitStoreImm8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t uImm, uint8_t iGprBase,
2480	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2481	{
2482	#ifdef RT_ARCH_AMD64
2483	/* mov mem8, imm8 */
2484	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2485	if (iGprBase >= 8)
2486	pCodeBuf[off++] = X86_OP_REX_B;
2487	pCodeBuf[off++] = 0xc6;
2488	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2489	pCodeBuf[off++] = uImm;
2490	RT_NOREF(iGprImmTmp, iGprTmp);
2491
2492	#elif defined(RT_ARCH_ARM64)
2493	if (uImm == 0)
2494	iGprImmTmp = ARMV8_A64_REG_XZR;
2495	else
2496	{
2497	Assert(iGprImmTmp < 31);
2498	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2499	}
2500	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2501	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2502
2503	#else
2504	# error "port me"
2505	#endif
2506	return off;
2507	}
2508
2509
2510
2511	/*********************************************************************************************************************************
2512	* Subtraction and Additions *
2513	*********************************************************************************************************************************/
2514
2515	/**
2516	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
2517	* @note The AMD64 version sets flags.
2518	*/
2519	DECL_INLINE_THROW(uint32_t)
2520	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2521	{
2522	#if defined(RT_ARCH_AMD64)
2523	/* sub Gv,Ev */
2524	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2525	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2526	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2527	pbCodeBuf[off++] = 0x2b;
2528	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2529
2530	#elif defined(RT_ARCH_ARM64)
2531	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2532	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
2533
2534	#else
2535	# error "Port me"
2536	#endif
2537	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2538	return off;
2539	}
2540
2541
2542	/**
2543	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2544	* @note The AMD64 version sets flags.
2545	*/
2546	DECL_FORCE_INLINE(uint32_t)
2547	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2548	{
2549	#if defined(RT_ARCH_AMD64)
2550	/* sub Gv,Ev */
2551	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
2552	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
2553	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2554	pCodeBuf[off++] = 0x2b;
2555	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2556
2557	#elif defined(RT_ARCH_ARM64)
2558	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
2559
2560	#else
2561	# error "Port me"
2562	#endif
2563	return off;
2564	}
2565
2566
2567	/**
2568	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2569	* @note The AMD64 version sets flags.
2570	*/
2571	DECL_INLINE_THROW(uint32_t)
2572	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2573	{
2574	#if defined(RT_ARCH_AMD64)
2575	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
2576	#elif defined(RT_ARCH_ARM64)
2577	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
2578	#else
2579	# error "Port me"
2580	#endif
2581	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2582	return off;
2583	}
2584
2585
2586	/**
2587	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2588	*
2589	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2590	*
2591	* @note Larger constants will require a temporary register. Failing to specify
2592	* one when needed will trigger fatal assertion / throw.
2593	*/
2594	DECL_FORCE_INLINE_THROW(uint32_t)
2595	iemNativeEmitSubGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2596	uint8_t iGprTmp = UINT8_MAX)
2597	{
2598	#ifdef RT_ARCH_AMD64
2599	pCodeBuf[off++] = iGprDst >= 8 ? X86_OP_REX_W \| X86_OP_REX_B : X86_OP_REX_W;
2600	if (iSubtrahend == 1)
2601	{
2602	/* dec r/m64 */
2603	pCodeBuf[off++] = 0xff;
2604	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2605	}
2606	else if (iSubtrahend == -1)
2607	{
2608	/* inc r/m64 */
2609	pCodeBuf[off++] = 0xff;
2610	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2611	}
2612	else if ((int8_t)iSubtrahend == iSubtrahend)
2613	{
2614	/* sub r/m64, imm8 */
2615	pCodeBuf[off++] = 0x83;
2616	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2617	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2618	}
2619	else if ((int32_t)iSubtrahend == iSubtrahend)
2620	{
2621	/* sub r/m64, imm32 */
2622	pCodeBuf[off++] = 0x81;
2623	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2624	pCodeBuf[off++] = RT_BYTE1((uint64_t)iSubtrahend);
2625	pCodeBuf[off++] = RT_BYTE2((uint64_t)iSubtrahend);
2626	pCodeBuf[off++] = RT_BYTE3((uint64_t)iSubtrahend);
2627	pCodeBuf[off++] = RT_BYTE4((uint64_t)iSubtrahend);
2628	}
2629	else if (iGprTmp != UINT8_MAX)
2630	{
2631	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off - 1, iGprTmp, (uint64_t)iSubtrahend);
2632	/* sub r/m64, r64 */
2633	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B) \| (iGprTmp < 8 ? 0 : X86_OP_REX_R);
2634	pCodeBuf[off++] = 0x29;
2635	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprTmp & 7, iGprDst & 7);
2636	}
2637	else
2638	# ifdef IEM_WITH_THROW_CATCH
2639	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2640	# else
2641	AssertReleaseFailedStmt(off = UINT32_MAX);
2642	# endif
2643
2644	#elif defined(RT_ARCH_ARM64)
2645	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2646	if (uAbsSubtrahend < 4096)
2647	{
2648	if (iSubtrahend >= 0)
2649	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2650	else
2651	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2652	}
2653	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2654	{
2655	if (iSubtrahend >= 0)
2656	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2657	true /f64Bit/, false /fSetFlags/, true /fShift/);
2658	else
2659	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2660	true /f64Bit/, false /fSetFlags/, true /fShift/);
2661	}
2662	else if (iGprTmp != UINT8_MAX)
2663	{
2664	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (uint64_t)iSubtrahend);
2665	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp);
2666	}
2667	else
2668	# ifdef IEM_WITH_THROW_CATCH
2669	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2670	# else
2671	AssertReleaseFailedStmt(off = UINT32_MAX);
2672	# endif
2673
2674	#else
2675	# error "Port me"
2676	#endif
2677	return off;
2678	}
2679
2680
2681	/**
2682	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2683	*
2684	* @note Larger constants will require a temporary register. Failing to specify
2685	* one when needed will trigger fatal assertion / throw.
2686	*/
2687	DECL_INLINE_THROW(uint32_t)
2688	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2689	uint8_t iGprTmp = UINT8_MAX)
2690
2691	{
2692	#ifdef RT_ARCH_AMD64
2693	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 13), off, iGprDst, iSubtrahend, iGprTmp);
2694	#elif defined(RT_ARCH_ARM64)
2695	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGprDst, iSubtrahend, iGprTmp);
2696	#else
2697	# error "Port me"
2698	#endif
2699	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2700	return off;
2701	}
2702
2703
2704	/**
2705	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2706	*
2707	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2708	*
2709	* @note ARM64: Larger constants will require a temporary register. Failing to
2710	* specify one when needed will trigger fatal assertion / throw.
2711	*/
2712	DECL_FORCE_INLINE_THROW(uint32_t)
2713	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2714	uint8_t iGprTmp = UINT8_MAX)
2715	{
2716	#ifdef RT_ARCH_AMD64
2717	if (iGprDst >= 8)
2718	pCodeBuf[off++] = X86_OP_REX_B;
2719	if (iSubtrahend == 1)
2720	{
2721	/* dec r/m32 */
2722	pCodeBuf[off++] = 0xff;
2723	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2724	}
2725	else if (iSubtrahend == -1)
2726	{
2727	/* inc r/m32 */
2728	pCodeBuf[off++] = 0xff;
2729	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2730	}
2731	else if (iSubtrahend < 128 && iSubtrahend >= -128)
2732	{
2733	/* sub r/m32, imm8 */
2734	pCodeBuf[off++] = 0x83;
2735	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2736	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2737	}
2738	else
2739	{
2740	/* sub r/m32, imm32 */
2741	pCodeBuf[off++] = 0x81;
2742	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2743	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
2744	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
2745	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
2746	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
2747	}
2748	RT_NOREF(iGprTmp);
2749
2750	#elif defined(RT_ARCH_ARM64)
2751	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2752	if (uAbsSubtrahend < 4096)
2753	{
2754	if (iSubtrahend >= 0)
2755	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2756	else
2757	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2758	}
2759	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2760	{
2761	if (iSubtrahend >= 0)
2762	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2763	false /f64Bit/, false /fSetFlags/, true /fShift/);
2764	else
2765	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2766	false /f64Bit/, false /fSetFlags/, true /fShift/);
2767	}
2768	else if (iGprTmp != UINT8_MAX)
2769	{
2770	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2771	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2772	}
2773	else
2774	# ifdef IEM_WITH_THROW_CATCH
2775	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2776	# else
2777	AssertReleaseFailedStmt(off = UINT32_MAX);
2778	# endif
2779
2780	#else
2781	# error "Port me"
2782	#endif
2783	return off;
2784	}
2785
2786
2787	/**
2788	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2789	*
2790	* @note ARM64: Larger constants will require a temporary register. Failing to
2791	* specify one when needed will trigger fatal assertion / throw.
2792	*/
2793	DECL_INLINE_THROW(uint32_t)
2794	iemNativeEmitSubGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2795	uint8_t iGprTmp = UINT8_MAX)
2796
2797	{
2798	#ifdef RT_ARCH_AMD64
2799	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iSubtrahend, iGprTmp);
2800	#elif defined(RT_ARCH_ARM64)
2801	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iSubtrahend, iGprTmp);
2802	#else
2803	# error "Port me"
2804	#endif
2805	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2806	return off;
2807	}
2808
2809
2810	/**
2811	* Emits a 16-bit GPR subtract with a signed immediate subtrahend.
2812	*
2813	* This will optimize using DEC/INC/whatever and ARM64 will not set flags,
2814	* so not suitable as a base for conditional jumps.
2815	*
2816	* @note ARM64: Will update the entire register.
2817	* @note AMD64: May perhaps only update the lower 16 bits of the register.
2818	* @note ARM64: Larger constants will require a temporary register. Failing to
2819	* specify one when needed will trigger fatal assertion / throw.
2820	*/
2821	DECL_FORCE_INLINE_THROW(uint32_t)
2822	iemNativeEmitSubGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iSubtrahend,
2823	uint8_t iGprTmp = UINT8_MAX)
2824	{
2825	#ifdef RT_ARCH_AMD64
2826	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2827	if (iGprDst >= 8)
2828	pCodeBuf[off++] = X86_OP_REX_B;
2829	if (iSubtrahend == 1)
2830	{
2831	/* dec r/m16 */
2832	pCodeBuf[off++] = 0xff;
2833	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2834	}
2835	else if (iSubtrahend == -1)
2836	{
2837	/* inc r/m16 */
2838	pCodeBuf[off++] = 0xff;
2839	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2840	}
2841	else if ((int8_t)iSubtrahend == iSubtrahend)
2842	{
2843	/* sub r/m16, imm8 */
2844	pCodeBuf[off++] = 0x83;
2845	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2846	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2847	}
2848	else
2849	{
2850	/* sub r/m16, imm16 */
2851	pCodeBuf[off++] = 0x81;
2852	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2853	pCodeBuf[off++] = RT_BYTE1((uint16_t)iSubtrahend);
2854	pCodeBuf[off++] = RT_BYTE2((uint16_t)iSubtrahend);
2855	}
2856	RT_NOREF(iGprTmp);
2857
2858	#elif defined(RT_ARCH_ARM64)
2859	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2860	if (uAbsSubtrahend < 4096)
2861	{
2862	if (iSubtrahend >= 0)
2863	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2864	else
2865	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2866	}
2867	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2868	{
2869	if (iSubtrahend >= 0)
2870	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2871	false /f64Bit/, false /fSetFlags/, true /fShift/);
2872	else
2873	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2874	false /f64Bit/, false /fSetFlags/, true /fShift/);
2875	}
2876	else if (iGprTmp != UINT8_MAX)
2877	{
2878	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2879	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2880	}
2881	else
2882	# ifdef IEM_WITH_THROW_CATCH
2883	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2884	# else
2885	AssertReleaseFailedStmt(off = UINT32_MAX);
2886	# endif
2887	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
2888
2889	#else
2890	# error "Port me"
2891	#endif
2892	return off;
2893	}
2894
2895
2896	/**
2897	* Emits adding a 64-bit GPR to another, storing the result in the first.
2898	* @note The AMD64 version sets flags.
2899	*/
2900	DECL_FORCE_INLINE(uint32_t)
2901	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2902	{
2903	#if defined(RT_ARCH_AMD64)
2904	/* add Gv,Ev */
2905	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2906	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
2907	pCodeBuf[off++] = 0x03;
2908	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
2909
2910	#elif defined(RT_ARCH_ARM64)
2911	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
2912
2913	#else
2914	# error "Port me"
2915	#endif
2916	return off;
2917	}
2918
2919
2920	/**
2921	* Emits adding a 64-bit GPR to another, storing the result in the first.
2922	* @note The AMD64 version sets flags.
2923	*/
2924	DECL_INLINE_THROW(uint32_t)
2925	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2926	{
2927	#if defined(RT_ARCH_AMD64)
2928	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
2929	#elif defined(RT_ARCH_ARM64)
2930	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
2931	#else
2932	# error "Port me"
2933	#endif
2934	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2935	return off;
2936	}
2937
2938
2939	/**
2940	* Emits adding a 64-bit GPR to another, storing the result in the first.
2941	* @note The AMD64 version sets flags.
2942	*/
2943	DECL_FORCE_INLINE(uint32_t)
2944	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2945	{
2946	#if defined(RT_ARCH_AMD64)
2947	/* add Gv,Ev */
2948	if (iGprDst >= 8 \|\| iGprAddend >= 8)
2949	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
2950	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
2951	pCodeBuf[off++] = 0x03;
2952	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
2953
2954	#elif defined(RT_ARCH_ARM64)
2955	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
2956
2957	#else
2958	# error "Port me"
2959	#endif
2960	return off;
2961	}
2962
2963
2964	/**
2965	* Emits adding a 64-bit GPR to another, storing the result in the first.
2966	* @note The AMD64 version sets flags.
2967	*/
2968	DECL_INLINE_THROW(uint32_t)
2969	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2970	{
2971	#if defined(RT_ARCH_AMD64)
2972	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
2973	#elif defined(RT_ARCH_ARM64)
2974	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
2975	#else
2976	# error "Port me"
2977	#endif
2978	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2979	return off;
2980	}
2981
2982
2983	/**
2984	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
2985	*/
2986	DECL_INLINE_THROW(uint32_t)
2987	iemNativeEmitAddGprImm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
2988	{
2989	#if defined(RT_ARCH_AMD64)
2990	/* add or inc */
2991	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2992	if (iImm8 != 1)
2993	{
2994	pCodeBuf[off++] = 0x83;
2995	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2996	pCodeBuf[off++] = (uint8_t)iImm8;
2997	}
2998	else
2999	{
3000	pCodeBuf[off++] = 0xff;
3001	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3002	}
3003
3004	#elif defined(RT_ARCH_ARM64)
3005	if (iImm8 >= 0)
3006	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint8_t)iImm8);
3007	else
3008	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint8_t)-iImm8);
3009
3010	#else
3011	# error "Port me"
3012	#endif
3013	return off;
3014	}
3015
3016
3017	/**
3018	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3019	*/
3020	DECL_INLINE_THROW(uint32_t)
3021	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3022	{
3023	#if defined(RT_ARCH_AMD64)
3024	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3025	#elif defined(RT_ARCH_ARM64)
3026	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3027	#else
3028	# error "Port me"
3029	#endif
3030	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3031	return off;
3032	}
3033
3034
3035	/**
3036	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3037	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3038	*/
3039	DECL_FORCE_INLINE(uint32_t)
3040	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3041	{
3042	#if defined(RT_ARCH_AMD64)
3043	/* add or inc */
3044	if (iGprDst >= 8)
3045	pCodeBuf[off++] = X86_OP_REX_B;
3046	if (iImm8 != 1)
3047	{
3048	pCodeBuf[off++] = 0x83;
3049	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3050	pCodeBuf[off++] = (uint8_t)iImm8;
3051	}
3052	else
3053	{
3054	pCodeBuf[off++] = 0xff;
3055	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3056	}
3057
3058	#elif defined(RT_ARCH_ARM64)
3059	if (iImm8 >= 0)
3060	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
3061	else
3062	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
3063
3064	#else
3065	# error "Port me"
3066	#endif
3067	return off;
3068	}
3069
3070
3071	/**
3072	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3073	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3074	*/
3075	DECL_INLINE_THROW(uint32_t)
3076	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3077	{
3078	#if defined(RT_ARCH_AMD64)
3079	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3080	#elif defined(RT_ARCH_ARM64)
3081	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3082	#else
3083	# error "Port me"
3084	#endif
3085	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3086	return off;
3087	}
3088
3089
3090	/**
3091	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3092	*
3093	* @note Will assert / throw if @a iGprTmp is not specified when needed.
3094	*/
3095	DECL_FORCE_INLINE_THROW(uint32_t)
3096	iemNativeEmitAddGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iAddend, uint8_t iGprTmp = UINT8_MAX)
3097	{
3098	#if defined(RT_ARCH_AMD64)
3099	if ((int8_t)iAddend == iAddend)
3100	return iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3101
3102	if ((int32_t)iAddend == iAddend)
3103	{
3104	/* add grp, imm32 */
3105	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3106	pCodeBuf[off++] = 0x81;
3107	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3108	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3109	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3110	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3111	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3112	}
3113	else if (iGprTmp != UINT8_MAX)
3114	{
3115	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3116
3117	/* add dst, tmpreg */
3118	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3119	\| (iGprTmp < 8 ? 0 : X86_OP_REX_B);
3120	pCodeBuf[off++] = 0x03;
3121	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprTmp & 7);
3122	}
3123	else
3124	# ifdef IEM_WITH_THROW_CATCH
3125	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3126	# else
3127	AssertReleaseFailedStmt(off = UINT32_MAX);
3128	# endif
3129
3130	#elif defined(RT_ARCH_ARM64)
3131	uint64_t const uAbsAddend = (uint64_t)RT_ABS(iAddend);
3132	if (uAbsAddend < 4096)
3133	{
3134	if (iAddend >= 0)
3135	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3136	else
3137	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3138	}
3139	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3140	{
3141	if (iAddend >= 0)
3142	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3143	true /f64Bit/, true /fShift12/);
3144	else
3145	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3146	true /f64Bit/, true /fShift12/);
3147	}
3148	else if (iGprTmp != UINT8_MAX)
3149	{
3150	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3151	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp);
3152	}
3153	else
3154	# ifdef IEM_WITH_THROW_CATCH
3155	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3156	# else
3157	AssertReleaseFailedStmt(off = UINT32_MAX);
3158	# endif
3159
3160	#else
3161	# error "Port me"
3162	#endif
3163	return off;
3164	}
3165
3166
3167	/**
3168	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3169	*/
3170	DECL_INLINE_THROW(uint32_t)
3171	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
3172	{
3173	#if defined(RT_ARCH_AMD64)
3174	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3175	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
3176
3177	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
3178	{
3179	/* add grp, imm32 */
3180	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3181	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3182	pbCodeBuf[off++] = 0x81;
3183	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3184	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3185	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3186	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3187	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3188	}
3189	else
3190	{
3191	/* Best to use a temporary register to deal with this in the simplest way: */
3192	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3193
3194	/* add dst, tmpreg */
3195	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3196	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3197	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3198	pbCodeBuf[off++] = 0x03;
3199	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
3200
3201	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3202	}
3203
3204	#elif defined(RT_ARCH_ARM64)
3205	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3206	{
3207	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3208	if (iAddend >= 0)
3209	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
3210	else
3211	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
3212	}
3213	else
3214	{
3215	/* Use temporary register for the immediate. */
3216	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3217
3218	/* add gprdst, gprdst, tmpreg */
3219	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3220	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
3221
3222	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3223	}
3224
3225	#else
3226	# error "Port me"
3227	#endif
3228	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3229	return off;
3230	}
3231
3232
3233	/**
3234	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3235	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3236	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
3237	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
3238	* the lower 12 bits always zero). The negative ranges are also allowed,
3239	* making it behave like a subtraction. If the constant does not conform,
3240	* bad stuff will happen.
3241	*/
3242	DECL_FORCE_INLINE_THROW(uint32_t)
3243	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3244	{
3245	#if defined(RT_ARCH_AMD64)
3246	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3247	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3248
3249	/* add grp, imm32 */
3250	if (iGprDst >= 8)
3251	pCodeBuf[off++] = X86_OP_REX_B;
3252	pCodeBuf[off++] = 0x81;
3253	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3254	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3255	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3256	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3257	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3258
3259	#elif defined(RT_ARCH_ARM64)
3260	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
3261	if (uAbsAddend <= 0xfff)
3262	{
3263	if (iAddend >= 0)
3264	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3265	else
3266	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3267	}
3268	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3269	{
3270	if (iAddend >= 0)
3271	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3272	false /f64Bit/, true /fShift12/);
3273	else
3274	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3275	false /f64Bit/, true /fShift12/);
3276	}
3277	else
3278	# ifdef IEM_WITH_THROW_CATCH
3279	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3280	# else
3281	AssertReleaseFailedStmt(off = UINT32_MAX);
3282	# endif
3283
3284	#else
3285	# error "Port me"
3286	#endif
3287	return off;
3288	}
3289
3290
3291	/**
3292	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3293	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3294	*/
3295	DECL_INLINE_THROW(uint32_t)
3296	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3297	{
3298	#if defined(RT_ARCH_AMD64)
3299	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
3300
3301	#elif defined(RT_ARCH_ARM64)
3302	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3303	{
3304	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3305	if (iAddend >= 0)
3306	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
3307	else
3308	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
3309	}
3310	else
3311	{
3312	/* Use temporary register for the immediate. */
3313	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
3314
3315	/* add gprdst, gprdst, tmpreg */
3316	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3317	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
3318
3319	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3320	}
3321
3322	#else
3323	# error "Port me"
3324	#endif
3325	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3326	return off;
3327	}
3328
3329
3330	/**
3331	* Adds two 64-bit GPRs together, storing the result in a third register.
3332	*/
3333	DECL_FORCE_INLINE(uint32_t)
3334	iemNativeEmitGprEqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3335	{
3336	#ifdef RT_ARCH_AMD64
3337	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3338	{
3339	/** @todo consider LEA */
3340	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend1);
3341	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend2);
3342	}
3343	else
3344	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3345
3346	#elif defined(RT_ARCH_ARM64)
3347	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2);
3348
3349	#else
3350	# error "Port me!"
3351	#endif
3352	return off;
3353	}
3354
3355
3356
3357	/**
3358	* Adds two 32-bit GPRs together, storing the result in a third register.
3359	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3360	*/
3361	DECL_FORCE_INLINE(uint32_t)
3362	iemNativeEmitGpr32EqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3363	{
3364	#ifdef RT_ARCH_AMD64
3365	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3366	{
3367	/** @todo consider LEA */
3368	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend1);
3369	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend2);
3370	}
3371	else
3372	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3373
3374	#elif defined(RT_ARCH_ARM64)
3375	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2, false /f64Bit/);
3376
3377	#else
3378	# error "Port me!"
3379	#endif
3380	return off;
3381	}
3382
3383
3384	/**
3385	* Adds a 64-bit GPR and a 64-bit unsigned constant, storing the result in a
3386	* third register.
3387	*
3388	* @note The ARM64 version does not work for non-trivial constants if the
3389	* two registers are the same. Will assert / throw exception.
3390	*/
3391	DECL_FORCE_INLINE_THROW(uint32_t)
3392	iemNativeEmitGprEqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int64_t iImmAddend)
3393	{
3394	#ifdef RT_ARCH_AMD64
3395	/** @todo consider LEA */
3396	if ((int8_t)iImmAddend == iImmAddend)
3397	{
3398	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend);
3399	off = iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3400	}
3401	else
3402	{
3403	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3404	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3405	}
3406
3407	#elif defined(RT_ARCH_ARM64)
3408	uint64_t const uAbsImmAddend = RT_ABS(iImmAddend);
3409	if (uAbsImmAddend < 4096)
3410	{
3411	if (iImmAddend >= 0)
3412	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3413	else
3414	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3415	}
3416	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3417	{
3418	if (iImmAddend >= 0)
3419	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3420	else
3421	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3422	}
3423	else if (iGprDst != iGprAddend)
3424	{
3425	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, (uint64_t)iImmAddend);
3426	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3427	}
3428	else
3429	# ifdef IEM_WITH_THROW_CATCH
3430	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3431	# else
3432	AssertReleaseFailedStmt(off = UINT32_MAX);
3433	# endif
3434
3435	#else
3436	# error "Port me!"
3437	#endif
3438	return off;
3439	}
3440
3441
3442	/**
3443	* Adds a 32-bit GPR and a 32-bit unsigned constant, storing the result in a
3444	* third register.
3445	*
3446	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3447	*
3448	* @note The ARM64 version does not work for non-trivial constants if the
3449	* two registers are the same. Will assert / throw exception.
3450	*/
3451	DECL_FORCE_INLINE_THROW(uint32_t)
3452	iemNativeEmitGpr32EqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
3453	{
3454	#ifdef RT_ARCH_AMD64
3455	/** @todo consider LEA */
3456	if ((int8_t)iImmAddend == iImmAddend)
3457	{
3458	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3459	off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3460	}
3461	else
3462	{
3463	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3464	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3465	}
3466
3467	#elif defined(RT_ARCH_ARM64)
3468	uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
3469	if (uAbsImmAddend < 4096)
3470	{
3471	if (iImmAddend >= 0)
3472	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3473	else
3474	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3475	}
3476	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3477	{
3478	if (iImmAddend >= 0)
3479	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3480	else
3481	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3482	}
3483	else if (iGprDst != iGprAddend)
3484	{
3485	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
3486	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3487	}
3488	else
3489	# ifdef IEM_WITH_THROW_CATCH
3490	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3491	# else
3492	AssertReleaseFailedStmt(off = UINT32_MAX);
3493	# endif
3494
3495	#else
3496	# error "Port me!"
3497	#endif
3498	return off;
3499	}
3500
3501
3502	/*********************************************************************************************************************************
3503	* Unary Operations *
3504	*********************************************************************************************************************************/
3505
3506	/**
3507	* Emits code for two complement negation of a 64-bit GPR.
3508	*/
3509	DECL_FORCE_INLINE_THROW(uint32_t)
3510	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3511	{
3512	#if defined(RT_ARCH_AMD64)
3513	/* neg Ev */
3514	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3515	pCodeBuf[off++] = 0xf7;
3516	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3517
3518	#elif defined(RT_ARCH_ARM64)
3519	/* sub dst, xzr, dst */
3520	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
3521
3522	#else
3523	# error "Port me"
3524	#endif
3525	return off;
3526	}
3527
3528
3529	/**
3530	* Emits code for two complement negation of a 64-bit GPR.
3531	*/
3532	DECL_INLINE_THROW(uint32_t)
3533	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3534	{
3535	#if defined(RT_ARCH_AMD64)
3536	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3537	#elif defined(RT_ARCH_ARM64)
3538	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3539	#else
3540	# error "Port me"
3541	#endif
3542	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3543	return off;
3544	}
3545
3546
3547	/**
3548	* Emits code for two complement negation of a 32-bit GPR.
3549	* @note bit 32 thru 63 are set to zero.
3550	*/
3551	DECL_FORCE_INLINE_THROW(uint32_t)
3552	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3553	{
3554	#if defined(RT_ARCH_AMD64)
3555	/* neg Ev */
3556	if (iGprDst >= 8)
3557	pCodeBuf[off++] = X86_OP_REX_B;
3558	pCodeBuf[off++] = 0xf7;
3559	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3560
3561	#elif defined(RT_ARCH_ARM64)
3562	/* sub dst, xzr, dst */
3563	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
3564
3565	#else
3566	# error "Port me"
3567	#endif
3568	return off;
3569	}
3570
3571
3572	/**
3573	* Emits code for two complement negation of a 32-bit GPR.
3574	* @note bit 32 thru 63 are set to zero.
3575	*/
3576	DECL_INLINE_THROW(uint32_t)
3577	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3578	{
3579	#if defined(RT_ARCH_AMD64)
3580	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3581	#elif defined(RT_ARCH_ARM64)
3582	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3583	#else
3584	# error "Port me"
3585	#endif
3586	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3587	return off;
3588	}
3589
3590
3591
3592	/*********************************************************************************************************************************
3593	* Bit Operations *
3594	*********************************************************************************************************************************/
3595
3596	/**
3597	* Emits code for clearing bits 16 thru 63 in the GPR.
3598	*/
3599	DECL_INLINE_THROW(uint32_t)
3600	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3601	{
3602	#if defined(RT_ARCH_AMD64)
3603	/* movzx Gv,Ew */
3604	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3605	if (iGprDst >= 8)
3606	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
3607	pbCodeBuf[off++] = 0x0f;
3608	pbCodeBuf[off++] = 0xb7;
3609	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
3610
3611	#elif defined(RT_ARCH_ARM64)
3612	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3613	# if 1
3614	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
3615	# else
3616	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
3617	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
3618	# endif
3619	#else
3620	# error "Port me"
3621	#endif
3622	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3623	return off;
3624	}
3625
3626
3627	/**
3628	* Emits code for AND'ing two 64-bit GPRs.
3629	*
3630	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3631	* and ARM64 hosts.
3632	*/
3633	DECL_INLINE_THROW(uint32_t)
3634	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3635	{
3636	#if defined(RT_ARCH_AMD64)
3637	/* and Gv, Ev */
3638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3639	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3640	pbCodeBuf[off++] = 0x23;
3641	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3642	RT_NOREF(fSetFlags);
3643
3644	#elif defined(RT_ARCH_ARM64)
3645	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3646	if (!fSetFlags)
3647	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
3648	else
3649	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
3650
3651	#else
3652	# error "Port me"
3653	#endif
3654	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3655	return off;
3656	}
3657
3658
3659	/**
3660	* Emits code for AND'ing two 32-bit GPRs.
3661	*/
3662	DECL_FORCE_INLINE(uint32_t)
3663	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3664	{
3665	#if defined(RT_ARCH_AMD64)
3666	/* and Gv, Ev */
3667	if (iGprDst >= 8 \|\| iGprSrc >= 8)
3668	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3669	pCodeBuf[off++] = 0x23;
3670	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3671	RT_NOREF(fSetFlags);
3672
3673	#elif defined(RT_ARCH_ARM64)
3674	if (!fSetFlags)
3675	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3676	else
3677	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3678
3679	#else
3680	# error "Port me"
3681	#endif
3682	return off;
3683	}
3684
3685
3686	/**
3687	* Emits code for AND'ing two 32-bit GPRs.
3688	*/
3689	DECL_INLINE_THROW(uint32_t)
3690	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3691	{
3692	#if defined(RT_ARCH_AMD64)
3693	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
3694	#elif defined(RT_ARCH_ARM64)
3695	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
3696	#else
3697	# error "Port me"
3698	#endif
3699	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3700	return off;
3701	}
3702
3703
3704	/**
3705	* Emits code for AND'ing a 64-bit GPRs with a constant.
3706	*
3707	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3708	* and ARM64 hosts.
3709	*/
3710	DECL_INLINE_THROW(uint32_t)
3711	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
3712	{
3713	#if defined(RT_ARCH_AMD64)
3714	if ((int64_t)uImm == (int8_t)uImm)
3715	{
3716	/* and Ev, imm8 */
3717	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3718	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3719	pbCodeBuf[off++] = 0x83;
3720	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3721	pbCodeBuf[off++] = (uint8_t)uImm;
3722	}
3723	else if ((int64_t)uImm == (int32_t)uImm)
3724	{
3725	/* and Ev, imm32 */
3726	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3727	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3728	pbCodeBuf[off++] = 0x81;
3729	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3730	pbCodeBuf[off++] = RT_BYTE1(uImm);
3731	pbCodeBuf[off++] = RT_BYTE2(uImm);
3732	pbCodeBuf[off++] = RT_BYTE3(uImm);
3733	pbCodeBuf[off++] = RT_BYTE4(uImm);
3734	}
3735	else
3736	{
3737	/* Use temporary register for the 64-bit immediate. */
3738	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3739	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
3740	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3741	}
3742	RT_NOREF(fSetFlags);
3743
3744	#elif defined(RT_ARCH_ARM64)
3745	uint32_t uImmR = 0;
3746	uint32_t uImmNandS = 0;
3747	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
3748	{
3749	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3750	if (!fSetFlags)
3751	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
3752	else
3753	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
3754	}
3755	else
3756	{
3757	/* Use temporary register for the 64-bit immediate. */
3758	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3759	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
3760	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3761	}
3762
3763	#else
3764	# error "Port me"
3765	#endif
3766	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3767	return off;
3768	}
3769
3770
3771	/**
3772	* Emits code for AND'ing an 32-bit GPRs with a constant.
3773	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3774	* @note For ARM64 this only supports @a uImm values that can be expressed using
3775	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
3776	* make sure this is possible!
3777	*/
3778	DECL_FORCE_INLINE_THROW(uint32_t)
3779	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
3780	{
3781	#if defined(RT_ARCH_AMD64)
3782	/* and Ev, imm */
3783	if (iGprDst >= 8)
3784	pCodeBuf[off++] = X86_OP_REX_B;
3785	if ((int32_t)uImm == (int8_t)uImm)
3786	{
3787	pCodeBuf[off++] = 0x83;
3788	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3789	pCodeBuf[off++] = (uint8_t)uImm;
3790	}
3791	else
3792	{
3793	pCodeBuf[off++] = 0x81;
3794	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3795	pCodeBuf[off++] = RT_BYTE1(uImm);
3796	pCodeBuf[off++] = RT_BYTE2(uImm);
3797	pCodeBuf[off++] = RT_BYTE3(uImm);
3798	pCodeBuf[off++] = RT_BYTE4(uImm);
3799	}
3800	RT_NOREF(fSetFlags);
3801
3802	#elif defined(RT_ARCH_ARM64)
3803	uint32_t uImmR = 0;
3804	uint32_t uImmNandS = 0;
3805	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
3806	{
3807	if (!fSetFlags)
3808	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3809	else
3810	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3811	}
3812	else
3813	# ifdef IEM_WITH_THROW_CATCH
3814	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3815	# else
3816	AssertReleaseFailedStmt(off = UINT32_MAX);
3817	# endif
3818
3819	#else
3820	# error "Port me"
3821	#endif
3822	return off;
3823	}
3824
3825
3826	/**
3827	* Emits code for AND'ing an 32-bit GPRs with a constant.
3828	*
3829	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3830	*/
3831	DECL_INLINE_THROW(uint32_t)
3832	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
3833	{
3834	#if defined(RT_ARCH_AMD64)
3835	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
3836
3837	#elif defined(RT_ARCH_ARM64)
3838	uint32_t uImmR = 0;
3839	uint32_t uImmNandS = 0;
3840	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
3841	{
3842	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3843	if (!fSetFlags)
3844	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3845	else
3846	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3847	}
3848	else
3849	{
3850	/* Use temporary register for the 64-bit immediate. */
3851	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3852	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
3853	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3854	}
3855
3856	#else
3857	# error "Port me"
3858	#endif
3859	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3860	return off;
3861	}
3862
3863
3864	/**
3865	* Emits code for AND'ing an 32-bit GPRs with a constant.
3866	*
3867	* @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
3868	* encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
3869	* the same.
3870	*
3871	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3872	*/
3873	DECL_FORCE_INLINE_THROW(uint32_t)
3874	iemNativeEmitGpr32EqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint32_t uImm,
3875	bool fSetFlags = false)
3876	{
3877	#if defined(RT_ARCH_AMD64)
3878	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
3879	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
3880	RT_NOREF(fSetFlags);
3881
3882	#elif defined(RT_ARCH_ARM64)
3883	uint32_t uImmR = 0;
3884	uint32_t uImmNandS = 0;
3885	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
3886	{
3887	if (!fSetFlags)
3888	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3889	else
3890	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3891	}
3892	else if (iGprDst != iGprSrc)
3893	{
3894	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
3895	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
3896	}
3897	else
3898	# ifdef IEM_WITH_THROW_CATCH
3899	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3900	# else
3901	AssertReleaseFailedStmt(off = UINT32_MAX);
3902	# endif
3903
3904	#else
3905	# error "Port me"
3906	#endif
3907	return off;
3908	}
3909
3910
3911	/**
3912	* Emits code for OR'ing two 64-bit GPRs.
3913	*/
3914	DECL_INLINE_THROW(uint32_t)
3915	iemNativeEmitOrGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
3916	{
3917	#if defined(RT_ARCH_AMD64)
3918	/* or Gv, Ev */
3919	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3920	pCodeBuf[off++] = 0x0b;
3921	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3922
3923	#elif defined(RT_ARCH_ARM64)
3924	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc);
3925
3926	#else
3927	# error "Port me"
3928	#endif
3929	return off;
3930	}
3931
3932
3933
3934	/**
3935	* Emits code for XOR'ing two 64-bit GPRs.
3936	*/
3937	DECL_INLINE_THROW(uint32_t)
3938	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
3939	{
3940	#if defined(RT_ARCH_AMD64)
3941	/* and Gv, Ev */
3942	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3943	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3944	pbCodeBuf[off++] = 0x33;
3945	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3946
3947	#elif defined(RT_ARCH_ARM64)
3948	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3949	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
3950
3951	#else
3952	# error "Port me"
3953	#endif
3954	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3955	return off;
3956	}
3957
3958
3959	/**
3960	* Emits code for XOR'ing two 32-bit GPRs.
3961	*/
3962	DECL_INLINE_THROW(uint32_t)
3963	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
3964	{
3965	#if defined(RT_ARCH_AMD64)
3966	/* and Gv, Ev */
3967	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3968	if (iGprDst >= 8 \|\| iGprSrc >= 8)
3969	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3970	pbCodeBuf[off++] = 0x33;
3971	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3972
3973	#elif defined(RT_ARCH_ARM64)
3974	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3975	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3976
3977	#else
3978	# error "Port me"
3979	#endif
3980	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3981	return off;
3982	}
3983
3984
3985	/**
3986	* Emits code for XOR'ing an 32-bit GPRs with a constant.
3987	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3988	* @note For ARM64 this only supports @a uImm values that can be expressed using
3989	* the two 6-bit immediates of the EOR instructions. The caller must make
3990	* sure this is possible!
3991	*/
3992	DECL_FORCE_INLINE_THROW(uint32_t)
3993	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
3994	{
3995	#if defined(RT_ARCH_AMD64)
3996	/* and Ev, imm */
3997	if (iGprDst >= 8)
3998	pCodeBuf[off++] = X86_OP_REX_B;
3999	if ((int32_t)uImm == (int8_t)uImm)
4000	{
4001	pCodeBuf[off++] = 0x83;
4002	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4003	pCodeBuf[off++] = (uint8_t)uImm;
4004	}
4005	else
4006	{
4007	pCodeBuf[off++] = 0x81;
4008	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4009	pCodeBuf[off++] = RT_BYTE1(uImm);
4010	pCodeBuf[off++] = RT_BYTE2(uImm);
4011	pCodeBuf[off++] = RT_BYTE3(uImm);
4012	pCodeBuf[off++] = RT_BYTE4(uImm);
4013	}
4014
4015	#elif defined(RT_ARCH_ARM64)
4016	uint32_t uImmR = 0;
4017	uint32_t uImmNandS = 0;
4018	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4019	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4020	else
4021	# ifdef IEM_WITH_THROW_CATCH
4022	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4023	# else
4024	AssertReleaseFailedStmt(off = UINT32_MAX);
4025	# endif
4026
4027	#else
4028	# error "Port me"
4029	#endif
4030	return off;
4031	}
4032
4033
4034	/*********************************************************************************************************************************
4035	* Shifting *
4036	*********************************************************************************************************************************/
4037
4038	/**
4039	* Emits code for shifting a GPR a fixed number of bits to the left.
4040	*/
4041	DECL_FORCE_INLINE(uint32_t)
4042	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4043	{
4044	Assert(cShift > 0 && cShift < 64);
4045
4046	#if defined(RT_ARCH_AMD64)
4047	/* shl dst, cShift */
4048	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4049	if (cShift != 1)
4050	{
4051	pCodeBuf[off++] = 0xc1;
4052	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4053	pCodeBuf[off++] = cShift;
4054	}
4055	else
4056	{
4057	pCodeBuf[off++] = 0xd1;
4058	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4059	}
4060
4061	#elif defined(RT_ARCH_ARM64)
4062	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
4063
4064	#else
4065	# error "Port me"
4066	#endif
4067	return off;
4068	}
4069
4070
4071	/**
4072	* Emits code for shifting a GPR a fixed number of bits to the left.
4073	*/
4074	DECL_INLINE_THROW(uint32_t)
4075	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4076	{
4077	#if defined(RT_ARCH_AMD64)
4078	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4079	#elif defined(RT_ARCH_ARM64)
4080	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4081	#else
4082	# error "Port me"
4083	#endif
4084	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4085	return off;
4086	}
4087
4088
4089	/**
4090	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4091	*/
4092	DECL_FORCE_INLINE(uint32_t)
4093	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4094	{
4095	Assert(cShift > 0 && cShift < 32);
4096
4097	#if defined(RT_ARCH_AMD64)
4098	/* shl dst, cShift */
4099	if (iGprDst >= 8)
4100	pCodeBuf[off++] = X86_OP_REX_B;
4101	if (cShift != 1)
4102	{
4103	pCodeBuf[off++] = 0xc1;
4104	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4105	pCodeBuf[off++] = cShift;
4106	}
4107	else
4108	{
4109	pCodeBuf[off++] = 0xd1;
4110	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4111	}
4112
4113	#elif defined(RT_ARCH_ARM64)
4114	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
4115
4116	#else
4117	# error "Port me"
4118	#endif
4119	return off;
4120	}
4121
4122
4123	/**
4124	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4125	*/
4126	DECL_INLINE_THROW(uint32_t)
4127	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4128	{
4129	#if defined(RT_ARCH_AMD64)
4130	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4131	#elif defined(RT_ARCH_ARM64)
4132	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4133	#else
4134	# error "Port me"
4135	#endif
4136	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4137	return off;
4138	}
4139
4140
4141	/**
4142	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4143	*/
4144	DECL_FORCE_INLINE(uint32_t)
4145	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4146	{
4147	Assert(cShift > 0 && cShift < 64);
4148
4149	#if defined(RT_ARCH_AMD64)
4150	/* shr dst, cShift */
4151	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4152	if (cShift != 1)
4153	{
4154	pCodeBuf[off++] = 0xc1;
4155	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4156	pCodeBuf[off++] = cShift;
4157	}
4158	else
4159	{
4160	pCodeBuf[off++] = 0xd1;
4161	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4162	}
4163
4164	#elif defined(RT_ARCH_ARM64)
4165	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
4166
4167	#else
4168	# error "Port me"
4169	#endif
4170	return off;
4171	}
4172
4173
4174	/**
4175	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4176	*/
4177	DECL_INLINE_THROW(uint32_t)
4178	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4179	{
4180	#if defined(RT_ARCH_AMD64)
4181	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4182	#elif defined(RT_ARCH_ARM64)
4183	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4184	#else
4185	# error "Port me"
4186	#endif
4187	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4188	return off;
4189	}
4190
4191
4192	/**
4193	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4194	* right.
4195	*/
4196	DECL_FORCE_INLINE(uint32_t)
4197	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4198	{
4199	Assert(cShift > 0 && cShift < 32);
4200
4201	#if defined(RT_ARCH_AMD64)
4202	/* shr dst, cShift */
4203	if (iGprDst >= 8)
4204	pCodeBuf[off++] = X86_OP_REX_B;
4205	if (cShift != 1)
4206	{
4207	pCodeBuf[off++] = 0xc1;
4208	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4209	pCodeBuf[off++] = cShift;
4210	}
4211	else
4212	{
4213	pCodeBuf[off++] = 0xd1;
4214	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4215	}
4216
4217	#elif defined(RT_ARCH_ARM64)
4218	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
4219
4220	#else
4221	# error "Port me"
4222	#endif
4223	return off;
4224	}
4225
4226
4227	/**
4228	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4229	* right.
4230	*/
4231	DECL_INLINE_THROW(uint32_t)
4232	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4233	{
4234	#if defined(RT_ARCH_AMD64)
4235	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4236	#elif defined(RT_ARCH_ARM64)
4237	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4238	#else
4239	# error "Port me"
4240	#endif
4241	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4242	return off;
4243	}
4244
4245
4246	/**
4247	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4248	* right and assigning it to a different GPR.
4249	*/
4250	DECL_INLINE_THROW(uint32_t)
4251	iemNativeEmitGpr32EqGprShiftRightImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint8_t cShift)
4252	{
4253	Assert(cShift > 0); Assert(cShift < 32);
4254	#if defined(RT_ARCH_AMD64)
4255	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4256	off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, iGprDst, cShift);
4257
4258	#elif defined(RT_ARCH_ARM64)
4259	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprSrc, cShift, false /64Bit/);
4260
4261	#else
4262	# error "Port me"
4263	#endif
4264	return off;
4265	}
4266
4267
4268	/**
4269	* Emits code for rotating a GPR a fixed number of bits to the left.
4270	*/
4271	DECL_FORCE_INLINE(uint32_t)
4272	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4273	{
4274	Assert(cShift > 0 && cShift < 64);
4275
4276	#if defined(RT_ARCH_AMD64)
4277	/* rol dst, cShift */
4278	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4279	if (cShift != 1)
4280	{
4281	pCodeBuf[off++] = 0xc1;
4282	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4283	pCodeBuf[off++] = cShift;
4284	}
4285	else
4286	{
4287	pCodeBuf[off++] = 0xd1;
4288	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4289	}
4290
4291	#elif defined(RT_ARCH_ARM64)
4292	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
4293
4294	#else
4295	# error "Port me"
4296	#endif
4297	return off;
4298	}
4299
4300
4301
4302	/*********************************************************************************************************************************
4303	* Compare and Testing *
4304	*********************************************************************************************************************************/
4305
4306
4307	#ifdef RT_ARCH_ARM64
4308	/**
4309	* Emits an ARM64 compare instruction.
4310	*/
4311	DECL_INLINE_THROW(uint32_t)
4312	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
4313	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
4314	{
4315	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4316	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
4317	f64Bit, true /fSetFlags/, cShift, enmShift);
4318	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4319	return off;
4320	}
4321	#endif
4322
4323
4324	/**
4325	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4326	* with conditional instruction.
4327	*/
4328	DECL_FORCE_INLINE(uint32_t)
4329	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4330	{
4331	#ifdef RT_ARCH_AMD64
4332	/* cmp Gv, Ev */
4333	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4334	pCodeBuf[off++] = 0x3b;
4335	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4336
4337	#elif defined(RT_ARCH_ARM64)
4338	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
4339
4340	#else
4341	# error "Port me!"
4342	#endif
4343	return off;
4344	}
4345
4346
4347	/**
4348	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4349	* with conditional instruction.
4350	*/
4351	DECL_INLINE_THROW(uint32_t)
4352	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4353	{
4354	#ifdef RT_ARCH_AMD64
4355	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4356	#elif defined(RT_ARCH_ARM64)
4357	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4358	#else
4359	# error "Port me!"
4360	#endif
4361	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4362	return off;
4363	}
4364
4365
4366	/**
4367	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4368	* with conditional instruction.
4369	*/
4370	DECL_FORCE_INLINE(uint32_t)
4371	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4372	{
4373	#ifdef RT_ARCH_AMD64
4374	/* cmp Gv, Ev */
4375	if (iGprLeft >= 8 \|\| iGprRight >= 8)
4376	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4377	pCodeBuf[off++] = 0x3b;
4378	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4379
4380	#elif defined(RT_ARCH_ARM64)
4381	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
4382
4383	#else
4384	# error "Port me!"
4385	#endif
4386	return off;
4387	}
4388
4389
4390	/**
4391	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4392	* with conditional instruction.
4393	*/
4394	DECL_INLINE_THROW(uint32_t)
4395	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4396	{
4397	#ifdef RT_ARCH_AMD64
4398	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4399	#elif defined(RT_ARCH_ARM64)
4400	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4401	#else
4402	# error "Port me!"
4403	#endif
4404	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4405	return off;
4406	}
4407
4408
4409	/**
4410	* Emits a compare of a 64-bit GPR with a constant value, settings status
4411	* flags/whatever for use with conditional instruction.
4412	*/
4413	DECL_INLINE_THROW(uint32_t)
4414	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
4415	{
4416	#ifdef RT_ARCH_AMD64
4417	if (uImm <= UINT32_C(0xff))
4418	{
4419	/* cmp Ev, Ib */
4420	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
4421	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4422	pbCodeBuf[off++] = 0x83;
4423	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4424	pbCodeBuf[off++] = (uint8_t)uImm;
4425	}
4426	else if ((int64_t)uImm == (int32_t)uImm)
4427	{
4428	/* cmp Ev, imm */
4429	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
4430	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4431	pbCodeBuf[off++] = 0x81;
4432	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4433	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4434	pbCodeBuf[off++] = RT_BYTE1(uImm);
4435	pbCodeBuf[off++] = RT_BYTE2(uImm);
4436	pbCodeBuf[off++] = RT_BYTE3(uImm);
4437	pbCodeBuf[off++] = RT_BYTE4(uImm);
4438	}
4439	else
4440	{
4441	/* Use temporary register for the immediate. */
4442	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4443	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4444	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4445	}
4446
4447	#elif defined(RT_ARCH_ARM64)
4448	/** @todo guess there are clevere things we can do here... */
4449	if (uImm < _4K)
4450	{
4451	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4452	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4453	true /64Bit/, true /fSetFlags/);
4454	}
4455	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4456	{
4457	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4458	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
4459	true /64Bit/, true /fSetFlags/, true /fShift12/);
4460	}
4461	else
4462	{
4463	/* Use temporary register for the immediate. */
4464	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4465	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4466	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4467	}
4468
4469	#else
4470	# error "Port me!"
4471	#endif
4472
4473	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4474	return off;
4475	}
4476
4477
4478	/**
4479	* Emits a compare of a 32-bit GPR with a constant value, settings status
4480	* flags/whatever for use with conditional instruction.
4481	*
4482	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
4483	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
4484	* bits all zero). Will release assert or throw exception if the caller
4485	* violates this restriction.
4486	*/
4487	DECL_FORCE_INLINE_THROW(uint32_t)
4488	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4489	{
4490	#ifdef RT_ARCH_AMD64
4491	if (iGprLeft >= 8)
4492	pCodeBuf[off++] = X86_OP_REX_B;
4493	if (uImm <= UINT32_C(0x7f))
4494	{
4495	/* cmp Ev, Ib */
4496	pCodeBuf[off++] = 0x83;
4497	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4498	pCodeBuf[off++] = (uint8_t)uImm;
4499	}
4500	else
4501	{
4502	/* cmp Ev, imm */
4503	pCodeBuf[off++] = 0x81;
4504	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4505	pCodeBuf[off++] = RT_BYTE1(uImm);
4506	pCodeBuf[off++] = RT_BYTE2(uImm);
4507	pCodeBuf[off++] = RT_BYTE3(uImm);
4508	pCodeBuf[off++] = RT_BYTE4(uImm);
4509	}
4510
4511	#elif defined(RT_ARCH_ARM64)
4512	/** @todo guess there are clevere things we can do here... */
4513	if (uImm < _4K)
4514	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4515	false /64Bit/, true /fSetFlags/);
4516	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4517	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4518	false /64Bit/, true /fSetFlags/, true /fShift12/);
4519	else
4520	# ifdef IEM_WITH_THROW_CATCH
4521	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4522	# else
4523	AssertReleaseFailedStmt(off = UINT32_MAX);
4524	# endif
4525
4526	#else
4527	# error "Port me!"
4528	#endif
4529	return off;
4530	}
4531
4532
4533	/**
4534	* Emits a compare of a 32-bit GPR with a constant value, settings status
4535	* flags/whatever for use with conditional instruction.
4536	*/
4537	DECL_INLINE_THROW(uint32_t)
4538	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4539	{
4540	#ifdef RT_ARCH_AMD64
4541	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
4542
4543	#elif defined(RT_ARCH_ARM64)
4544	/** @todo guess there are clevere things we can do here... */
4545	if (uImm < _4K)
4546	{
4547	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4548	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4549	false /64Bit/, true /fSetFlags/);
4550	}
4551	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4552	{
4553	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4554	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4555	false /64Bit/, true /fSetFlags/, true /fShift12/);
4556	}
4557	else
4558	{
4559	/* Use temporary register for the immediate. */
4560	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4561	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
4562	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4563	}
4564
4565	#else
4566	# error "Port me!"
4567	#endif
4568
4569	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4570	return off;
4571	}
4572
4573
4574
4575	/*********************************************************************************************************************************
4576	* Branching *
4577	*********************************************************************************************************************************/
4578
4579	/**
4580	* Emits a JMP rel32 / B imm19 to the given label.
4581	*/
4582	DECL_FORCE_INLINE_THROW(uint32_t)
4583	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
4584	{
4585	Assert(idxLabel < pReNative->cLabels);
4586
4587	#ifdef RT_ARCH_AMD64
4588	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4589	{
4590	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
4591	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
4592	{
4593	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
4594	pCodeBuf[off++] = (uint8_t)offRel;
4595	}
4596	else
4597	{
4598	offRel -= 3;
4599	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4600	pCodeBuf[off++] = RT_BYTE1(offRel);
4601	pCodeBuf[off++] = RT_BYTE2(offRel);
4602	pCodeBuf[off++] = RT_BYTE3(offRel);
4603	pCodeBuf[off++] = RT_BYTE4(offRel);
4604	}
4605	}
4606	else
4607	{
4608	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4609	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4610	pCodeBuf[off++] = 0xfe;
4611	pCodeBuf[off++] = 0xff;
4612	pCodeBuf[off++] = 0xff;
4613	pCodeBuf[off++] = 0xff;
4614	}
4615	pCodeBuf[off++] = 0xcc; /* int3 poison */
4616
4617	#elif defined(RT_ARCH_ARM64)
4618	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4619	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
4620	else
4621	{
4622	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
4623	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
4624	}
4625
4626	#else
4627	# error "Port me!"
4628	#endif
4629	return off;
4630	}
4631
4632
4633	/**
4634	* Emits a JMP rel32 / B imm19 to the given label.
4635	*/
4636	DECL_INLINE_THROW(uint32_t)
4637	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4638	{
4639	#ifdef RT_ARCH_AMD64
4640	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
4641	#elif defined(RT_ARCH_ARM64)
4642	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
4643	#else
4644	# error "Port me!"
4645	#endif
4646	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4647	return off;
4648	}
4649
4650
4651	/**
4652	* Emits a JMP rel32 / B imm19 to a new undefined label.
4653	*/
4654	DECL_INLINE_THROW(uint32_t)
4655	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4656	{
4657	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4658	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
4659	}
4660
4661	/** Condition type. */
4662	#ifdef RT_ARCH_AMD64
4663	typedef enum IEMNATIVEINSTRCOND : uint8_t
4664	{
4665	kIemNativeInstrCond_o = 0,
4666	kIemNativeInstrCond_no,
4667	kIemNativeInstrCond_c,
4668	kIemNativeInstrCond_nc,
4669	kIemNativeInstrCond_e,
4670	kIemNativeInstrCond_ne,
4671	kIemNativeInstrCond_be,
4672	kIemNativeInstrCond_nbe,
4673	kIemNativeInstrCond_s,
4674	kIemNativeInstrCond_ns,
4675	kIemNativeInstrCond_p,
4676	kIemNativeInstrCond_np,
4677	kIemNativeInstrCond_l,
4678	kIemNativeInstrCond_nl,
4679	kIemNativeInstrCond_le,
4680	kIemNativeInstrCond_nle
4681	} IEMNATIVEINSTRCOND;
4682	#elif defined(RT_ARCH_ARM64)
4683	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
4684	# define kIemNativeInstrCond_o todo_conditional_codes
4685	# define kIemNativeInstrCond_no todo_conditional_codes
4686	# define kIemNativeInstrCond_c todo_conditional_codes
4687	# define kIemNativeInstrCond_nc todo_conditional_codes
4688	# define kIemNativeInstrCond_e kArmv8InstrCond_Eq
4689	# define kIemNativeInstrCond_ne kArmv8InstrCond_Ne
4690	# define kIemNativeInstrCond_be kArmv8InstrCond_Ls
4691	# define kIemNativeInstrCond_nbe kArmv8InstrCond_Hi
4692	# define kIemNativeInstrCond_s todo_conditional_codes
4693	# define kIemNativeInstrCond_ns todo_conditional_codes
4694	# define kIemNativeInstrCond_p todo_conditional_codes
4695	# define kIemNativeInstrCond_np todo_conditional_codes
4696	# define kIemNativeInstrCond_l kArmv8InstrCond_Lt
4697	# define kIemNativeInstrCond_nl kArmv8InstrCond_Ge
4698	# define kIemNativeInstrCond_le kArmv8InstrCond_Le
4699	# define kIemNativeInstrCond_nle kArmv8InstrCond_Gt
4700	#else
4701	# error "Port me!"
4702	#endif
4703
4704
4705	/**
4706	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
4707	*/
4708	DECL_FORCE_INLINE_THROW(uint32_t)
4709	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
4710	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
4711	{
4712	Assert(idxLabel < pReNative->cLabels);
4713
4714	uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
4715	#ifdef RT_ARCH_AMD64
4716	if (offLabel >= off)
4717	{
4718	/* jcc rel32 */
4719	pCodeBuf[off++] = 0x0f;
4720	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4721	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4722	pCodeBuf[off++] = 0x00;
4723	pCodeBuf[off++] = 0x00;
4724	pCodeBuf[off++] = 0x00;
4725	pCodeBuf[off++] = 0x00;
4726	}
4727	else
4728	{
4729	int32_t offDisp = offLabel - (off + 2);
4730	if ((int8_t)offDisp == offDisp)
4731	{
4732	/* jcc rel8 */
4733	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
4734	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4735	}
4736	else
4737	{
4738	/* jcc rel32 */
4739	offDisp -= 4;
4740	pCodeBuf[off++] = 0x0f;
4741	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4742	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4743	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
4744	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
4745	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
4746	}
4747	}
4748
4749	#elif defined(RT_ARCH_ARM64)
4750	if (offLabel >= off)
4751	{
4752	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4753	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
4754	}
4755	else
4756	{
4757	Assert(off - offLabel <= 0x3ffffU);
4758	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
4759	}
4760
4761	#else
4762	# error "Port me!"
4763	#endif
4764	return off;
4765	}
4766
4767
4768	/**
4769	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
4770	*/
4771	DECL_INLINE_THROW(uint32_t)
4772	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
4773	{
4774	#ifdef RT_ARCH_AMD64
4775	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
4776	#elif defined(RT_ARCH_ARM64)
4777	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
4778	#else
4779	# error "Port me!"
4780	#endif
4781	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4782	return off;
4783	}
4784
4785
4786	/**
4787	* Emits a Jcc rel32 / B.cc imm19 to a new label.
4788	*/
4789	DECL_INLINE_THROW(uint32_t)
4790	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4791	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
4792	{
4793	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4794	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
4795	}
4796
4797
4798	/**
4799	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
4800	*/
4801	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4802	{
4803	#ifdef RT_ARCH_AMD64
4804	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
4805	#elif defined(RT_ARCH_ARM64)
4806	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
4807	#else
4808	# error "Port me!"
4809	#endif
4810	}
4811
4812	/**
4813	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
4814	*/
4815	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4816	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4817	{
4818	#ifdef RT_ARCH_AMD64
4819	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
4820	#elif defined(RT_ARCH_ARM64)
4821	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
4822	#else
4823	# error "Port me!"
4824	#endif
4825	}
4826
4827
4828	/**
4829	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
4830	*/
4831	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4832	{
4833	#ifdef RT_ARCH_AMD64
4834	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
4835	#elif defined(RT_ARCH_ARM64)
4836	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
4837	#else
4838	# error "Port me!"
4839	#endif
4840	}
4841
4842	/**
4843	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
4844	*/
4845	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4846	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4847	{
4848	#ifdef RT_ARCH_AMD64
4849	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
4850	#elif defined(RT_ARCH_ARM64)
4851	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
4852	#else
4853	# error "Port me!"
4854	#endif
4855	}
4856
4857
4858	/**
4859	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
4860	*/
4861	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4862	{
4863	#ifdef RT_ARCH_AMD64
4864	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
4865	#elif defined(RT_ARCH_ARM64)
4866	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
4867	#else
4868	# error "Port me!"
4869	#endif
4870	}
4871
4872	/**
4873	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
4874	*/
4875	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4876	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4877	{
4878	#ifdef RT_ARCH_AMD64
4879	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
4880	#elif defined(RT_ARCH_ARM64)
4881	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
4882	#else
4883	# error "Port me!"
4884	#endif
4885	}
4886
4887
4888	/**
4889	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
4890	*/
4891	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4892	{
4893	#ifdef RT_ARCH_AMD64
4894	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
4895	#elif defined(RT_ARCH_ARM64)
4896	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
4897	#else
4898	# error "Port me!"
4899	#endif
4900	}
4901
4902	/**
4903	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
4904	*/
4905	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4906	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4907	{
4908	#ifdef RT_ARCH_AMD64
4909	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
4910	#elif defined(RT_ARCH_ARM64)
4911	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
4912	#else
4913	# error "Port me!"
4914	#endif
4915	}
4916
4917
4918	/**
4919	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
4920	*/
4921	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4922	{
4923	#ifdef RT_ARCH_AMD64
4924	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
4925	#elif defined(RT_ARCH_ARM64)
4926	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
4927	#else
4928	# error "Port me!"
4929	#endif
4930	}
4931
4932	/**
4933	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
4934	*/
4935	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4936	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4937	{
4938	#ifdef RT_ARCH_AMD64
4939	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
4940	#elif defined(RT_ARCH_ARM64)
4941	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
4942	#else
4943	# error "Port me!"
4944	#endif
4945	}
4946
4947
4948	/**
4949	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
4950	*
4951	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
4952	*
4953	* Only use hardcoded jumps forward when emitting for exactly one
4954	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
4955	* the right target address on all platforms!
4956	*
4957	* Please also note that on x86 it is necessary pass off + 256 or higher
4958	* for @a offTarget one believe the intervening code is more than 127
4959	* bytes long.
4960	*/
4961	DECL_FORCE_INLINE(uint32_t)
4962	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
4963	{
4964	#ifdef RT_ARCH_AMD64
4965	/* jcc rel8 / rel32 */
4966	int32_t offDisp = (int32_t)(offTarget - (off + 2));
4967	if (offDisp < 128 && offDisp >= -128)
4968	{
4969	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
4970	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4971	}
4972	else
4973	{
4974	offDisp -= 4;
4975	pCodeBuf[off++] = 0x0f;
4976	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4977	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4978	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
4979	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
4980	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
4981	}
4982
4983	#elif defined(RT_ARCH_ARM64)
4984	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
4985
4986	#else
4987	# error "Port me!"
4988	#endif
4989	return off;
4990	}
4991
4992
4993	/**
4994	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
4995	*
4996	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
4997	*
4998	* Only use hardcoded jumps forward when emitting for exactly one
4999	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5000	* the right target address on all platforms!
5001	*
5002	* Please also note that on x86 it is necessary pass off + 256 or higher
5003	* for @a offTarget one believe the intervening code is more than 127
5004	* bytes long.
5005	*/
5006	DECL_INLINE_THROW(uint32_t)
5007	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5008	{
5009	#ifdef RT_ARCH_AMD64
5010	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
5011	#elif defined(RT_ARCH_ARM64)
5012	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
5013	#else
5014	# error "Port me!"
5015	#endif
5016	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5017	return off;
5018	}
5019
5020
5021	/**
5022	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
5023	*
5024	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5025	*/
5026	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5027	{
5028	#ifdef RT_ARCH_AMD64
5029	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
5030	#elif defined(RT_ARCH_ARM64)
5031	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
5032	#else
5033	# error "Port me!"
5034	#endif
5035	}
5036
5037
5038	/**
5039	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
5040	*
5041	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5042	*/
5043	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5044	{
5045	#ifdef RT_ARCH_AMD64
5046	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
5047	#elif defined(RT_ARCH_ARM64)
5048	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
5049	#else
5050	# error "Port me!"
5051	#endif
5052	}
5053
5054
5055	/**
5056	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
5057	*
5058	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5059	*/
5060	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5061	{
5062	#ifdef RT_ARCH_AMD64
5063	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
5064	#elif defined(RT_ARCH_ARM64)
5065	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
5066	#else
5067	# error "Port me!"
5068	#endif
5069	}
5070
5071
5072	/**
5073	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
5074	*
5075	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5076	*/
5077	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5078	{
5079	#ifdef RT_ARCH_AMD64
5080	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
5081	#elif defined(RT_ARCH_ARM64)
5082	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
5083	#else
5084	# error "Port me!"
5085	#endif
5086	}
5087
5088
5089	/**
5090	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5091	*
5092	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5093	*/
5094	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
5095	{
5096	#ifdef RT_ARCH_AMD64
5097	/* jmp rel8 or rel32 */
5098	int32_t offDisp = offTarget - (off + 2);
5099	if (offDisp < 128 && offDisp >= -128)
5100	{
5101	pCodeBuf[off++] = 0xeb;
5102	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5103	}
5104	else
5105	{
5106	offDisp -= 3;
5107	pCodeBuf[off++] = 0xe9;
5108	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5109	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5110	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5111	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5112	}
5113
5114	#elif defined(RT_ARCH_ARM64)
5115	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
5116
5117	#else
5118	# error "Port me!"
5119	#endif
5120	return off;
5121	}
5122
5123
5124	/**
5125	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5126	*
5127	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5128	*/
5129	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5130	{
5131	#ifdef RT_ARCH_AMD64
5132	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
5133	#elif defined(RT_ARCH_ARM64)
5134	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
5135	#else
5136	# error "Port me!"
5137	#endif
5138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5139	return off;
5140	}
5141
5142
5143	/**
5144	* Fixes up a conditional jump to a fixed label.
5145	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
5146	* iemNativeEmitJzToFixed, ...
5147	*/
5148	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
5149	{
5150	#ifdef RT_ARCH_AMD64
5151	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
5152	uint8_t const bOpcode = pbCodeBuf[offFixup];
5153	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
5154	{
5155	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
5156	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
5157	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
5158	}
5159	else
5160	{
5161	if (bOpcode != 0x0f)
5162	Assert(bOpcode == 0xe9);
5163	else
5164	{
5165	offFixup += 1;
5166	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
5167	}
5168	uint32_t const offRel32 = offTarget - (offFixup + 5);
5169	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
5170	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
5171	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
5172	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
5173	}
5174
5175	#elif defined(RT_ARCH_ARM64)
5176	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
5177	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
5178	{
5179	/* B.COND + BC.COND */
5180	int32_t const offDisp = offTarget - offFixup;
5181	Assert(offDisp >= -262144 && offDisp < 262144);
5182	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
5183	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
5184	}
5185	else
5186	{
5187	/* B imm26 */
5188	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
5189	int32_t const offDisp = offTarget - offFixup;
5190	Assert(offDisp >= -33554432 && offDisp < 33554432);
5191	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
5192	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
5193	}
5194
5195	#else
5196	# error "Port me!"
5197	#endif
5198	}
5199
5200
5201	/**
5202	* Internal helper, don't call directly.
5203	*/
5204	DECL_INLINE_THROW(uint32_t)
5205	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5206	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
5207	{
5208	Assert(iBitNo < 64);
5209	#ifdef RT_ARCH_AMD64
5210	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5211	if (iBitNo < 8)
5212	{
5213	/* test Eb, imm8 */
5214	if (iGprSrc >= 4)
5215	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5216	pbCodeBuf[off++] = 0xf6;
5217	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5218	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
5219	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5220	}
5221	else
5222	{
5223	/* bt Ev, imm8 */
5224	if (iBitNo >= 32)
5225	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
5226	else if (iGprSrc >= 8)
5227	pbCodeBuf[off++] = X86_OP_REX_B;
5228	pbCodeBuf[off++] = 0x0f;
5229	pbCodeBuf[off++] = 0xba;
5230	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
5231	pbCodeBuf[off++] = iBitNo;
5232	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
5233	}
5234
5235	#elif defined(RT_ARCH_ARM64)
5236	/* Use the TBNZ instruction here. */
5237	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5238	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
5239	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
5240
5241	#else
5242	# error "Port me!"
5243	#endif
5244	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5245	return off;
5246	}
5247
5248
5249	/**
5250	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
5251	* @a iGprSrc.
5252	*
5253	* @note On ARM64 the range is only +/-8191 instructions.
5254	*/
5255	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5256	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5257	{
5258	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
5259	}
5260
5261
5262	/**
5263	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
5264	* _set_ in @a iGprSrc.
5265	*
5266	* @note On ARM64 the range is only +/-8191 instructions.
5267	*/
5268	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5269	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5270	{
5271	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
5272	}
5273
5274
5275	/**
5276	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
5277	* flags accordingly.
5278	*/
5279	DECL_INLINE_THROW(uint32_t)
5280	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
5281	{
5282	Assert(fBits != 0);
5283	#ifdef RT_ARCH_AMD64
5284
5285	if (fBits >= UINT32_MAX)
5286	{
5287	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5288
5289	/* test Ev,Gv */
5290	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5291	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
5292	pbCodeBuf[off++] = 0x85;
5293	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
5294
5295	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5296	}
5297	else if (fBits <= UINT32_MAX)
5298	{
5299	/* test Eb, imm8 or test Ev, imm32 */
5300	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
5301	if (fBits <= UINT8_MAX)
5302	{
5303	if (iGprSrc >= 4)
5304	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5305	pbCodeBuf[off++] = 0xf6;
5306	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5307	pbCodeBuf[off++] = (uint8_t)fBits;
5308	}
5309	else
5310	{
5311	if (iGprSrc >= 8)
5312	pbCodeBuf[off++] = X86_OP_REX_B;
5313	pbCodeBuf[off++] = 0xf7;
5314	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5315	pbCodeBuf[off++] = RT_BYTE1(fBits);
5316	pbCodeBuf[off++] = RT_BYTE2(fBits);
5317	pbCodeBuf[off++] = RT_BYTE3(fBits);
5318	pbCodeBuf[off++] = RT_BYTE4(fBits);
5319	}
5320	}
5321	/** @todo implement me. */
5322	else
5323	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
5324
5325	#elif defined(RT_ARCH_ARM64)
5326	uint32_t uImmR = 0;
5327	uint32_t uImmNandS = 0;
5328	if (Armv8A64ConvertMask64ToImmRImmS(fBits, &uImmNandS, &uImmR))
5329	{
5330	/* ands xzr, iGprSrc, #fBits */
5331	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5332	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR);
5333	}
5334	else
5335	{
5336	/* ands xzr, iGprSrc, iTmpReg */
5337	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5338	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5339	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
5340	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5341	}
5342
5343	#else
5344	# error "Port me!"
5345	#endif
5346	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5347	return off;
5348	}
5349
5350
5351	/**
5352	* Emits a test for any of the bits from @a fBits in the lower 32 bits of
5353	* @a iGprSrc, setting CPU flags accordingly.
5354	*
5355	* @note For ARM64 this only supports @a fBits values that can be expressed
5356	* using the two 6-bit immediates of the ANDS instruction. The caller
5357	* must make sure this is possible!
5358	*/
5359	DECL_FORCE_INLINE_THROW(uint32_t)
5360	iemNativeEmitTestAnyBitsInGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint32_t fBits)
5361	{
5362	Assert(fBits != 0);
5363
5364	#ifdef RT_ARCH_AMD64
5365	if (fBits <= UINT8_MAX)
5366	{
5367	/* test Eb, imm8 */
5368	if (iGprSrc >= 4)
5369	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5370	pCodeBuf[off++] = 0xf6;
5371	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5372	pCodeBuf[off++] = (uint8_t)fBits;
5373	}
5374	else
5375	{
5376	/* test Ev, imm32 */
5377	if (iGprSrc >= 8)
5378	pCodeBuf[off++] = X86_OP_REX_B;
5379	pCodeBuf[off++] = 0xf7;
5380	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5381	pCodeBuf[off++] = RT_BYTE1(fBits);
5382	pCodeBuf[off++] = RT_BYTE2(fBits);
5383	pCodeBuf[off++] = RT_BYTE3(fBits);
5384	pCodeBuf[off++] = RT_BYTE4(fBits);
5385	}
5386
5387	#elif defined(RT_ARCH_ARM64)
5388	/* ands xzr, src, #fBits */
5389	uint32_t uImmR = 0;
5390	uint32_t uImmNandS = 0;
5391	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5392	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5393	else
5394	# ifdef IEM_WITH_THROW_CATCH
5395	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5396	# else
5397	AssertReleaseFailedStmt(off = UINT32_MAX);
5398	# endif
5399
5400	#else
5401	# error "Port me!"
5402	#endif
5403	return off;
5404	}
5405
5406
5407
5408	/**
5409	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5410	* @a iGprSrc, setting CPU flags accordingly.
5411	*
5412	* @note For ARM64 this only supports @a fBits values that can be expressed
5413	* using the two 6-bit immediates of the ANDS instruction. The caller
5414	* must make sure this is possible!
5415	*/
5416	DECL_FORCE_INLINE_THROW(uint32_t)
5417	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5418	{
5419	Assert(fBits != 0);
5420
5421	#ifdef RT_ARCH_AMD64
5422	/* test Eb, imm8 */
5423	if (iGprSrc >= 4)
5424	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5425	pCodeBuf[off++] = 0xf6;
5426	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5427	pCodeBuf[off++] = fBits;
5428
5429	#elif defined(RT_ARCH_ARM64)
5430	/* ands xzr, src, #fBits */
5431	uint32_t uImmR = 0;
5432	uint32_t uImmNandS = 0;
5433	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5434	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5435	else
5436	# ifdef IEM_WITH_THROW_CATCH
5437	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5438	# else
5439	AssertReleaseFailedStmt(off = UINT32_MAX);
5440	# endif
5441
5442	#else
5443	# error "Port me!"
5444	#endif
5445	return off;
5446	}
5447
5448
5449	/**
5450	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5451	* @a iGprSrc, setting CPU flags accordingly.
5452	*/
5453	DECL_INLINE_THROW(uint32_t)
5454	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5455	{
5456	Assert(fBits != 0);
5457
5458	#ifdef RT_ARCH_AMD64
5459	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
5460
5461	#elif defined(RT_ARCH_ARM64)
5462	/* ands xzr, src, [tmp\|#imm] */
5463	uint32_t uImmR = 0;
5464	uint32_t uImmNandS = 0;
5465	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5466	{
5467	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5468	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5469	}
5470	else
5471	{
5472	/* Use temporary register for the 64-bit immediate. */
5473	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5474	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5475	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
5476	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5477	}
5478
5479	#else
5480	# error "Port me!"
5481	#endif
5482	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5483	return off;
5484	}
5485
5486
5487	/**
5488	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
5489	* are set in @a iGprSrc.
5490	*/
5491	DECL_INLINE_THROW(uint32_t)
5492	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5493	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5494	{
5495	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5496
5497	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5498	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5499
5500	return off;
5501	}
5502
5503
5504	/**
5505	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
5506	* are set in @a iGprSrc.
5507	*/
5508	DECL_INLINE_THROW(uint32_t)
5509	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5510	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5511	{
5512	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5513
5514	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5515	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
5516
5517	return off;
5518	}
5519
5520
5521	/**
5522	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5523	*
5524	* The operand size is given by @a f64Bit.
5525	*/
5526	DECL_FORCE_INLINE_THROW(uint32_t)
5527	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5528	uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5529	{
5530	Assert(idxLabel < pReNative->cLabels);
5531
5532	#ifdef RT_ARCH_AMD64
5533	/* test reg32,reg32 / test reg64,reg64 */
5534	if (f64Bit)
5535	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
5536	else if (iGprSrc >= 8)
5537	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
5538	pCodeBuf[off++] = 0x85;
5539	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
5540
5541	/* jnz idxLabel */
5542	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
5543	fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5544
5545	#elif defined(RT_ARCH_ARM64)
5546	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
5547	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(pReNative->paLabels[idxLabel].off - off),
5548	iGprSrc, f64Bit);
5549	else
5550	{
5551	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5552	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, 0, iGprSrc, f64Bit);
5553	}
5554
5555	#else
5556	# error "Port me!"
5557	#endif
5558	return off;
5559	}
5560
5561
5562	/**
5563	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5564	*
5565	* The operand size is given by @a f64Bit.
5566	*/
5567	DECL_FORCE_INLINE_THROW(uint32_t)
5568	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5569	bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5570	{
5571	#ifdef RT_ARCH_AMD64
5572	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
5573	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5574	#elif defined(RT_ARCH_ARM64)
5575	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1),
5576	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5577	#else
5578	# error "Port me!"
5579	#endif
5580	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5581	return off;
5582	}
5583
5584
5585	/**
5586	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5587	*
5588	* The operand size is given by @a f64Bit.
5589	*/
5590	DECL_FORCE_INLINE_THROW(uint32_t)
5591	iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5592	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5593	{
5594	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5595	f64Bit, false /fJmpIfNotZero/, idxLabel);
5596	}
5597
5598
5599	/**
5600	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5601	*
5602	* The operand size is given by @a f64Bit.
5603	*/
5604	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5605	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5606	{
5607	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /fJmpIfNotZero/, idxLabel);
5608	}
5609
5610
5611	/**
5612	* Emits code that jumps to a new label if @a iGprSrc is zero.
5613	*
5614	* The operand size is given by @a f64Bit.
5615	*/
5616	DECL_INLINE_THROW(uint32_t)
5617	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5618	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5619	{
5620	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5621	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5622	}
5623
5624
5625	/**
5626	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5627	*
5628	* The operand size is given by @a f64Bit.
5629	*/
5630	DECL_FORCE_INLINE_THROW(uint32_t)
5631	iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5632	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5633	{
5634	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5635	f64Bit, true /fJmpIfNotZero/, idxLabel);
5636	}
5637
5638
5639	/**
5640	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5641	*
5642	* The operand size is given by @a f64Bit.
5643	*/
5644	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5645	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5646	{
5647	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /fJmpIfNotZero/, idxLabel);
5648	}
5649
5650
5651	/**
5652	* Emits code that jumps to a new label if @a iGprSrc is not zero.
5653	*
5654	* The operand size is given by @a f64Bit.
5655	*/
5656	DECL_INLINE_THROW(uint32_t)
5657	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5658	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5659	{
5660	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5661	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5662	}
5663
5664
5665	/**
5666	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
5667	* differs.
5668	*/
5669	DECL_INLINE_THROW(uint32_t)
5670	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5671	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
5672	{
5673	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
5674	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5675	return off;
5676	}
5677
5678
5679	/**
5680	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
5681	*/
5682	DECL_INLINE_THROW(uint32_t)
5683	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5684	uint8_t iGprLeft, uint8_t iGprRight,
5685	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5686	{
5687	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5688	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
5689	}
5690
5691
5692	/**
5693	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
5694	*/
5695	DECL_INLINE_THROW(uint32_t)
5696	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5697	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
5698	{
5699	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
5700	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5701	return off;
5702	}
5703
5704
5705	/**
5706	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
5707	*/
5708	DECL_INLINE_THROW(uint32_t)
5709	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5710	uint8_t iGprSrc, uint64_t uImm,
5711	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5712	{
5713	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5714	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
5715	}
5716
5717
5718	/**
5719	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
5720	* @a uImm.
5721	*/
5722	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5723	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
5724	{
5725	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
5726	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5727	return off;
5728	}
5729
5730
5731	/**
5732	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
5733	* @a uImm.
5734	*/
5735	DECL_INLINE_THROW(uint32_t)
5736	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5737	uint8_t iGprSrc, uint32_t uImm,
5738	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5739	{
5740	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5741	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
5742	}
5743
5744
5745	/*********************************************************************************************************************************
5746	* Calls. *
5747	*********************************************************************************************************************************/
5748
5749	/**
5750	* Emits a call to a 64-bit address.
5751	*/
5752	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
5753	{
5754	#ifdef RT_ARCH_AMD64
5755	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
5756
5757	/* call rax */
5758	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
5759	pbCodeBuf[off++] = 0xff;
5760	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
5761
5762	#elif defined(RT_ARCH_ARM64)
5763	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
5764
5765	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5766	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
5767
5768	#else
5769	# error "port me"
5770	#endif
5771	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5772	return off;
5773	}
5774
5775
5776	/**
5777	* Emits code to load a stack variable into an argument GPR.
5778	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
5779	*/
5780	DECL_FORCE_INLINE_THROW(uint32_t)
5781	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
5782	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = UINT32_MAX,
5783	bool fSpilledVarsInVolatileRegs = false)
5784	{
5785	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
5786	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
5787	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
5788
5789	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
5790	if ( idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs)
5791	&& ( (RT_BIT_32(idxRegVar) & (~IEMNATIVE_CALL_VOLATILE_GREG_MASK \| fHstVolatileRegsAllowed))
5792	\|\| !fSpilledVarsInVolatileRegs ))
5793	{
5794	AssertStmt( !(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK)
5795	\|\| (RT_BIT_32(idxRegVar) & fHstVolatileRegsAllowed),
5796	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_13));
5797	if (!offAddend)
5798	{
5799	if (idxRegArg != idxRegVar)
5800	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
5801	}
5802	else
5803	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
5804	}
5805	else
5806	{
5807	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
5808	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
5809	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
5810	if (offAddend)
5811	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
5812	}
5813	return off;
5814	}
5815
5816
5817	/**
5818	* Emits code to load a stack or immediate variable value into an argument GPR,
5819	* optional with a addend.
5820	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
5821	*/
5822	DECL_FORCE_INLINE_THROW(uint32_t)
5823	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
5824	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = 0,
5825	bool fSpilledVarsInVolatileRegs = false)
5826	{
5827	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
5828	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
5829	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
5830	else
5831	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend,
5832	fHstVolatileRegsAllowed, fSpilledVarsInVolatileRegs);
5833	return off;
5834	}
5835
5836
5837	/**
5838	* Emits code to load the variable address into an argument GRP.
5839	*
5840	* This only works for uninitialized and stack variables.
5841	*/
5842	DECL_FORCE_INLINE_THROW(uint32_t)
5843	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
5844	bool fFlushShadows)
5845	{
5846	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
5847	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
5848	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
5849	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
5850
5851	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
5852	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
5853
5854	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
5855	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
5856	{
5857	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
5858	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
5859	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
5860	}
5861	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
5862	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
5863
5864	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
5865	}
5866
5867
5868	/** @} */
5869
5870	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
5871

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102765

Download in other formats: