IEMN8veRecompilerEmit.h@ 102790

Last change on this file since 102790 was 102790, checked in by vboxsync, 13 months ago
VMM/IEM: Emit TLB lookup for POP GPR instructions. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 206.5 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102790 2024-01-09 01:41:28Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	pCodeBuf[off++] = 0xcc;
88	RT_NOREF(uInfo); /** @todo use multibyte nop for info? */
89
90	#elif defined(RT_ARCH_ARM64)
91	pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
92
93	#else
94	# error "error"
95	#endif
96	return off;
97	}
98
99
100	/**
101	* Emit a breakpoint instruction.
102	*/
103	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
104	{
105	#ifdef RT_ARCH_AMD64
106	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
107	#elif defined(RT_ARCH_ARM64)
108	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
109	#else
110	# error "error"
111	#endif
112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
113	return off;
114	}
115
116
117	/*********************************************************************************************************************************
118	* Loads, Stores and Related Stuff. *
119	*********************************************************************************************************************************/
120
121	#ifdef RT_ARCH_AMD64
122	/**
123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
124	*/
125	DECL_FORCE_INLINE(uint32_t)
126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
127	{
128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
129	{
130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
133	}
134	else if (offDisp == (int8_t)offDisp)
135	{
136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
139	pbCodeBuf[off++] = (uint8_t)offDisp;
140	}
141	else
142	{
143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
150	}
151	return off;
152	}
153	#endif /* RT_ARCH_AMD64 */
154
155	/**
156	* Emits setting a GPR to zero.
157	*/
158	DECL_INLINE_THROW(uint32_t)
159	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
160	{
161	#ifdef RT_ARCH_AMD64
162	/* xor gpr32, gpr32 */
163	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
164	if (iGpr >= 8)
165	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
166	pbCodeBuf[off++] = 0x33;
167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
168
169	#elif defined(RT_ARCH_ARM64)
170	/* mov gpr, #0x0 */
171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
172	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
173
174	#else
175	# error "port me"
176	#endif
177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
178	return off;
179	}
180
181
182	/**
183	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
184	* buffer space.
185	*
186	* Max buffer consumption:
187	* - AMD64: 10 instruction bytes.
188	* - ARM64: 4 instruction words (16 bytes).
189	*/
190	DECL_FORCE_INLINE(uint32_t)
191	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
192	{
193	#ifdef RT_ARCH_AMD64
194	if (uImm64 == 0)
195	{
196	/* xor gpr, gpr */
197	if (iGpr >= 8)
198	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
199	pCodeBuf[off++] = 0x33;
200	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
201	}
202	else if (uImm64 <= UINT32_MAX)
203	{
204	/* mov gpr, imm32 */
205	if (iGpr >= 8)
206	pCodeBuf[off++] = X86_OP_REX_B;
207	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else if (uImm64 == (uint64_t)(int32_t)uImm64)
214	{
215	/* mov gpr, sx(imm32) */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xc7;
221	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
222	pCodeBuf[off++] = RT_BYTE1(uImm64);
223	pCodeBuf[off++] = RT_BYTE2(uImm64);
224	pCodeBuf[off++] = RT_BYTE3(uImm64);
225	pCodeBuf[off++] = RT_BYTE4(uImm64);
226	}
227	else
228	{
229	/* mov gpr, imm64 */
230	if (iGpr < 8)
231	pCodeBuf[off++] = X86_OP_REX_W;
232	else
233	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
234	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
235	pCodeBuf[off++] = RT_BYTE1(uImm64);
236	pCodeBuf[off++] = RT_BYTE2(uImm64);
237	pCodeBuf[off++] = RT_BYTE3(uImm64);
238	pCodeBuf[off++] = RT_BYTE4(uImm64);
239	pCodeBuf[off++] = RT_BYTE5(uImm64);
240	pCodeBuf[off++] = RT_BYTE6(uImm64);
241	pCodeBuf[off++] = RT_BYTE7(uImm64);
242	pCodeBuf[off++] = RT_BYTE8(uImm64);
243	}
244
245	#elif defined(RT_ARCH_ARM64)
246	/*
247	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
248	* supply remaining bits using 'movk grp, imm16, lsl #x'.
249	*
250	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
251	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
252	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
253	* after the first non-zero immediate component so we switch to movk for
254	* the remainder.
255	*/
256	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
257	+ !((uImm64 >> 16) & UINT16_MAX)
258	+ !((uImm64 >> 32) & UINT16_MAX)
259	+ !((uImm64 >> 48) & UINT16_MAX);
260	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
261	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
262	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
263	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
264	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
265	if (cFfffHalfWords <= cZeroHalfWords)
266	{
267	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
268
269	/* movz gpr, imm16 */
270	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
271	if (uImmPart \|\| cZeroHalfWords == 4)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #16 */
277	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
278	if (uImmPart)
279	{
280	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
281	fMovBase \|= RT_BIT_32(29);
282	}
283	/* mov[z/k] gpr, imm16, lsl #32 */
284	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
285	if (uImmPart)
286	{
287	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
288	fMovBase \|= RT_BIT_32(29);
289	}
290	/* mov[z/k] gpr, imm16, lsl #48 */
291	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
292	if (uImmPart)
293	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
294	}
295	else
296	{
297	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
298
299	/* find the first half-word that isn't UINT16_MAX. */
300	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
301	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
302	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
303
304	/* movn gpr, imm16, lsl #iHwNotFfff16 /
305	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
306	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
307	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
308	/* movk gpr, imm16 */
309	if (iHwNotFfff != 0)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #16 */
316	if (iHwNotFfff != 1)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
321	}
322	/* movk gpr, imm16, lsl #32 */
323	if (iHwNotFfff != 2)
324	{
325	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
326	if (uImmPart != UINT32_C(0xffff))
327	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
328	}
329	/* movk gpr, imm16, lsl #48 */
330	if (iHwNotFfff != 3)
331	{
332	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
333	if (uImmPart != UINT32_C(0xffff))
334	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
335	}
336	}
337
338	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
339	* clang 12.x does that, only to use the 'x' version for the
340	* addressing in the following ldr). */
341
342	#else
343	# error "port me"
344	#endif
345	return off;
346	}
347
348
349	/**
350	* Emits loading a constant into a 64-bit GPR
351	*/
352	DECL_INLINE_THROW(uint32_t)
353	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
354	{
355	#ifdef RT_ARCH_AMD64
356	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
357	#elif defined(RT_ARCH_ARM64)
358	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
359	#else
360	# error "port me"
361	#endif
362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
363	return off;
364	}
365
366
367	/**
368	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
369	* buffer space.
370	*
371	* Max buffer consumption:
372	* - AMD64: 6 instruction bytes.
373	* - ARM64: 2 instruction words (8 bytes).
374	*
375	* @note The top 32 bits will be cleared.
376	*/
377	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
378	{
379	#ifdef RT_ARCH_AMD64
380	if (uImm32 == 0)
381	{
382	/* xor gpr, gpr */
383	if (iGpr >= 8)
384	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
385	pCodeBuf[off++] = 0x33;
386	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
387	}
388	else
389	{
390	/* mov gpr, imm32 */
391	if (iGpr >= 8)
392	pCodeBuf[off++] = X86_OP_REX_B;
393	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
394	pCodeBuf[off++] = RT_BYTE1(uImm32);
395	pCodeBuf[off++] = RT_BYTE2(uImm32);
396	pCodeBuf[off++] = RT_BYTE3(uImm32);
397	pCodeBuf[off++] = RT_BYTE4(uImm32);
398	}
399
400	#elif defined(RT_ARCH_ARM64)
401	if ((uImm32 >> 16) == 0)
402	/* movz gpr, imm16 */
403	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
404	else if ((uImm32 & UINT32_C(0xffff)) == 0)
405	/* movz gpr, imm16, lsl #16 */
406	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
407	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
408	/* movn gpr, imm16, lsl #16 */
409	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
410	else if ((uImm32 >> 16) == UINT32_C(0xffff))
411	/* movn gpr, imm16 */
412	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
413	else
414	{
415	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
416	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
417	}
418
419	#else
420	# error "port me"
421	#endif
422	return off;
423	}
424
425
426	/**
427	* Emits loading a constant into a 32-bit GPR.
428	* @note The top 32 bits will be cleared.
429	*/
430	DECL_INLINE_THROW(uint32_t)
431	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
432	{
433	#ifdef RT_ARCH_AMD64
434	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
435	#elif defined(RT_ARCH_ARM64)
436	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
437	#else
438	# error "port me"
439	#endif
440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
441	return off;
442	}
443
444
445	/**
446	* Emits loading a constant into a 8-bit GPR
447	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
448	* only the ARM64 version does that.
449	*/
450	DECL_INLINE_THROW(uint32_t)
451	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
452	{
453	#ifdef RT_ARCH_AMD64
454	/* mov gpr, imm8 */
455	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
456	if (iGpr >= 8)
457	pbCodeBuf[off++] = X86_OP_REX_B;
458	else if (iGpr >= 4)
459	pbCodeBuf[off++] = X86_OP_REX;
460	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
461	pbCodeBuf[off++] = RT_BYTE1(uImm8);
462
463	#elif defined(RT_ARCH_ARM64)
464	/* movz gpr, imm16, lsl #0 */
465	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
466	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
467
468	#else
469	# error "port me"
470	#endif
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472	return off;
473	}
474
475
476	#ifdef RT_ARCH_AMD64
477	/**
478	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
479	*/
480	DECL_FORCE_INLINE(uint32_t)
481	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
482	{
483	if (offVCpu < 128)
484	{
485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
486	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
487	}
488	else
489	{
490	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
491	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
492	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
493	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
494	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
495	}
496	return off;
497	}
498
499	#elif defined(RT_ARCH_ARM64)
500
501	/**
502	* Common bit of iemNativeEmitLoadGprFromVCpuU64Ex and friends.
503	*
504	* @note Loads can use @a iGprReg for large offsets, stores requires a temporary
505	* registers (@a iGprTmp).
506	* @note DON'T try this with prefetch.
507	*/
508	DECL_FORCE_INLINE_THROW(uint32_t)
509	iemNativeEmitGprByVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu,
510	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
511	{
512	/*
513	* There are a couple of ldr variants that takes an immediate offset, so
514	* try use those if we can, otherwise we have to use the temporary register
515	* help with the addressing.
516	*/
517	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
518	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
519	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
520	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
521	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
522	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
523	else if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) \|\| iGprTmp != UINT8_MAX)
524	{
525	/* The offset is too large, so we must load it into a register and use
526	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
527	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
528	if (iGprTmp == UINT8_MAX)
529	iGprTmp = iGprReg;
530	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, offVCpu);
531	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, iGprTmp);
532	}
533	else
534	# ifdef IEM_WITH_THROW_CATCH
535	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
536	# else
537	AssertReleaseFailedStmt(off = UINT32_MAX);
538	# endif
539
540	return off;
541	}
542
543	/**
544	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
545	*/
546	DECL_FORCE_INLINE_THROW(uint32_t)
547	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
548	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
549	{
550	/*
551	* There are a couple of ldr variants that takes an immediate offset, so
552	* try use those if we can, otherwise we have to use the temporary register
553	* help with the addressing.
554	*/
555	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
556	{
557	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
558	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
559	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
560	}
561	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
562	{
563	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
564	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
565	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
566	}
567	else
568	{
569	/* The offset is too large, so we must load it into a register and use
570	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
571	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
572	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
574	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
575	IEMNATIVE_REG_FIXED_TMP0);
576	}
577	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
578	return off;
579	}
580
581	#endif /* RT_ARCH_ARM64 */
582
583
584	/**
585	* Emits a 64-bit GPR load of a VCpu value.
586	*/
587	DECL_FORCE_INLINE_THROW(uint32_t)
588	iemNativeEmitLoadGprFromVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
589	{
590	#ifdef RT_ARCH_AMD64
591	/* mov reg64, mem64 */
592	if (iGpr < 8)
593	pCodeBuf[off++] = X86_OP_REX_W;
594	else
595	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
596	pCodeBuf[off++] = 0x8b;
597	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
598
599	#elif defined(RT_ARCH_ARM64)
600	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
601
602	#else
603	# error "port me"
604	#endif
605	return off;
606	}
607
608
609	/**
610	* Emits a 64-bit GPR load of a VCpu value.
611	*/
612	DECL_INLINE_THROW(uint32_t)
613	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
614	{
615	#ifdef RT_ARCH_AMD64
616	off = iemNativeEmitLoadGprFromVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
617	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
618
619	#elif defined(RT_ARCH_ARM64)
620	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
621
622	#else
623	# error "port me"
624	#endif
625	return off;
626	}
627
628
629	/**
630	* Emits a 32-bit GPR load of a VCpu value.
631	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
632	*/
633	DECL_INLINE_THROW(uint32_t)
634	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
635	{
636	#ifdef RT_ARCH_AMD64
637	/* mov reg32, mem32 */
638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
639	if (iGpr >= 8)
640	pbCodeBuf[off++] = X86_OP_REX_R;
641	pbCodeBuf[off++] = 0x8b;
642	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
644
645	#elif defined(RT_ARCH_ARM64)
646	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
647
648	#else
649	# error "port me"
650	#endif
651	return off;
652	}
653
654
655	/**
656	* Emits a 16-bit GPR load of a VCpu value.
657	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
658	*/
659	DECL_INLINE_THROW(uint32_t)
660	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
661	{
662	#ifdef RT_ARCH_AMD64
663	/* movzx reg32, mem16 */
664	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
665	if (iGpr >= 8)
666	pbCodeBuf[off++] = X86_OP_REX_R;
667	pbCodeBuf[off++] = 0x0f;
668	pbCodeBuf[off++] = 0xb7;
669	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
670	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
671
672	#elif defined(RT_ARCH_ARM64)
673	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
674
675	#else
676	# error "port me"
677	#endif
678	return off;
679	}
680
681
682	/**
683	* Emits a 8-bit GPR load of a VCpu value.
684	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
685	*/
686	DECL_INLINE_THROW(uint32_t)
687	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
688	{
689	#ifdef RT_ARCH_AMD64
690	/* movzx reg32, mem8 */
691	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
692	if (iGpr >= 8)
693	pbCodeBuf[off++] = X86_OP_REX_R;
694	pbCodeBuf[off++] = 0x0f;
695	pbCodeBuf[off++] = 0xb6;
696	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
697	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
698
699	#elif defined(RT_ARCH_ARM64)
700	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
701
702	#else
703	# error "port me"
704	#endif
705	return off;
706	}
707
708
709	/**
710	* Emits a store of a GPR value to a 64-bit VCpu field.
711	*/
712	DECL_INLINE_THROW(uint32_t)
713	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
714	{
715	#ifdef RT_ARCH_AMD64
716	/* mov mem64, reg64 */
717	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
718	if (iGpr < 8)
719	pbCodeBuf[off++] = X86_OP_REX_W;
720	else
721	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
722	pbCodeBuf[off++] = 0x89;
723	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
724	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
725
726	#elif defined(RT_ARCH_ARM64)
727	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
728
729	#else
730	# error "port me"
731	#endif
732	return off;
733	}
734
735
736	/**
737	* Emits a store of a GPR value to a 32-bit VCpu field.
738	*/
739	DECL_INLINE_THROW(uint32_t)
740	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
741	{
742	#ifdef RT_ARCH_AMD64
743	/* mov mem32, reg32 */
744	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
745	if (iGpr >= 8)
746	pbCodeBuf[off++] = X86_OP_REX_R;
747	pbCodeBuf[off++] = 0x89;
748	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
749	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
750
751	#elif defined(RT_ARCH_ARM64)
752	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
753
754	#else
755	# error "port me"
756	#endif
757	return off;
758	}
759
760
761	/**
762	* Emits a store of a GPR value to a 16-bit VCpu field.
763	*/
764	DECL_INLINE_THROW(uint32_t)
765	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
766	{
767	#ifdef RT_ARCH_AMD64
768	/* mov mem16, reg16 */
769	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
770	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
771	if (iGpr >= 8)
772	pbCodeBuf[off++] = X86_OP_REX_R;
773	pbCodeBuf[off++] = 0x89;
774	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
775	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
776
777	#elif defined(RT_ARCH_ARM64)
778	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
779
780	#else
781	# error "port me"
782	#endif
783	return off;
784	}
785
786
787	/**
788	* Emits a store of a GPR value to a 8-bit VCpu field.
789	*/
790	DECL_INLINE_THROW(uint32_t)
791	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
792	{
793	#ifdef RT_ARCH_AMD64
794	/* mov mem8, reg8 */
795	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
796	if (iGpr >= 8)
797	pbCodeBuf[off++] = X86_OP_REX_R;
798	pbCodeBuf[off++] = 0x88;
799	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
800	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
801
802	#elif defined(RT_ARCH_ARM64)
803	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
804
805	#else
806	# error "port me"
807	#endif
808	return off;
809	}
810
811
812	/**
813	* Emits a store of an immediate value to a 8-bit VCpu field.
814	*/
815	DECL_INLINE_THROW(uint32_t)
816	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
817	{
818	#ifdef RT_ARCH_AMD64
819	/* mov mem8, imm8 */
820	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
821	pbCodeBuf[off++] = 0xc6;
822	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
823	pbCodeBuf[off++] = bImm;
824	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
825
826	#elif defined(RT_ARCH_ARM64)
827	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
828	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
829	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
830	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
831
832	#else
833	# error "port me"
834	#endif
835	return off;
836	}
837
838
839	/**
840	* Emits a load effective address to a GRP of a VCpu field.
841	*/
842	DECL_INLINE_THROW(uint32_t)
843	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
844	{
845	#ifdef RT_ARCH_AMD64
846	/* lea gprdst, [rbx + offDisp] */
847	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
848	if (iGprDst < 8)
849	pbCodeBuf[off++] = X86_OP_REX_W;
850	else
851	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
852	pbCodeBuf[off++] = 0x8d;
853	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
854
855	#elif defined(RT_ARCH_ARM64)
856	if (offVCpu < (unsigned)_4K)
857	{
858	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
859	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
860	}
861	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
862	{
863	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
864	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
865	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
866	}
867	else
868	{
869	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
870	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
871	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
872	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
873	}
874
875	#else
876	# error "port me"
877	#endif
878	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
879	return off;
880	}
881
882
883	/**
884	* Emits a gprdst = gprsrc load.
885	*/
886	DECL_FORCE_INLINE(uint32_t)
887	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
888	{
889	#ifdef RT_ARCH_AMD64
890	/* mov gprdst, gprsrc */
891	if ((iGprDst \| iGprSrc) >= 8)
892	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
893	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
894	: X86_OP_REX_W \| X86_OP_REX_R;
895	else
896	pCodeBuf[off++] = X86_OP_REX_W;
897	pCodeBuf[off++] = 0x8b;
898	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
899
900	#elif defined(RT_ARCH_ARM64)
901	/* mov dst, src; alias for: orr dst, xzr, src */
902	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
903
904	#else
905	# error "port me"
906	#endif
907	return off;
908	}
909
910
911	/**
912	* Emits a gprdst = gprsrc load.
913	*/
914	DECL_INLINE_THROW(uint32_t)
915	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
916	{
917	#ifdef RT_ARCH_AMD64
918	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
919	#elif defined(RT_ARCH_ARM64)
920	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
921	#else
922	# error "port me"
923	#endif
924	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
925	return off;
926	}
927
928
929	/**
930	* Emits a gprdst = gprsrc[31:0] load.
931	* @note Bits 63 thru 32 are cleared.
932	*/
933	DECL_FORCE_INLINE(uint32_t)
934	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
935	{
936	#ifdef RT_ARCH_AMD64
937	/* mov gprdst, gprsrc */
938	if ((iGprDst \| iGprSrc) >= 8)
939	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
940	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
941	: X86_OP_REX_R;
942	pCodeBuf[off++] = 0x8b;
943	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
944
945	#elif defined(RT_ARCH_ARM64)
946	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
947	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
948
949	#else
950	# error "port me"
951	#endif
952	return off;
953	}
954
955
956	/**
957	* Emits a gprdst = gprsrc[31:0] load.
958	* @note Bits 63 thru 32 are cleared.
959	*/
960	DECL_INLINE_THROW(uint32_t)
961	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
962	{
963	#ifdef RT_ARCH_AMD64
964	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
965	#elif defined(RT_ARCH_ARM64)
966	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
967	#else
968	# error "port me"
969	#endif
970	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
971	return off;
972	}
973
974
975	/**
976	* Emits a gprdst = gprsrc[15:0] load.
977	* @note Bits 63 thru 15 are cleared.
978	*/
979	DECL_INLINE_THROW(uint32_t)
980	iemNativeEmitLoadGprFromGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
981	{
982	#ifdef RT_ARCH_AMD64
983	/* movzx Gv,Ew */
984	if ((iGprDst \| iGprSrc) >= 8)
985	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
986	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
987	: X86_OP_REX_R;
988	pCodeBuf[off++] = 0x0f;
989	pCodeBuf[off++] = 0xb7;
990	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
991
992	#elif defined(RT_ARCH_ARM64)
993	/* and gprdst, gprsrc, #0xffff */
994	# if 1
995	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
996	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
997	# else
998	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
999	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1000	# endif
1001
1002	#else
1003	# error "port me"
1004	#endif
1005	return off;
1006	}
1007
1008
1009	/**
1010	* Emits a gprdst = gprsrc[15:0] load.
1011	* @note Bits 63 thru 15 are cleared.
1012	*/
1013	DECL_INLINE_THROW(uint32_t)
1014	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1015	{
1016	#ifdef RT_ARCH_AMD64
1017	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1018	#elif defined(RT_ARCH_ARM64)
1019	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1020	#else
1021	# error "port me"
1022	#endif
1023	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1024	return off;
1025	}
1026
1027
1028	/**
1029	* Emits a gprdst = gprsrc[7:0] load.
1030	* @note Bits 63 thru 8 are cleared.
1031	*/
1032	DECL_FORCE_INLINE(uint32_t)
1033	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1034	{
1035	#ifdef RT_ARCH_AMD64
1036	/* movzx Gv,Eb */
1037	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1038	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1039	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1040	: X86_OP_REX_R;
1041	else if (iGprSrc >= 4)
1042	pCodeBuf[off++] = X86_OP_REX;
1043	pCodeBuf[off++] = 0x0f;
1044	pCodeBuf[off++] = 0xb6;
1045	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1046
1047	#elif defined(RT_ARCH_ARM64)
1048	/* and gprdst, gprsrc, #0xff */
1049	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1050	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
1051
1052	#else
1053	# error "port me"
1054	#endif
1055	return off;
1056	}
1057
1058
1059	/**
1060	* Emits a gprdst = gprsrc[7:0] load.
1061	* @note Bits 63 thru 8 are cleared.
1062	*/
1063	DECL_INLINE_THROW(uint32_t)
1064	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1065	{
1066	#ifdef RT_ARCH_AMD64
1067	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1068	#elif defined(RT_ARCH_ARM64)
1069	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1070	#else
1071	# error "port me"
1072	#endif
1073	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1074	return off;
1075	}
1076
1077
1078	/**
1079	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1080	* @note Bits 63 thru 8 are cleared.
1081	*/
1082	DECL_INLINE_THROW(uint32_t)
1083	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1084	{
1085	#ifdef RT_ARCH_AMD64
1086	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1087
1088	/* movzx Gv,Ew */
1089	if ((iGprDst \| iGprSrc) >= 8)
1090	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1091	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1092	: X86_OP_REX_R;
1093	pbCodeBuf[off++] = 0x0f;
1094	pbCodeBuf[off++] = 0xb7;
1095	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1096
1097	/* shr Ev,8 */
1098	if (iGprDst >= 8)
1099	pbCodeBuf[off++] = X86_OP_REX_B;
1100	pbCodeBuf[off++] = 0xc1;
1101	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1102	pbCodeBuf[off++] = 8;
1103
1104	#elif defined(RT_ARCH_ARM64)
1105	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1106	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1107	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1108
1109	#else
1110	# error "port me"
1111	#endif
1112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1113	return off;
1114	}
1115
1116
1117	/**
1118	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1119	*/
1120	DECL_INLINE_THROW(uint32_t)
1121	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1122	{
1123	#ifdef RT_ARCH_AMD64
1124	/* movsxd r64, r/m32 */
1125	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1126	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1127	pbCodeBuf[off++] = 0x63;
1128	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1129
1130	#elif defined(RT_ARCH_ARM64)
1131	/* sxtw dst, src */
1132	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1133	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1134
1135	#else
1136	# error "port me"
1137	#endif
1138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1139	return off;
1140	}
1141
1142
1143	/**
1144	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1145	*/
1146	DECL_INLINE_THROW(uint32_t)
1147	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1148	{
1149	#ifdef RT_ARCH_AMD64
1150	/* movsx r64, r/m16 */
1151	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1152	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1153	pbCodeBuf[off++] = 0x0f;
1154	pbCodeBuf[off++] = 0xbf;
1155	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1156
1157	#elif defined(RT_ARCH_ARM64)
1158	/* sxth dst, src */
1159	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1160	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1161
1162	#else
1163	# error "port me"
1164	#endif
1165	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1166	return off;
1167	}
1168
1169
1170	/**
1171	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1172	*/
1173	DECL_INLINE_THROW(uint32_t)
1174	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1175	{
1176	#ifdef RT_ARCH_AMD64
1177	/* movsx r64, r/m16 */
1178	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1179	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1180	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1181	pbCodeBuf[off++] = 0x0f;
1182	pbCodeBuf[off++] = 0xbf;
1183	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1184
1185	#elif defined(RT_ARCH_ARM64)
1186	/* sxth dst32, src */
1187	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1188	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1189
1190	#else
1191	# error "port me"
1192	#endif
1193	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1194	return off;
1195	}
1196
1197
1198	/**
1199	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1200	*/
1201	DECL_INLINE_THROW(uint32_t)
1202	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1203	{
1204	#ifdef RT_ARCH_AMD64
1205	/* movsx r64, r/m8 */
1206	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1207	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1208	pbCodeBuf[off++] = 0x0f;
1209	pbCodeBuf[off++] = 0xbe;
1210	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1211
1212	#elif defined(RT_ARCH_ARM64)
1213	/* sxtb dst, src */
1214	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1215	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1216
1217	#else
1218	# error "port me"
1219	#endif
1220	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1221	return off;
1222	}
1223
1224
1225	/**
1226	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1227	* @note Bits 63 thru 32 are cleared.
1228	*/
1229	DECL_INLINE_THROW(uint32_t)
1230	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1231	{
1232	#ifdef RT_ARCH_AMD64
1233	/* movsx r32, r/m8 */
1234	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1235	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1236	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1237	else if (iGprSrc >= 4)
1238	pbCodeBuf[off++] = X86_OP_REX;
1239	pbCodeBuf[off++] = 0x0f;
1240	pbCodeBuf[off++] = 0xbe;
1241	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1242
1243	#elif defined(RT_ARCH_ARM64)
1244	/* sxtb dst32, src32 */
1245	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1246	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1247
1248	#else
1249	# error "port me"
1250	#endif
1251	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1252	return off;
1253	}
1254
1255
1256	/**
1257	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1258	* @note Bits 63 thru 16 are cleared.
1259	*/
1260	DECL_INLINE_THROW(uint32_t)
1261	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1262	{
1263	#ifdef RT_ARCH_AMD64
1264	/* movsx r16, r/m8 */
1265	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1266	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1267	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1268	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1269	else if (iGprSrc >= 4)
1270	pbCodeBuf[off++] = X86_OP_REX;
1271	pbCodeBuf[off++] = 0x0f;
1272	pbCodeBuf[off++] = 0xbe;
1273	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1274
1275	/* movzx r32, r/m16 */
1276	if (iGprDst >= 8)
1277	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1278	pbCodeBuf[off++] = 0x0f;
1279	pbCodeBuf[off++] = 0xb7;
1280	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1281
1282	#elif defined(RT_ARCH_ARM64)
1283	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1284	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1285	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1286	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1287	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1288
1289	#else
1290	# error "port me"
1291	#endif
1292	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1293	return off;
1294	}
1295
1296
1297	/**
1298	* Emits a gprdst = gprsrc + addend load.
1299	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1300	*/
1301	#ifdef RT_ARCH_AMD64
1302	DECL_INLINE_THROW(uint32_t)
1303	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1304	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1305	{
1306	Assert(iAddend != 0);
1307
1308	/* lea gprdst, [gprsrc + iAddend] */
1309	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1310	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1311	pbCodeBuf[off++] = 0x8d;
1312	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1313	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1314	return off;
1315	}
1316
1317	#elif defined(RT_ARCH_ARM64)
1318	DECL_INLINE_THROW(uint32_t)
1319	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1320	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1321	{
1322	if ((uint32_t)iAddend < 4096)
1323	{
1324	/* add dst, src, uimm12 */
1325	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1326	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1327	}
1328	else if ((uint32_t)-iAddend < 4096)
1329	{
1330	/* sub dst, src, uimm12 */
1331	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1332	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1333	}
1334	else
1335	{
1336	Assert(iGprSrc != iGprDst);
1337	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1338	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1339	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1340	}
1341	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1342	return off;
1343	}
1344	#else
1345	# error "port me"
1346	#endif
1347
1348	/**
1349	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1350	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1351	*/
1352	#ifdef RT_ARCH_AMD64
1353	DECL_INLINE_THROW(uint32_t)
1354	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1355	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1356	#else
1357	DECL_INLINE_THROW(uint32_t)
1358	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1359	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1360	#endif
1361	{
1362	if (iAddend != 0)
1363	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1364	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1365	}
1366
1367
1368	/**
1369	* Emits a gprdst = gprsrc32 + addend load.
1370	* @note Bits 63 thru 32 are cleared.
1371	*/
1372	DECL_INLINE_THROW(uint32_t)
1373	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1374	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1375	{
1376	Assert(iAddend != 0);
1377
1378	#ifdef RT_ARCH_AMD64
1379	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1380	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1381	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1382	if ((iGprDst \| iGprSrc) >= 8)
1383	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1384	pbCodeBuf[off++] = 0x8d;
1385	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1386
1387	#elif defined(RT_ARCH_ARM64)
1388	if ((uint32_t)iAddend < 4096)
1389	{
1390	/* add dst, src, uimm12 */
1391	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1392	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1393	}
1394	else if ((uint32_t)-iAddend < 4096)
1395	{
1396	/* sub dst, src, uimm12 */
1397	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1398	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1399	}
1400	else
1401	{
1402	Assert(iGprSrc != iGprDst);
1403	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1404	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1405	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1406	}
1407
1408	#else
1409	# error "port me"
1410	#endif
1411	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1412	return off;
1413	}
1414
1415
1416	/**
1417	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1418	*/
1419	DECL_INLINE_THROW(uint32_t)
1420	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1421	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1422	{
1423	if (iAddend != 0)
1424	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1425	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1426	}
1427
1428
1429	/**
1430	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1431	* destination.
1432	*/
1433	DECL_FORCE_INLINE(uint32_t)
1434	iemNativeEmitGprMergeInGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1435	{
1436	#ifdef RT_ARCH_AMD64
1437	/* mov reg16, r/m16 */
1438	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1439	if (idxDst >= 8 \|\| idxSrc >= 8)
1440	pCodeBuf[off++] = (idxDst < 8 ? 0 : X86_OP_REX_R) \| (idxSrc < 8 ? 0 : X86_OP_REX_B);
1441	pCodeBuf[off++] = 0x8b;
1442	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxDst & 7, idxSrc & 7);
1443
1444	#elif defined(RT_ARCH_ARM64)
1445	/* bfi w1, w2, 0, 16 - moves bits 15:0 from idxSrc to idxDst bits 15:0. */
1446	pCodeBuf[off++] = Armv8A64MkInstrBfi(idxDst, idxSrc, 0, 16);
1447
1448	#else
1449	# error "Port me!"
1450	#endif
1451	return off;
1452	}
1453
1454
1455	/**
1456	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1457	* destination.
1458	*/
1459	DECL_INLINE_THROW(uint32_t)
1460	iemNativeEmitGprMergeInGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1461	{
1462	#ifdef RT_ARCH_AMD64
1463	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, idxDst, idxSrc);
1464	#elif defined(RT_ARCH_ARM64)
1465	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, idxDst, idxSrc);
1466	#else
1467	# error "Port me!"
1468	#endif
1469	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1470	return off;
1471	}
1472
1473
1474	#ifdef RT_ARCH_AMD64
1475	/**
1476	* Common bit of iemNativeEmitLoadGprByBp and friends.
1477	*/
1478	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1479	PIEMRECOMPILERSTATE pReNativeAssert)
1480	{
1481	if (offDisp < 128 && offDisp >= -128)
1482	{
1483	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1484	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1485	}
1486	else
1487	{
1488	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1489	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1490	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1491	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1492	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1493	}
1494	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1495	return off;
1496	}
1497	#elif defined(RT_ARCH_ARM64)
1498	/**
1499	* Common bit of iemNativeEmitLoadGprByBp and friends.
1500	*/
1501	DECL_FORCE_INLINE_THROW(uint32_t)
1502	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1503	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1504	{
1505	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1506	{
1507	/* str w/ unsigned imm12 (scaled) */
1508	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1509	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1510	}
1511	else if (offDisp >= -256 && offDisp <= 256)
1512	{
1513	/* stur w/ signed imm9 (unscaled) */
1514	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1515	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1516	}
1517	else
1518	{
1519	/* Use temporary indexing register. */
1520	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1521	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1522	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1523	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1524	}
1525	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1526	return off;
1527	}
1528	#endif
1529
1530
1531	/**
1532	* Emits a 64-bit GRP load instruction with an BP relative source address.
1533	*/
1534	DECL_INLINE_THROW(uint32_t)
1535	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1536	{
1537	#ifdef RT_ARCH_AMD64
1538	/* mov gprdst, qword [rbp + offDisp] */
1539	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1540	if (iGprDst < 8)
1541	pbCodeBuf[off++] = X86_OP_REX_W;
1542	else
1543	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1544	pbCodeBuf[off++] = 0x8b;
1545	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1546
1547	#elif defined(RT_ARCH_ARM64)
1548	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1549
1550	#else
1551	# error "port me"
1552	#endif
1553	}
1554
1555
1556	/**
1557	* Emits a 32-bit GRP load instruction with an BP relative source address.
1558	* @note Bits 63 thru 32 of the GPR will be cleared.
1559	*/
1560	DECL_INLINE_THROW(uint32_t)
1561	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1562	{
1563	#ifdef RT_ARCH_AMD64
1564	/* mov gprdst, dword [rbp + offDisp] */
1565	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1566	if (iGprDst >= 8)
1567	pbCodeBuf[off++] = X86_OP_REX_R;
1568	pbCodeBuf[off++] = 0x8b;
1569	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1570
1571	#elif defined(RT_ARCH_ARM64)
1572	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1573
1574	#else
1575	# error "port me"
1576	#endif
1577	}
1578
1579
1580	/**
1581	* Emits a 16-bit GRP load instruction with an BP relative source address.
1582	* @note Bits 63 thru 16 of the GPR will be cleared.
1583	*/
1584	DECL_INLINE_THROW(uint32_t)
1585	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1586	{
1587	#ifdef RT_ARCH_AMD64
1588	/* movzx gprdst, word [rbp + offDisp] */
1589	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1590	if (iGprDst >= 8)
1591	pbCodeBuf[off++] = X86_OP_REX_R;
1592	pbCodeBuf[off++] = 0x0f;
1593	pbCodeBuf[off++] = 0xb7;
1594	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1595
1596	#elif defined(RT_ARCH_ARM64)
1597	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1598
1599	#else
1600	# error "port me"
1601	#endif
1602	}
1603
1604
1605	/**
1606	* Emits a 8-bit GRP load instruction with an BP relative source address.
1607	* @note Bits 63 thru 8 of the GPR will be cleared.
1608	*/
1609	DECL_INLINE_THROW(uint32_t)
1610	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1611	{
1612	#ifdef RT_ARCH_AMD64
1613	/* movzx gprdst, byte [rbp + offDisp] */
1614	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1615	if (iGprDst >= 8)
1616	pbCodeBuf[off++] = X86_OP_REX_R;
1617	pbCodeBuf[off++] = 0x0f;
1618	pbCodeBuf[off++] = 0xb6;
1619	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1620
1621	#elif defined(RT_ARCH_ARM64)
1622	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1623
1624	#else
1625	# error "port me"
1626	#endif
1627	}
1628
1629
1630	/**
1631	* Emits a load effective address to a GRP with an BP relative source address.
1632	*/
1633	DECL_INLINE_THROW(uint32_t)
1634	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1635	{
1636	#ifdef RT_ARCH_AMD64
1637	/* lea gprdst, [rbp + offDisp] */
1638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1639	if (iGprDst < 8)
1640	pbCodeBuf[off++] = X86_OP_REX_W;
1641	else
1642	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1643	pbCodeBuf[off++] = 0x8d;
1644	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1645
1646	#elif defined(RT_ARCH_ARM64)
1647	if ((uint32_t)offDisp < (unsigned)_4K)
1648	{
1649	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1650	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1651	}
1652	else if ((uint32_t)-offDisp < (unsigned)_4K)
1653	{
1654	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1655	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1656	}
1657	else
1658	{
1659	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1660	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1661	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1662	if (offDisp >= 0)
1663	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1664	else
1665	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1666	}
1667
1668	#else
1669	# error "port me"
1670	#endif
1671
1672	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1673	return off;
1674	}
1675
1676
1677	/**
1678	* Emits a 64-bit GPR store with an BP relative destination address.
1679	*
1680	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1681	*/
1682	DECL_INLINE_THROW(uint32_t)
1683	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1684	{
1685	#ifdef RT_ARCH_AMD64
1686	/* mov qword [rbp + offDisp], gprdst */
1687	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1688	if (iGprSrc < 8)
1689	pbCodeBuf[off++] = X86_OP_REX_W;
1690	else
1691	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1692	pbCodeBuf[off++] = 0x89;
1693	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1694
1695	#elif defined(RT_ARCH_ARM64)
1696	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1697	{
1698	/* str w/ unsigned imm12 (scaled) */
1699	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1700	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1701	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1702	}
1703	else if (offDisp >= -256 && offDisp <= 256)
1704	{
1705	/* stur w/ signed imm9 (unscaled) */
1706	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1707	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1708	}
1709	else if ((uint32_t)-offDisp < (unsigned)_4K)
1710	{
1711	/* Use temporary indexing register w/ sub uimm12. */
1712	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1713	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1714	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1715	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1716	}
1717	else
1718	{
1719	/* Use temporary indexing register. */
1720	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1721	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1722	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1723	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1724	}
1725	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1726	return off;
1727
1728	#else
1729	# error "Port me!"
1730	#endif
1731	}
1732
1733
1734	/**
1735	* Emits a 64-bit immediate store with an BP relative destination address.
1736	*
1737	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1738	*/
1739	DECL_INLINE_THROW(uint32_t)
1740	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1741	{
1742	#ifdef RT_ARCH_AMD64
1743	if ((int64_t)uImm64 == (int32_t)uImm64)
1744	{
1745	/* mov qword [rbp + offDisp], imm32 - sign extended */
1746	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1747	pbCodeBuf[off++] = X86_OP_REX_W;
1748	pbCodeBuf[off++] = 0xc7;
1749	if (offDisp < 128 && offDisp >= -128)
1750	{
1751	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1752	pbCodeBuf[off++] = (uint8_t)offDisp;
1753	}
1754	else
1755	{
1756	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1757	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1758	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1759	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1760	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1761	}
1762	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1763	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1764	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1765	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1766	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1767	return off;
1768	}
1769	#endif
1770
1771	/* Load tmp0, imm64; Store tmp to bp+disp. */
1772	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1773	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1774	}
1775
1776	#if defined(RT_ARCH_ARM64)
1777
1778	/**
1779	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1780	*
1781	* @note Odd and large @a offDisp values requires a temporary, unless it's a
1782	* load and @a iGprReg differs from @a iGprBase. Will assert / throw if
1783	* caller does not heed this.
1784	*
1785	* @note DON'T try this with prefetch.
1786	*/
1787	DECL_FORCE_INLINE_THROW(uint32_t)
1788	iemNativeEmitGprByGprLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp,
1789	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
1790	{
1791	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1792	{
1793	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1794	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1795	}
1796	else if ( ( !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation)
1797	&& iGprReg != iGprBase)
1798	\|\| iGprTmp != UINT8_MAX)
1799	{
1800	/* The offset is too large, so we must load it into a register and use
1801	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1802	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1803	if (iGprTmp == UINT8_MAX)
1804	iGprTmp = iGprReg;
1805	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (int64_t)offDisp);
1806	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, iGprTmp);
1807	}
1808	else
1809	# ifdef IEM_WITH_THROW_CATCH
1810	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
1811	# else
1812	AssertReleaseFailedStmt(off = UINT32_MAX);
1813	# endif
1814	return off;
1815	}
1816
1817	/**
1818	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1819	*/
1820	DECL_FORCE_INLINE_THROW(uint32_t)
1821	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1822	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1823	{
1824	/*
1825	* There are a couple of ldr variants that takes an immediate offset, so
1826	* try use those if we can, otherwise we have to use the temporary register
1827	* help with the addressing.
1828	*/
1829	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1830	{
1831	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1832	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1833	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1834	}
1835	else
1836	{
1837	/* The offset is too large, so we must load it into a register and use
1838	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1839	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1840	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (int64_t)offDisp);
1841
1842	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1843	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1844
1845	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1846	}
1847	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1848	return off;
1849	}
1850
1851	#endif /* RT_ARCH_ARM64 */
1852
1853	/**
1854	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1855	*
1856	* @note ARM64: Misaligned @a offDisp values and values not in the
1857	* -0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
1858	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
1859	* does not heed this.
1860	*/
1861	DECL_FORCE_INLINE_THROW(uint32_t)
1862	iemNativeEmitLoadGprByGprU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1863	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1864	{
1865	#ifdef RT_ARCH_AMD64
1866	/* mov reg64, mem64 */
1867	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1868	pCodeBuf[off++] = 0x8b;
1869	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1870	RT_NOREF(iGprTmp);
1871
1872	#elif defined(RT_ARCH_ARM64)
1873	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1874	kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t), iGprTmp);
1875
1876	#else
1877	# error "port me"
1878	#endif
1879	return off;
1880	}
1881
1882
1883	/**
1884	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1885	*/
1886	DECL_INLINE_THROW(uint32_t)
1887	iemNativeEmitLoadGprByGprU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1888	{
1889	#ifdef RT_ARCH_AMD64
1890	off = iemNativeEmitLoadGprByGprU64Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
1891	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1892
1893	#elif defined(RT_ARCH_ARM64)
1894	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1895
1896	#else
1897	# error "port me"
1898	#endif
1899	return off;
1900	}
1901
1902
1903	/**
1904	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1905	*
1906	* @note ARM64: Misaligned @a offDisp values and values not in the
1907	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
1908	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1909	* caller does not heed this.
1910	*
1911	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1912	*/
1913	DECL_FORCE_INLINE_THROW(uint32_t)
1914	iemNativeEmitLoadGprByGprU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1915	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1916	{
1917	#ifdef RT_ARCH_AMD64
1918	/* mov reg32, mem32 */
1919	if (iGprDst >= 8 \|\| iGprBase >= 8)
1920	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1921	pCodeBuf[off++] = 0x8b;
1922	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1923	RT_NOREF(iGprTmp);
1924
1925	#elif defined(RT_ARCH_ARM64)
1926	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1927	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t), iGprTmp);
1928
1929	#else
1930	# error "port me"
1931	#endif
1932	return off;
1933	}
1934
1935
1936	/**
1937	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1938	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1939	*/
1940	DECL_INLINE_THROW(uint32_t)
1941	iemNativeEmitLoadGprByGprU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1942	{
1943	#ifdef RT_ARCH_AMD64
1944	off = iemNativeEmitLoadGprByGprU32Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
1945	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1946
1947	#elif defined(RT_ARCH_ARM64)
1948	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1949
1950	#else
1951	# error "port me"
1952	#endif
1953	return off;
1954	}
1955
1956
1957	/**
1958	* Emits a 32-bit GPR load via a GPR base address with a displacement,
1959	* sign-extending the value to 64 bits.
1960	*
1961	* @note ARM64: Misaligned @a offDisp values and values not in the
1962	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
1963	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1964	* caller does not heed this.
1965	*/
1966	DECL_FORCE_INLINE_THROW(uint32_t)
1967	iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
1968	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
1969	{
1970	#ifdef RT_ARCH_AMD64
1971	/* movsxd reg64, mem32 */
1972	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1973	pCodeBuf[off++] = 0x63;
1974	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
1975	RT_NOREF(iGprTmp);
1976
1977	#elif defined(RT_ARCH_ARM64)
1978	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
1979	kArmv8A64InstrLdStType_Ld_SignWord64, sizeof(uint32_t), iGprTmp);
1980
1981	#else
1982	# error "port me"
1983	#endif
1984	return off;
1985	}
1986
1987
1988	/**
1989	* Emits a 16-bit GPR load via a GPR base address with a displacement.
1990	*
1991	* @note ARM64: Misaligned @a offDisp values and values not in the
1992	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
1993	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
1994	* caller does not heed this.
1995	*
1996	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
1997	*/
1998	DECL_FORCE_INLINE_THROW(uint32_t)
1999	iemNativeEmitLoadGprByGprU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2000	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2001	{
2002	#ifdef RT_ARCH_AMD64
2003	/* movzx reg32, mem16 */
2004	if (iGprDst >= 8 \|\| iGprBase >= 8)
2005	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2006	pCodeBuf[off++] = 0x0f;
2007	pCodeBuf[off++] = 0xb7;
2008	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2009	RT_NOREF(iGprTmp);
2010
2011	#elif defined(RT_ARCH_ARM64)
2012	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2013	kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t), iGprTmp);
2014
2015	#else
2016	# error "port me"
2017	#endif
2018	return off;
2019	}
2020
2021
2022	/**
2023	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2024	* sign-extending the value to 64 bits.
2025	*
2026	* @note ARM64: Misaligned @a offDisp values and values not in the
2027	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2028	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2029	* caller does not heed this.
2030	*/
2031	DECL_FORCE_INLINE_THROW(uint32_t)
2032	iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2033	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2034	{
2035	#ifdef RT_ARCH_AMD64
2036	/* movsx reg64, mem16 */
2037	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2038	pCodeBuf[off++] = 0x0f;
2039	pCodeBuf[off++] = 0xbf;
2040	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2041	RT_NOREF(iGprTmp);
2042
2043	#elif defined(RT_ARCH_ARM64)
2044	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2045	kArmv8A64InstrLdStType_Ld_SignHalf64, sizeof(uint16_t), iGprTmp);
2046
2047	#else
2048	# error "port me"
2049	#endif
2050	return off;
2051	}
2052
2053
2054	/**
2055	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2056	* sign-extending the value to 32 bits.
2057	*
2058	* @note ARM64: Misaligned @a offDisp values and values not in the
2059	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2060	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2061	* caller does not heed this.
2062	*
2063	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2064	*/
2065	DECL_FORCE_INLINE_THROW(uint32_t)
2066	iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2067	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2068	{
2069	#ifdef RT_ARCH_AMD64
2070	/* movsx reg32, mem16 */
2071	if (iGprDst >= 8 \|\| iGprBase >= 8)
2072	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2073	pCodeBuf[off++] = 0x0f;
2074	pCodeBuf[off++] = 0xbf;
2075	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2076	RT_NOREF(iGprTmp);
2077
2078	#elif defined(RT_ARCH_ARM64)
2079	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2080	kArmv8A64InstrLdStType_Ld_SignHalf32, sizeof(uint16_t), iGprTmp);
2081
2082	#else
2083	# error "port me"
2084	#endif
2085	return off;
2086	}
2087
2088
2089	/**
2090	* Emits a 8-bit GPR load via a GPR base address with a displacement.
2091	*
2092	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2093	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2094	* same. Will assert / throw if caller does not heed this.
2095	*
2096	* @note Bits 63 thru 8 in @a iGprDst will be cleared.
2097	*/
2098	DECL_FORCE_INLINE_THROW(uint32_t)
2099	iemNativeEmitLoadGprByGprU8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2100	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2101	{
2102	#ifdef RT_ARCH_AMD64
2103	/* movzx reg32, mem8 */
2104	if (iGprDst >= 8 \|\| iGprBase >= 8)
2105	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2106	pCodeBuf[off++] = 0x0f;
2107	pCodeBuf[off++] = 0xb6;
2108	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2109	RT_NOREF(iGprTmp);
2110
2111	#elif defined(RT_ARCH_ARM64)
2112	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2113	kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t), iGprTmp);
2114
2115	#else
2116	# error "port me"
2117	#endif
2118	return off;
2119	}
2120
2121
2122	/**
2123	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2124	* sign-extending the value to 64 bits.
2125	*
2126	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2127	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2128	* same. Will assert / throw if caller does not heed this.
2129	*/
2130	DECL_FORCE_INLINE_THROW(uint32_t)
2131	iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2132	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2133	{
2134	#ifdef RT_ARCH_AMD64
2135	/* movsx reg64, mem8 */
2136	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2137	pCodeBuf[off++] = 0x0f;
2138	pCodeBuf[off++] = 0xbe;
2139	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2140	RT_NOREF(iGprTmp);
2141
2142	#elif defined(RT_ARCH_ARM64)
2143	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2144	kArmv8A64InstrLdStType_Ld_SignByte64, sizeof(uint8_t), iGprTmp);
2145
2146	#else
2147	# error "port me"
2148	#endif
2149	return off;
2150	}
2151
2152
2153	/**
2154	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2155	* sign-extending the value to 32 bits.
2156	*
2157	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2158	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2159	* same. Will assert / throw if caller does not heed this.
2160	*
2161	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2162	*/
2163	DECL_FORCE_INLINE_THROW(uint32_t)
2164	iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2165	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2166	{
2167	#ifdef RT_ARCH_AMD64
2168	/* movsx reg32, mem8 */
2169	if (iGprDst >= 8 \|\| iGprBase >= 8)
2170	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2171	pCodeBuf[off++] = 0x0f;
2172	pCodeBuf[off++] = 0xbe;
2173	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2174	RT_NOREF(iGprTmp);
2175
2176	#elif defined(RT_ARCH_ARM64)
2177	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2178	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2179
2180	#else
2181	# error "port me"
2182	#endif
2183	return off;
2184	}
2185
2186
2187	/**
2188	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2189	* sign-extending the value to 16 bits.
2190	*
2191	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2192	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2193	* same. Will assert / throw if caller does not heed this.
2194	*
2195	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2196	*/
2197	DECL_FORCE_INLINE_THROW(uint32_t)
2198	iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2199	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2200	{
2201	#ifdef RT_ARCH_AMD64
2202	/* movsx reg32, mem8 */
2203	if (iGprDst >= 8 \|\| iGprBase >= 8)
2204	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2205	pCodeBuf[off++] = 0x0f;
2206	pCodeBuf[off++] = 0xbe;
2207	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2208	# if 1 /** @todo use 'movzx reg32, reg16' instead of 'and reg32, 0ffffh' ? */
2209	/* and reg32, 0xffffh */
2210	if (iGprDst >= 8)
2211	pCodeBuf[off++] = X86_OP_REX_B;
2212	pCodeBuf[off++] = 0x81;
2213	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2214	pCodeBuf[off++] = 0xff;
2215	pCodeBuf[off++] = 0xff;
2216	pCodeBuf[off++] = 0;
2217	pCodeBuf[off++] = 0;
2218	# else
2219	/* movzx reg32, reg16 */
2220	if (iGprDst >= 8)
2221	pCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2222	pCodeBuf[off++] = 0x0f;
2223	pCodeBuf[off++] = 0xb7;
2224	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2225	# endif
2226	RT_NOREF(iGprTmp);
2227
2228	#elif defined(RT_ARCH_ARM64)
2229	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2230	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2231	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
2232	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /64Bit/);
2233
2234	#else
2235	# error "port me"
2236	#endif
2237	return off;
2238	}
2239
2240
2241	/**
2242	* Emits a 64-bit GPR store via a GPR base address with a displacement.
2243	*
2244	* @note ARM64: Misaligned @a offDisp values and values not in the
2245	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2246	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2247	* does not heed this.
2248	*/
2249	DECL_FORCE_INLINE_THROW(uint32_t)
2250	iemNativeEmitStoreGpr64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2251	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2252	{
2253	#ifdef RT_ARCH_AMD64
2254	/* mov mem64, reg64 */
2255	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2256	pCodeBuf[off++] = 0x89;
2257	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2258	RT_NOREF(iGprTmp);
2259
2260	#elif defined(RT_ARCH_ARM64)
2261	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2262	kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
2263
2264	#else
2265	# error "port me"
2266	#endif
2267	return off;
2268	}
2269
2270
2271	/**
2272	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2273	*
2274	* @note ARM64: Misaligned @a offDisp values and values not in the
2275	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2276	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2277	* does not heed this.
2278	*/
2279	DECL_FORCE_INLINE_THROW(uint32_t)
2280	iemNativeEmitStoreGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2281	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2282	{
2283	#ifdef RT_ARCH_AMD64
2284	/* mov mem32, reg32 */
2285	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2286	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2287	pCodeBuf[off++] = 0x89;
2288	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2289	RT_NOREF(iGprTmp);
2290
2291	#elif defined(RT_ARCH_ARM64)
2292	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2293	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2294
2295	#else
2296	# error "port me"
2297	#endif
2298	return off;
2299	}
2300
2301
2302	/**
2303	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2304	*
2305	* @note ARM64: Misaligned @a offDisp values and values not in the
2306	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2307	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2308	* does not heed this.
2309	*/
2310	DECL_FORCE_INLINE_THROW(uint32_t)
2311	iemNativeEmitStoreGpr16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2312	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2313	{
2314	#ifdef RT_ARCH_AMD64
2315	/* mov mem16, reg16 */
2316	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2317	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2318	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2319	pCodeBuf[off++] = 0x89;
2320	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2321	RT_NOREF(iGprTmp);
2322
2323	#elif defined(RT_ARCH_ARM64)
2324	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2325	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2326
2327	#else
2328	# error "port me"
2329	#endif
2330	return off;
2331	}
2332
2333
2334	/**
2335	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2336	*
2337	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2338	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2339	* same. Will assert / throw if caller does not heed this.
2340	*/
2341	DECL_FORCE_INLINE_THROW(uint32_t)
2342	iemNativeEmitStoreGpr8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2343	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2344	{
2345	#ifdef RT_ARCH_AMD64
2346	/* mov mem8, reg8 */
2347	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2348	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2349	else if (iGprSrc >= 4)
2350	pCodeBuf[off++] = X86_OP_REX;
2351	pCodeBuf[off++] = 0x88;
2352	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2353	RT_NOREF(iGprTmp);
2354
2355	#elif defined(RT_ARCH_ARM64)
2356	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2357	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2358
2359	#else
2360	# error "port me"
2361	#endif
2362	return off;
2363	}
2364
2365
2366	/**
2367	* Emits a 64-bit immediate store via a GPR base address with a displacement.
2368	*
2369	* @note This will always require @a iGprTmpImm on ARM (except for uImm=0), on
2370	* AMD64 it depends on the immediate value.
2371	*
2372	* @note ARM64: Misaligned @a offDisp values and values not in the
2373	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2374	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2375	* does not heed this.
2376	*/
2377	DECL_FORCE_INLINE_THROW(uint32_t)
2378	iemNativeEmitStoreImm64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint64_t uImm, uint8_t iGprBase,
2379	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2380	{
2381	#ifdef RT_ARCH_AMD64
2382	if ((int32_t)uImm == (int64_t)uImm)
2383	{
2384	/* mov mem64, imm32 (sign-extended) */
2385	pCodeBuf[off++] = X86_OP_REX_W \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2386	pCodeBuf[off++] = 0xc7;
2387	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2388	pCodeBuf[off++] = RT_BYTE1(uImm);
2389	pCodeBuf[off++] = RT_BYTE2(uImm);
2390	pCodeBuf[off++] = RT_BYTE3(uImm);
2391	pCodeBuf[off++] = RT_BYTE4(uImm);
2392	}
2393	else if (iGprImmTmp != UINT8_MAX \|\| iGprTmp != UINT8_MAX)
2394	{
2395	/* require temporary register. */
2396	if (iGprImmTmp == UINT8_MAX)
2397	iGprImmTmp = iGprTmp;
2398	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2399	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp);
2400	}
2401	else
2402	# ifdef IEM_WITH_THROW_CATCH
2403	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2404	# else
2405	AssertReleaseFailedStmt(off = UINT32_MAX);
2406	# endif
2407
2408	#elif defined(RT_ARCH_ARM64)
2409	if (uImm == 0)
2410	iGprImmTmp = ARMV8_A64_REG_XZR;
2411	else
2412	{
2413	Assert(iGprImmTmp < 31);
2414	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2415	}
2416	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp, iGprTmp);
2417
2418	#else
2419	# error "port me"
2420	#endif
2421	return off;
2422	}
2423
2424
2425	/**
2426	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2427	*
2428	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2429	*
2430	* @note ARM64: Misaligned @a offDisp values and values not in the
2431	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2432	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2433	* does not heed this.
2434	*/
2435	DECL_FORCE_INLINE_THROW(uint32_t)
2436	iemNativeEmitStoreImm32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uImm, uint8_t iGprBase,
2437	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2438	{
2439	#ifdef RT_ARCH_AMD64
2440	/* mov mem32, imm32 */
2441	if (iGprBase >= 8)
2442	pCodeBuf[off++] = X86_OP_REX_B;
2443	pCodeBuf[off++] = 0xc7;
2444	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2445	pCodeBuf[off++] = RT_BYTE1(uImm);
2446	pCodeBuf[off++] = RT_BYTE2(uImm);
2447	pCodeBuf[off++] = RT_BYTE3(uImm);
2448	pCodeBuf[off++] = RT_BYTE4(uImm);
2449	RT_NOREF(iGprImmTmp, iGprTmp);
2450
2451	#elif defined(RT_ARCH_ARM64)
2452	Assert(iGprImmTmp < 31);
2453	if (uImm == 0)
2454	iGprImmTmp = ARMV8_A64_REG_XZR;
2455	else
2456	{
2457	Assert(iGprImmTmp < 31);
2458	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2459	}
2460	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2461	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2462
2463	#else
2464	# error "port me"
2465	#endif
2466	return off;
2467	}
2468
2469
2470	/**
2471	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2472	*
2473	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2474	*
2475	* @note ARM64: Misaligned @a offDisp values and values not in the
2476	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2477	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2478	* does not heed this.
2479	*/
2480	DECL_FORCE_INLINE_THROW(uint32_t)
2481	iemNativeEmitStoreImm16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint8_t iGprBase,
2482	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2483	{
2484	#ifdef RT_ARCH_AMD64
2485	/* mov mem16, imm16 */
2486	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2487	if (iGprBase >= 8)
2488	pCodeBuf[off++] = X86_OP_REX_B;
2489	pCodeBuf[off++] = 0xc7;
2490	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2491	pCodeBuf[off++] = RT_BYTE1(uImm);
2492	pCodeBuf[off++] = RT_BYTE2(uImm);
2493	RT_NOREF(iGprImmTmp, iGprTmp);
2494
2495	#elif defined(RT_ARCH_ARM64)
2496	if (uImm == 0)
2497	iGprImmTmp = ARMV8_A64_REG_XZR;
2498	else
2499	{
2500	Assert(iGprImmTmp < 31);
2501	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2502	}
2503	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2504	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2505
2506	#else
2507	# error "port me"
2508	#endif
2509	return off;
2510	}
2511
2512
2513	/**
2514	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2515	*
2516	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2517	*
2518	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2519	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2520	* same. Will assert / throw if caller does not heed this.
2521	*/
2522	DECL_FORCE_INLINE_THROW(uint32_t)
2523	iemNativeEmitStoreImm8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t uImm, uint8_t iGprBase,
2524	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2525	{
2526	#ifdef RT_ARCH_AMD64
2527	/* mov mem8, imm8 */
2528	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2529	if (iGprBase >= 8)
2530	pCodeBuf[off++] = X86_OP_REX_B;
2531	pCodeBuf[off++] = 0xc6;
2532	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2533	pCodeBuf[off++] = uImm;
2534	RT_NOREF(iGprImmTmp, iGprTmp);
2535
2536	#elif defined(RT_ARCH_ARM64)
2537	if (uImm == 0)
2538	iGprImmTmp = ARMV8_A64_REG_XZR;
2539	else
2540	{
2541	Assert(iGprImmTmp < 31);
2542	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2543	}
2544	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2545	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2546
2547	#else
2548	# error "port me"
2549	#endif
2550	return off;
2551	}
2552
2553
2554
2555	/*********************************************************************************************************************************
2556	* Subtraction and Additions *
2557	*********************************************************************************************************************************/
2558
2559	/**
2560	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
2561	* @note The AMD64 version sets flags.
2562	*/
2563	DECL_INLINE_THROW(uint32_t)
2564	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2565	{
2566	#if defined(RT_ARCH_AMD64)
2567	/* sub Gv,Ev */
2568	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2569	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2570	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2571	pbCodeBuf[off++] = 0x2b;
2572	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2573
2574	#elif defined(RT_ARCH_ARM64)
2575	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2576	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
2577
2578	#else
2579	# error "Port me"
2580	#endif
2581	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2582	return off;
2583	}
2584
2585
2586	/**
2587	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2588	* @note The AMD64 version sets flags.
2589	*/
2590	DECL_FORCE_INLINE(uint32_t)
2591	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2592	{
2593	#if defined(RT_ARCH_AMD64)
2594	/* sub Gv,Ev */
2595	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
2596	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
2597	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2598	pCodeBuf[off++] = 0x2b;
2599	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2600
2601	#elif defined(RT_ARCH_ARM64)
2602	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
2603
2604	#else
2605	# error "Port me"
2606	#endif
2607	return off;
2608	}
2609
2610
2611	/**
2612	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2613	* @note The AMD64 version sets flags.
2614	*/
2615	DECL_INLINE_THROW(uint32_t)
2616	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2617	{
2618	#if defined(RT_ARCH_AMD64)
2619	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
2620	#elif defined(RT_ARCH_ARM64)
2621	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
2622	#else
2623	# error "Port me"
2624	#endif
2625	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2626	return off;
2627	}
2628
2629
2630	/**
2631	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2632	*
2633	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2634	*
2635	* @note Larger constants will require a temporary register. Failing to specify
2636	* one when needed will trigger fatal assertion / throw.
2637	*/
2638	DECL_FORCE_INLINE_THROW(uint32_t)
2639	iemNativeEmitSubGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2640	uint8_t iGprTmp = UINT8_MAX)
2641	{
2642	#ifdef RT_ARCH_AMD64
2643	pCodeBuf[off++] = iGprDst >= 8 ? X86_OP_REX_W \| X86_OP_REX_B : X86_OP_REX_W;
2644	if (iSubtrahend == 1)
2645	{
2646	/* dec r/m64 */
2647	pCodeBuf[off++] = 0xff;
2648	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2649	}
2650	else if (iSubtrahend == -1)
2651	{
2652	/* inc r/m64 */
2653	pCodeBuf[off++] = 0xff;
2654	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2655	}
2656	else if ((int8_t)iSubtrahend == iSubtrahend)
2657	{
2658	/* sub r/m64, imm8 */
2659	pCodeBuf[off++] = 0x83;
2660	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2661	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2662	}
2663	else if ((int32_t)iSubtrahend == iSubtrahend)
2664	{
2665	/* sub r/m64, imm32 */
2666	pCodeBuf[off++] = 0x81;
2667	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2668	pCodeBuf[off++] = RT_BYTE1((uint64_t)iSubtrahend);
2669	pCodeBuf[off++] = RT_BYTE2((uint64_t)iSubtrahend);
2670	pCodeBuf[off++] = RT_BYTE3((uint64_t)iSubtrahend);
2671	pCodeBuf[off++] = RT_BYTE4((uint64_t)iSubtrahend);
2672	}
2673	else if (iGprTmp != UINT8_MAX)
2674	{
2675	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off - 1, iGprTmp, (uint64_t)iSubtrahend);
2676	/* sub r/m64, r64 */
2677	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B) \| (iGprTmp < 8 ? 0 : X86_OP_REX_R);
2678	pCodeBuf[off++] = 0x29;
2679	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprTmp & 7, iGprDst & 7);
2680	}
2681	else
2682	# ifdef IEM_WITH_THROW_CATCH
2683	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2684	# else
2685	AssertReleaseFailedStmt(off = UINT32_MAX);
2686	# endif
2687
2688	#elif defined(RT_ARCH_ARM64)
2689	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2690	if (uAbsSubtrahend < 4096)
2691	{
2692	if (iSubtrahend >= 0)
2693	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2694	else
2695	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2696	}
2697	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2698	{
2699	if (iSubtrahend >= 0)
2700	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2701	true /f64Bit/, false /fSetFlags/, true /fShift/);
2702	else
2703	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2704	true /f64Bit/, false /fSetFlags/, true /fShift/);
2705	}
2706	else if (iGprTmp != UINT8_MAX)
2707	{
2708	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (uint64_t)iSubtrahend);
2709	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp);
2710	}
2711	else
2712	# ifdef IEM_WITH_THROW_CATCH
2713	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2714	# else
2715	AssertReleaseFailedStmt(off = UINT32_MAX);
2716	# endif
2717
2718	#else
2719	# error "Port me"
2720	#endif
2721	return off;
2722	}
2723
2724
2725	/**
2726	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2727	*
2728	* @note Larger constants will require a temporary register. Failing to specify
2729	* one when needed will trigger fatal assertion / throw.
2730	*/
2731	DECL_INLINE_THROW(uint32_t)
2732	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2733	uint8_t iGprTmp = UINT8_MAX)
2734
2735	{
2736	#ifdef RT_ARCH_AMD64
2737	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 13), off, iGprDst, iSubtrahend, iGprTmp);
2738	#elif defined(RT_ARCH_ARM64)
2739	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGprDst, iSubtrahend, iGprTmp);
2740	#else
2741	# error "Port me"
2742	#endif
2743	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2744	return off;
2745	}
2746
2747
2748	/**
2749	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2750	*
2751	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2752	*
2753	* @note ARM64: Larger constants will require a temporary register. Failing to
2754	* specify one when needed will trigger fatal assertion / throw.
2755	*/
2756	DECL_FORCE_INLINE_THROW(uint32_t)
2757	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2758	uint8_t iGprTmp = UINT8_MAX)
2759	{
2760	#ifdef RT_ARCH_AMD64
2761	if (iGprDst >= 8)
2762	pCodeBuf[off++] = X86_OP_REX_B;
2763	if (iSubtrahend == 1)
2764	{
2765	/* dec r/m32 */
2766	pCodeBuf[off++] = 0xff;
2767	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2768	}
2769	else if (iSubtrahend == -1)
2770	{
2771	/* inc r/m32 */
2772	pCodeBuf[off++] = 0xff;
2773	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2774	}
2775	else if (iSubtrahend < 128 && iSubtrahend >= -128)
2776	{
2777	/* sub r/m32, imm8 */
2778	pCodeBuf[off++] = 0x83;
2779	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2780	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2781	}
2782	else
2783	{
2784	/* sub r/m32, imm32 */
2785	pCodeBuf[off++] = 0x81;
2786	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2787	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
2788	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
2789	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
2790	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
2791	}
2792	RT_NOREF(iGprTmp);
2793
2794	#elif defined(RT_ARCH_ARM64)
2795	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2796	if (uAbsSubtrahend < 4096)
2797	{
2798	if (iSubtrahend >= 0)
2799	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2800	else
2801	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2802	}
2803	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2804	{
2805	if (iSubtrahend >= 0)
2806	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2807	false /f64Bit/, false /fSetFlags/, true /fShift/);
2808	else
2809	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2810	false /f64Bit/, false /fSetFlags/, true /fShift/);
2811	}
2812	else if (iGprTmp != UINT8_MAX)
2813	{
2814	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2815	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2816	}
2817	else
2818	# ifdef IEM_WITH_THROW_CATCH
2819	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2820	# else
2821	AssertReleaseFailedStmt(off = UINT32_MAX);
2822	# endif
2823
2824	#else
2825	# error "Port me"
2826	#endif
2827	return off;
2828	}
2829
2830
2831	/**
2832	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2833	*
2834	* @note ARM64: Larger constants will require a temporary register. Failing to
2835	* specify one when needed will trigger fatal assertion / throw.
2836	*/
2837	DECL_INLINE_THROW(uint32_t)
2838	iemNativeEmitSubGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2839	uint8_t iGprTmp = UINT8_MAX)
2840
2841	{
2842	#ifdef RT_ARCH_AMD64
2843	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iSubtrahend, iGprTmp);
2844	#elif defined(RT_ARCH_ARM64)
2845	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iSubtrahend, iGprTmp);
2846	#else
2847	# error "Port me"
2848	#endif
2849	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2850	return off;
2851	}
2852
2853
2854	/**
2855	* Emits a 16-bit GPR subtract with a signed immediate subtrahend.
2856	*
2857	* This will optimize using DEC/INC/whatever and ARM64 will not set flags,
2858	* so not suitable as a base for conditional jumps.
2859	*
2860	* @note AMD64: Will only update the lower 16 bits of the register.
2861	* @note ARM64: Will update the entire register.
2862	* @note ARM64: Larger constants will require a temporary register. Failing to
2863	* specify one when needed will trigger fatal assertion / throw.
2864	*/
2865	DECL_FORCE_INLINE_THROW(uint32_t)
2866	iemNativeEmitSubGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iSubtrahend,
2867	uint8_t iGprTmp = UINT8_MAX)
2868	{
2869	#ifdef RT_ARCH_AMD64
2870	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2871	if (iGprDst >= 8)
2872	pCodeBuf[off++] = X86_OP_REX_B;
2873	if (iSubtrahend == 1)
2874	{
2875	/* dec r/m16 */
2876	pCodeBuf[off++] = 0xff;
2877	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2878	}
2879	else if (iSubtrahend == -1)
2880	{
2881	/* inc r/m16 */
2882	pCodeBuf[off++] = 0xff;
2883	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2884	}
2885	else if ((int8_t)iSubtrahend == iSubtrahend)
2886	{
2887	/* sub r/m16, imm8 */
2888	pCodeBuf[off++] = 0x83;
2889	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2890	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2891	}
2892	else
2893	{
2894	/* sub r/m16, imm16 */
2895	pCodeBuf[off++] = 0x81;
2896	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2897	pCodeBuf[off++] = RT_BYTE1((uint16_t)iSubtrahend);
2898	pCodeBuf[off++] = RT_BYTE2((uint16_t)iSubtrahend);
2899	}
2900	RT_NOREF(iGprTmp);
2901
2902	#elif defined(RT_ARCH_ARM64)
2903	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2904	if (uAbsSubtrahend < 4096)
2905	{
2906	if (iSubtrahend >= 0)
2907	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2908	else
2909	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2910	}
2911	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2912	{
2913	if (iSubtrahend >= 0)
2914	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2915	false /f64Bit/, false /fSetFlags/, true /fShift/);
2916	else
2917	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2918	false /f64Bit/, false /fSetFlags/, true /fShift/);
2919	}
2920	else if (iGprTmp != UINT8_MAX)
2921	{
2922	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2923	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2924	}
2925	else
2926	# ifdef IEM_WITH_THROW_CATCH
2927	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2928	# else
2929	AssertReleaseFailedStmt(off = UINT32_MAX);
2930	# endif
2931	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
2932
2933	#else
2934	# error "Port me"
2935	#endif
2936	return off;
2937	}
2938
2939
2940	/**
2941	* Emits adding a 64-bit GPR to another, storing the result in the first.
2942	* @note The AMD64 version sets flags.
2943	*/
2944	DECL_FORCE_INLINE(uint32_t)
2945	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2946	{
2947	#if defined(RT_ARCH_AMD64)
2948	/* add Gv,Ev */
2949	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2950	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
2951	pCodeBuf[off++] = 0x03;
2952	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
2953
2954	#elif defined(RT_ARCH_ARM64)
2955	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
2956
2957	#else
2958	# error "Port me"
2959	#endif
2960	return off;
2961	}
2962
2963
2964	/**
2965	* Emits adding a 64-bit GPR to another, storing the result in the first.
2966	* @note The AMD64 version sets flags.
2967	*/
2968	DECL_INLINE_THROW(uint32_t)
2969	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2970	{
2971	#if defined(RT_ARCH_AMD64)
2972	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
2973	#elif defined(RT_ARCH_ARM64)
2974	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
2975	#else
2976	# error "Port me"
2977	#endif
2978	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2979	return off;
2980	}
2981
2982
2983	/**
2984	* Emits adding a 64-bit GPR to another, storing the result in the first.
2985	* @note The AMD64 version sets flags.
2986	*/
2987	DECL_FORCE_INLINE(uint32_t)
2988	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
2989	{
2990	#if defined(RT_ARCH_AMD64)
2991	/* add Gv,Ev */
2992	if (iGprDst >= 8 \|\| iGprAddend >= 8)
2993	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
2994	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
2995	pCodeBuf[off++] = 0x03;
2996	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
2997
2998	#elif defined(RT_ARCH_ARM64)
2999	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
3000
3001	#else
3002	# error "Port me"
3003	#endif
3004	return off;
3005	}
3006
3007
3008	/**
3009	* Emits adding a 64-bit GPR to another, storing the result in the first.
3010	* @note The AMD64 version sets flags.
3011	*/
3012	DECL_INLINE_THROW(uint32_t)
3013	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3014	{
3015	#if defined(RT_ARCH_AMD64)
3016	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
3017	#elif defined(RT_ARCH_ARM64)
3018	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
3019	#else
3020	# error "Port me"
3021	#endif
3022	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3023	return off;
3024	}
3025
3026
3027	/**
3028	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3029	*/
3030	DECL_INLINE_THROW(uint32_t)
3031	iemNativeEmitAddGprImm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3032	{
3033	#if defined(RT_ARCH_AMD64)
3034	/* add or inc */
3035	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3036	if (iImm8 != 1)
3037	{
3038	pCodeBuf[off++] = 0x83;
3039	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3040	pCodeBuf[off++] = (uint8_t)iImm8;
3041	}
3042	else
3043	{
3044	pCodeBuf[off++] = 0xff;
3045	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3046	}
3047
3048	#elif defined(RT_ARCH_ARM64)
3049	if (iImm8 >= 0)
3050	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint8_t)iImm8);
3051	else
3052	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint8_t)-iImm8);
3053
3054	#else
3055	# error "Port me"
3056	#endif
3057	return off;
3058	}
3059
3060
3061	/**
3062	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3063	*/
3064	DECL_INLINE_THROW(uint32_t)
3065	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3066	{
3067	#if defined(RT_ARCH_AMD64)
3068	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3069	#elif defined(RT_ARCH_ARM64)
3070	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3071	#else
3072	# error "Port me"
3073	#endif
3074	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3075	return off;
3076	}
3077
3078
3079	/**
3080	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3081	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3082	*/
3083	DECL_FORCE_INLINE(uint32_t)
3084	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3085	{
3086	#if defined(RT_ARCH_AMD64)
3087	/* add or inc */
3088	if (iGprDst >= 8)
3089	pCodeBuf[off++] = X86_OP_REX_B;
3090	if (iImm8 != 1)
3091	{
3092	pCodeBuf[off++] = 0x83;
3093	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3094	pCodeBuf[off++] = (uint8_t)iImm8;
3095	}
3096	else
3097	{
3098	pCodeBuf[off++] = 0xff;
3099	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3100	}
3101
3102	#elif defined(RT_ARCH_ARM64)
3103	if (iImm8 >= 0)
3104	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
3105	else
3106	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
3107
3108	#else
3109	# error "Port me"
3110	#endif
3111	return off;
3112	}
3113
3114
3115	/**
3116	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3117	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3118	*/
3119	DECL_INLINE_THROW(uint32_t)
3120	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3121	{
3122	#if defined(RT_ARCH_AMD64)
3123	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3124	#elif defined(RT_ARCH_ARM64)
3125	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3126	#else
3127	# error "Port me"
3128	#endif
3129	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3130	return off;
3131	}
3132
3133
3134	/**
3135	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3136	*
3137	* @note Will assert / throw if @a iGprTmp is not specified when needed.
3138	*/
3139	DECL_FORCE_INLINE_THROW(uint32_t)
3140	iemNativeEmitAddGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iAddend, uint8_t iGprTmp = UINT8_MAX)
3141	{
3142	#if defined(RT_ARCH_AMD64)
3143	if ((int8_t)iAddend == iAddend)
3144	return iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3145
3146	if ((int32_t)iAddend == iAddend)
3147	{
3148	/* add grp, imm32 */
3149	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3150	pCodeBuf[off++] = 0x81;
3151	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3152	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3153	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3154	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3155	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3156	}
3157	else if (iGprTmp != UINT8_MAX)
3158	{
3159	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3160
3161	/* add dst, tmpreg */
3162	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3163	\| (iGprTmp < 8 ? 0 : X86_OP_REX_B);
3164	pCodeBuf[off++] = 0x03;
3165	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprTmp & 7);
3166	}
3167	else
3168	# ifdef IEM_WITH_THROW_CATCH
3169	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3170	# else
3171	AssertReleaseFailedStmt(off = UINT32_MAX);
3172	# endif
3173
3174	#elif defined(RT_ARCH_ARM64)
3175	uint64_t const uAbsAddend = (uint64_t)RT_ABS(iAddend);
3176	if (uAbsAddend < 4096)
3177	{
3178	if (iAddend >= 0)
3179	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3180	else
3181	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3182	}
3183	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3184	{
3185	if (iAddend >= 0)
3186	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3187	true /f64Bit/, true /fShift12/);
3188	else
3189	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3190	true /f64Bit/, true /fShift12/);
3191	}
3192	else if (iGprTmp != UINT8_MAX)
3193	{
3194	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3195	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp);
3196	}
3197	else
3198	# ifdef IEM_WITH_THROW_CATCH
3199	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3200	# else
3201	AssertReleaseFailedStmt(off = UINT32_MAX);
3202	# endif
3203
3204	#else
3205	# error "Port me"
3206	#endif
3207	return off;
3208	}
3209
3210
3211	/**
3212	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3213	*/
3214	DECL_INLINE_THROW(uint32_t)
3215	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
3216	{
3217	#if defined(RT_ARCH_AMD64)
3218	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3219	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
3220
3221	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
3222	{
3223	/* add grp, imm32 */
3224	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3225	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3226	pbCodeBuf[off++] = 0x81;
3227	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3228	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3229	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3230	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3231	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3232	}
3233	else
3234	{
3235	/* Best to use a temporary register to deal with this in the simplest way: */
3236	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3237
3238	/* add dst, tmpreg */
3239	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3240	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3241	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3242	pbCodeBuf[off++] = 0x03;
3243	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
3244
3245	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3246	}
3247
3248	#elif defined(RT_ARCH_ARM64)
3249	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3250	{
3251	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3252	if (iAddend >= 0)
3253	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
3254	else
3255	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
3256	}
3257	else
3258	{
3259	/* Use temporary register for the immediate. */
3260	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3261
3262	/* add gprdst, gprdst, tmpreg */
3263	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3264	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
3265
3266	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3267	}
3268
3269	#else
3270	# error "Port me"
3271	#endif
3272	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3273	return off;
3274	}
3275
3276
3277	/**
3278	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3279	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3280	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
3281	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
3282	* the lower 12 bits always zero). The negative ranges are also allowed,
3283	* making it behave like a subtraction. If the constant does not conform,
3284	* bad stuff will happen.
3285	*/
3286	DECL_FORCE_INLINE_THROW(uint32_t)
3287	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3288	{
3289	#if defined(RT_ARCH_AMD64)
3290	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3291	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3292
3293	/* add grp, imm32 */
3294	if (iGprDst >= 8)
3295	pCodeBuf[off++] = X86_OP_REX_B;
3296	pCodeBuf[off++] = 0x81;
3297	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3298	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3299	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3300	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3301	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3302
3303	#elif defined(RT_ARCH_ARM64)
3304	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
3305	if (uAbsAddend <= 0xfff)
3306	{
3307	if (iAddend >= 0)
3308	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3309	else
3310	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3311	}
3312	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3313	{
3314	if (iAddend >= 0)
3315	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3316	false /f64Bit/, true /fShift12/);
3317	else
3318	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3319	false /f64Bit/, true /fShift12/);
3320	}
3321	else
3322	# ifdef IEM_WITH_THROW_CATCH
3323	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3324	# else
3325	AssertReleaseFailedStmt(off = UINT32_MAX);
3326	# endif
3327
3328	#else
3329	# error "Port me"
3330	#endif
3331	return off;
3332	}
3333
3334
3335	/**
3336	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3337	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3338	*/
3339	DECL_INLINE_THROW(uint32_t)
3340	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3341	{
3342	#if defined(RT_ARCH_AMD64)
3343	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
3344
3345	#elif defined(RT_ARCH_ARM64)
3346	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3347	{
3348	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3349	if (iAddend >= 0)
3350	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
3351	else
3352	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
3353	}
3354	else
3355	{
3356	/* Use temporary register for the immediate. */
3357	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
3358
3359	/* add gprdst, gprdst, tmpreg */
3360	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3361	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
3362
3363	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3364	}
3365
3366	#else
3367	# error "Port me"
3368	#endif
3369	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3370	return off;
3371	}
3372
3373
3374	/**
3375	* Emits a 16-bit GPR add with a signed immediate addend.
3376	*
3377	* This will optimize using INC/DEC/whatever and ARM64 will not set flags,
3378	* so not suitable as a base for conditional jumps.
3379	*
3380	* @note AMD64: Will only update the lower 16 bits of the register.
3381	* @note ARM64: Will update the entire register.
3382	* @note ARM64: Larger constants will require a temporary register. Failing to
3383	* specify one when needed will trigger fatal assertion / throw.
3384	* @sa iemNativeEmitSubGpr16ImmEx
3385	*/
3386	DECL_FORCE_INLINE_THROW(uint32_t)
3387	iemNativeEmitAddGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iAddend,
3388	uint8_t iGprTmp = UINT8_MAX)
3389	{
3390	#ifdef RT_ARCH_AMD64
3391	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
3392	if (iGprDst >= 8)
3393	pCodeBuf[off++] = X86_OP_REX_B;
3394	if (iAddend == 1)
3395	{
3396	/* inc r/m16 */
3397	pCodeBuf[off++] = 0xff;
3398	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3399	}
3400	else if (iAddend == -1)
3401	{
3402	/* dec r/m16 */
3403	pCodeBuf[off++] = 0xff;
3404	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
3405	}
3406	else if ((int8_t)iAddend == iAddend)
3407	{
3408	/* add r/m16, imm8 */
3409	pCodeBuf[off++] = 0x83;
3410	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3411	pCodeBuf[off++] = (uint8_t)iAddend;
3412	}
3413	else
3414	{
3415	/* add r/m16, imm16 */
3416	pCodeBuf[off++] = 0x81;
3417	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3418	pCodeBuf[off++] = RT_BYTE1((uint16_t)iAddend);
3419	pCodeBuf[off++] = RT_BYTE2((uint16_t)iAddend);
3420	}
3421	RT_NOREF(iGprTmp);
3422
3423	#elif defined(RT_ARCH_ARM64)
3424	uint32_t uAbsAddend = RT_ABS(iAddend);
3425	if (uAbsAddend < 4096)
3426	{
3427	if (iAddend >= 0)
3428	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3429	else
3430	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3431	}
3432	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3433	{
3434	if (iAddend >= 0)
3435	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3436	false /f64Bit/, false /fSetFlags/, true /fShift/);
3437	else
3438	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3439	false /f64Bit/, false /fSetFlags/, true /fShift/);
3440	}
3441	else if (iGprTmp != UINT8_MAX)
3442	{
3443	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iAddend);
3444	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
3445	}
3446	else
3447	# ifdef IEM_WITH_THROW_CATCH
3448	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3449	# else
3450	AssertReleaseFailedStmt(off = UINT32_MAX);
3451	# endif
3452	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
3453
3454	#else
3455	# error "Port me"
3456	#endif
3457	return off;
3458	}
3459
3460
3461
3462	/**
3463	* Adds two 64-bit GPRs together, storing the result in a third register.
3464	*/
3465	DECL_FORCE_INLINE(uint32_t)
3466	iemNativeEmitGprEqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3467	{
3468	#ifdef RT_ARCH_AMD64
3469	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3470	{
3471	/** @todo consider LEA */
3472	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend1);
3473	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend2);
3474	}
3475	else
3476	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3477
3478	#elif defined(RT_ARCH_ARM64)
3479	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2);
3480
3481	#else
3482	# error "Port me!"
3483	#endif
3484	return off;
3485	}
3486
3487
3488
3489	/**
3490	* Adds two 32-bit GPRs together, storing the result in a third register.
3491	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3492	*/
3493	DECL_FORCE_INLINE(uint32_t)
3494	iemNativeEmitGpr32EqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3495	{
3496	#ifdef RT_ARCH_AMD64
3497	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3498	{
3499	/** @todo consider LEA */
3500	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend1);
3501	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend2);
3502	}
3503	else
3504	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3505
3506	#elif defined(RT_ARCH_ARM64)
3507	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2, false /f64Bit/);
3508
3509	#else
3510	# error "Port me!"
3511	#endif
3512	return off;
3513	}
3514
3515
3516	/**
3517	* Adds a 64-bit GPR and a 64-bit unsigned constant, storing the result in a
3518	* third register.
3519	*
3520	* @note The ARM64 version does not work for non-trivial constants if the
3521	* two registers are the same. Will assert / throw exception.
3522	*/
3523	DECL_FORCE_INLINE_THROW(uint32_t)
3524	iemNativeEmitGprEqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int64_t iImmAddend)
3525	{
3526	#ifdef RT_ARCH_AMD64
3527	/** @todo consider LEA */
3528	if ((int8_t)iImmAddend == iImmAddend)
3529	{
3530	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend);
3531	off = iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3532	}
3533	else
3534	{
3535	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3536	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3537	}
3538
3539	#elif defined(RT_ARCH_ARM64)
3540	uint64_t const uAbsImmAddend = RT_ABS(iImmAddend);
3541	if (uAbsImmAddend < 4096)
3542	{
3543	if (iImmAddend >= 0)
3544	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3545	else
3546	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3547	}
3548	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3549	{
3550	if (iImmAddend >= 0)
3551	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3552	else
3553	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3554	}
3555	else if (iGprDst != iGprAddend)
3556	{
3557	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, (uint64_t)iImmAddend);
3558	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3559	}
3560	else
3561	# ifdef IEM_WITH_THROW_CATCH
3562	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3563	# else
3564	AssertReleaseFailedStmt(off = UINT32_MAX);
3565	# endif
3566
3567	#else
3568	# error "Port me!"
3569	#endif
3570	return off;
3571	}
3572
3573
3574	/**
3575	* Adds a 32-bit GPR and a 32-bit unsigned constant, storing the result in a
3576	* third register.
3577	*
3578	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3579	*
3580	* @note The ARM64 version does not work for non-trivial constants if the
3581	* two registers are the same. Will assert / throw exception.
3582	*/
3583	DECL_FORCE_INLINE_THROW(uint32_t)
3584	iemNativeEmitGpr32EqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
3585	{
3586	#ifdef RT_ARCH_AMD64
3587	/** @todo consider LEA */
3588	if ((int8_t)iImmAddend == iImmAddend)
3589	{
3590	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3591	off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3592	}
3593	else
3594	{
3595	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3596	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3597	}
3598
3599	#elif defined(RT_ARCH_ARM64)
3600	uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
3601	if (uAbsImmAddend < 4096)
3602	{
3603	if (iImmAddend >= 0)
3604	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3605	else
3606	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3607	}
3608	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3609	{
3610	if (iImmAddend >= 0)
3611	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3612	else
3613	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3614	}
3615	else if (iGprDst != iGprAddend)
3616	{
3617	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
3618	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3619	}
3620	else
3621	# ifdef IEM_WITH_THROW_CATCH
3622	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3623	# else
3624	AssertReleaseFailedStmt(off = UINT32_MAX);
3625	# endif
3626
3627	#else
3628	# error "Port me!"
3629	#endif
3630	return off;
3631	}
3632
3633
3634	/*********************************************************************************************************************************
3635	* Unary Operations *
3636	*********************************************************************************************************************************/
3637
3638	/**
3639	* Emits code for two complement negation of a 64-bit GPR.
3640	*/
3641	DECL_FORCE_INLINE_THROW(uint32_t)
3642	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3643	{
3644	#if defined(RT_ARCH_AMD64)
3645	/* neg Ev */
3646	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3647	pCodeBuf[off++] = 0xf7;
3648	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3649
3650	#elif defined(RT_ARCH_ARM64)
3651	/* sub dst, xzr, dst */
3652	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
3653
3654	#else
3655	# error "Port me"
3656	#endif
3657	return off;
3658	}
3659
3660
3661	/**
3662	* Emits code for two complement negation of a 64-bit GPR.
3663	*/
3664	DECL_INLINE_THROW(uint32_t)
3665	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3666	{
3667	#if defined(RT_ARCH_AMD64)
3668	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3669	#elif defined(RT_ARCH_ARM64)
3670	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3671	#else
3672	# error "Port me"
3673	#endif
3674	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3675	return off;
3676	}
3677
3678
3679	/**
3680	* Emits code for two complement negation of a 32-bit GPR.
3681	* @note bit 32 thru 63 are set to zero.
3682	*/
3683	DECL_FORCE_INLINE_THROW(uint32_t)
3684	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3685	{
3686	#if defined(RT_ARCH_AMD64)
3687	/* neg Ev */
3688	if (iGprDst >= 8)
3689	pCodeBuf[off++] = X86_OP_REX_B;
3690	pCodeBuf[off++] = 0xf7;
3691	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3692
3693	#elif defined(RT_ARCH_ARM64)
3694	/* sub dst, xzr, dst */
3695	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
3696
3697	#else
3698	# error "Port me"
3699	#endif
3700	return off;
3701	}
3702
3703
3704	/**
3705	* Emits code for two complement negation of a 32-bit GPR.
3706	* @note bit 32 thru 63 are set to zero.
3707	*/
3708	DECL_INLINE_THROW(uint32_t)
3709	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3710	{
3711	#if defined(RT_ARCH_AMD64)
3712	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3713	#elif defined(RT_ARCH_ARM64)
3714	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3715	#else
3716	# error "Port me"
3717	#endif
3718	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3719	return off;
3720	}
3721
3722
3723
3724	/*********************************************************************************************************************************
3725	* Bit Operations *
3726	*********************************************************************************************************************************/
3727
3728	/**
3729	* Emits code for clearing bits 16 thru 63 in the GPR.
3730	*/
3731	DECL_INLINE_THROW(uint32_t)
3732	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3733	{
3734	#if defined(RT_ARCH_AMD64)
3735	/* movzx Gv,Ew */
3736	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3737	if (iGprDst >= 8)
3738	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
3739	pbCodeBuf[off++] = 0x0f;
3740	pbCodeBuf[off++] = 0xb7;
3741	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
3742
3743	#elif defined(RT_ARCH_ARM64)
3744	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3745	# if 1
3746	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
3747	# else
3748	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
3749	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
3750	# endif
3751	#else
3752	# error "Port me"
3753	#endif
3754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3755	return off;
3756	}
3757
3758
3759	/**
3760	* Emits code for AND'ing two 64-bit GPRs.
3761	*
3762	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3763	* and ARM64 hosts.
3764	*/
3765	DECL_INLINE_THROW(uint32_t)
3766	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3767	{
3768	#if defined(RT_ARCH_AMD64)
3769	/* and Gv, Ev */
3770	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3771	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3772	pbCodeBuf[off++] = 0x23;
3773	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3774	RT_NOREF(fSetFlags);
3775
3776	#elif defined(RT_ARCH_ARM64)
3777	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3778	if (!fSetFlags)
3779	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
3780	else
3781	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
3782
3783	#else
3784	# error "Port me"
3785	#endif
3786	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3787	return off;
3788	}
3789
3790
3791	/**
3792	* Emits code for AND'ing two 32-bit GPRs.
3793	*/
3794	DECL_FORCE_INLINE(uint32_t)
3795	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3796	{
3797	#if defined(RT_ARCH_AMD64)
3798	/* and Gv, Ev */
3799	if (iGprDst >= 8 \|\| iGprSrc >= 8)
3800	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3801	pCodeBuf[off++] = 0x23;
3802	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3803	RT_NOREF(fSetFlags);
3804
3805	#elif defined(RT_ARCH_ARM64)
3806	if (!fSetFlags)
3807	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3808	else
3809	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3810
3811	#else
3812	# error "Port me"
3813	#endif
3814	return off;
3815	}
3816
3817
3818	/**
3819	* Emits code for AND'ing two 32-bit GPRs.
3820	*/
3821	DECL_INLINE_THROW(uint32_t)
3822	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3823	{
3824	#if defined(RT_ARCH_AMD64)
3825	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
3826	#elif defined(RT_ARCH_ARM64)
3827	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
3828	#else
3829	# error "Port me"
3830	#endif
3831	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3832	return off;
3833	}
3834
3835
3836	/**
3837	* Emits code for AND'ing a 64-bit GPRs with a constant.
3838	*
3839	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3840	* and ARM64 hosts.
3841	*/
3842	DECL_INLINE_THROW(uint32_t)
3843	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
3844	{
3845	#if defined(RT_ARCH_AMD64)
3846	if ((int64_t)uImm == (int8_t)uImm)
3847	{
3848	/* and Ev, imm8 */
3849	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3850	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3851	pbCodeBuf[off++] = 0x83;
3852	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3853	pbCodeBuf[off++] = (uint8_t)uImm;
3854	}
3855	else if ((int64_t)uImm == (int32_t)uImm)
3856	{
3857	/* and Ev, imm32 */
3858	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3859	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
3860	pbCodeBuf[off++] = 0x81;
3861	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3862	pbCodeBuf[off++] = RT_BYTE1(uImm);
3863	pbCodeBuf[off++] = RT_BYTE2(uImm);
3864	pbCodeBuf[off++] = RT_BYTE3(uImm);
3865	pbCodeBuf[off++] = RT_BYTE4(uImm);
3866	}
3867	else
3868	{
3869	/* Use temporary register for the 64-bit immediate. */
3870	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3871	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
3872	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3873	}
3874	RT_NOREF(fSetFlags);
3875
3876	#elif defined(RT_ARCH_ARM64)
3877	uint32_t uImmR = 0;
3878	uint32_t uImmNandS = 0;
3879	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
3880	{
3881	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3882	if (!fSetFlags)
3883	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
3884	else
3885	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
3886	}
3887	else
3888	{
3889	/* Use temporary register for the 64-bit immediate. */
3890	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3891	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
3892	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3893	}
3894
3895	#else
3896	# error "Port me"
3897	#endif
3898	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3899	return off;
3900	}
3901
3902
3903	/**
3904	* Emits code for AND'ing an 32-bit GPRs with a constant.
3905	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3906	* @note For ARM64 this only supports @a uImm values that can be expressed using
3907	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
3908	* make sure this is possible!
3909	*/
3910	DECL_FORCE_INLINE_THROW(uint32_t)
3911	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
3912	{
3913	#if defined(RT_ARCH_AMD64)
3914	/* and Ev, imm */
3915	if (iGprDst >= 8)
3916	pCodeBuf[off++] = X86_OP_REX_B;
3917	if ((int32_t)uImm == (int8_t)uImm)
3918	{
3919	pCodeBuf[off++] = 0x83;
3920	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3921	pCodeBuf[off++] = (uint8_t)uImm;
3922	}
3923	else
3924	{
3925	pCodeBuf[off++] = 0x81;
3926	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
3927	pCodeBuf[off++] = RT_BYTE1(uImm);
3928	pCodeBuf[off++] = RT_BYTE2(uImm);
3929	pCodeBuf[off++] = RT_BYTE3(uImm);
3930	pCodeBuf[off++] = RT_BYTE4(uImm);
3931	}
3932	RT_NOREF(fSetFlags);
3933
3934	#elif defined(RT_ARCH_ARM64)
3935	uint32_t uImmR = 0;
3936	uint32_t uImmNandS = 0;
3937	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
3938	{
3939	if (!fSetFlags)
3940	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3941	else
3942	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3943	}
3944	else
3945	# ifdef IEM_WITH_THROW_CATCH
3946	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3947	# else
3948	AssertReleaseFailedStmt(off = UINT32_MAX);
3949	# endif
3950
3951	#else
3952	# error "Port me"
3953	#endif
3954	return off;
3955	}
3956
3957
3958	/**
3959	* Emits code for AND'ing an 32-bit GPRs with a constant.
3960	*
3961	* @note Bits 32 thru 63 in the destination will be zero after the operation.
3962	*/
3963	DECL_INLINE_THROW(uint32_t)
3964	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
3965	{
3966	#if defined(RT_ARCH_AMD64)
3967	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
3968
3969	#elif defined(RT_ARCH_ARM64)
3970	uint32_t uImmR = 0;
3971	uint32_t uImmNandS = 0;
3972	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
3973	{
3974	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3975	if (!fSetFlags)
3976	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3977	else
3978	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
3979	}
3980	else
3981	{
3982	/* Use temporary register for the 64-bit immediate. */
3983	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3984	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
3985	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3986	}
3987
3988	#else
3989	# error "Port me"
3990	#endif
3991	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3992	return off;
3993	}
3994
3995
3996	/**
3997	* Emits code for AND'ing an 32-bit GPRs with a constant.
3998	*
3999	* @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
4000	* encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
4001	* the same.
4002	*
4003	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4004	*/
4005	DECL_FORCE_INLINE_THROW(uint32_t)
4006	iemNativeEmitGpr32EqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint32_t uImm,
4007	bool fSetFlags = false)
4008	{
4009	#if defined(RT_ARCH_AMD64)
4010	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4011	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4012	RT_NOREF(fSetFlags);
4013
4014	#elif defined(RT_ARCH_ARM64)
4015	uint32_t uImmR = 0;
4016	uint32_t uImmNandS = 0;
4017	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4018	{
4019	if (!fSetFlags)
4020	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4021	else
4022	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4023	}
4024	else if (iGprDst != iGprSrc)
4025	{
4026	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4027	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
4028	}
4029	else
4030	# ifdef IEM_WITH_THROW_CATCH
4031	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4032	# else
4033	AssertReleaseFailedStmt(off = UINT32_MAX);
4034	# endif
4035
4036	#else
4037	# error "Port me"
4038	#endif
4039	return off;
4040	}
4041
4042
4043	/**
4044	* Emits code for OR'ing two 64-bit GPRs.
4045	*/
4046	DECL_FORCE_INLINE(uint32_t)
4047	iemNativeEmitOrGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4048	{
4049	#if defined(RT_ARCH_AMD64)
4050	/* or Gv, Ev */
4051	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4052	pCodeBuf[off++] = 0x0b;
4053	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4054
4055	#elif defined(RT_ARCH_ARM64)
4056	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc);
4057
4058	#else
4059	# error "Port me"
4060	#endif
4061	return off;
4062	}
4063
4064
4065	/**
4066	* Emits code for OR'ing two 32-bit GPRs.
4067	* @note Bits 63:32 of the destination GPR will be cleared.
4068	*/
4069	DECL_FORCE_INLINE(uint32_t)
4070	iemNativeEmitOrGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4071	{
4072	#if defined(RT_ARCH_AMD64)
4073	/* or Gv, Ev */
4074	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4075	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4076	pCodeBuf[off++] = 0x0b;
4077	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4078
4079	#elif defined(RT_ARCH_ARM64)
4080	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4081
4082	#else
4083	# error "Port me"
4084	#endif
4085	return off;
4086	}
4087
4088
4089	/**
4090	* Emits code for OR'ing two 32-bit GPRs.
4091	* @note Bits 63:32 of the destination GPR will be cleared.
4092	*/
4093	DECL_INLINE_THROW(uint32_t)
4094	iemNativeEmitOrGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4095	{
4096	#if defined(RT_ARCH_AMD64)
4097	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
4098	#elif defined(RT_ARCH_ARM64)
4099	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
4100	#else
4101	# error "Port me"
4102	#endif
4103	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4104	return off;
4105	}
4106
4107
4108
4109	/**
4110	* Emits code for XOR'ing two 64-bit GPRs.
4111	*/
4112	DECL_INLINE_THROW(uint32_t)
4113	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4114	{
4115	#if defined(RT_ARCH_AMD64)
4116	/* and Gv, Ev */
4117	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4118	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4119	pbCodeBuf[off++] = 0x33;
4120	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4121
4122	#elif defined(RT_ARCH_ARM64)
4123	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4124	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
4125
4126	#else
4127	# error "Port me"
4128	#endif
4129	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4130	return off;
4131	}
4132
4133
4134	/**
4135	* Emits code for XOR'ing two 32-bit GPRs.
4136	*/
4137	DECL_INLINE_THROW(uint32_t)
4138	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4139	{
4140	#if defined(RT_ARCH_AMD64)
4141	/* and Gv, Ev */
4142	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4143	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4144	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4145	pbCodeBuf[off++] = 0x33;
4146	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4147
4148	#elif defined(RT_ARCH_ARM64)
4149	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4150	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4151
4152	#else
4153	# error "Port me"
4154	#endif
4155	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4156	return off;
4157	}
4158
4159
4160	/**
4161	* Emits code for XOR'ing an 32-bit GPRs with a constant.
4162	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4163	* @note For ARM64 this only supports @a uImm values that can be expressed using
4164	* the two 6-bit immediates of the EOR instructions. The caller must make
4165	* sure this is possible!
4166	*/
4167	DECL_FORCE_INLINE_THROW(uint32_t)
4168	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
4169	{
4170	#if defined(RT_ARCH_AMD64)
4171	/* and Ev, imm */
4172	if (iGprDst >= 8)
4173	pCodeBuf[off++] = X86_OP_REX_B;
4174	if ((int32_t)uImm == (int8_t)uImm)
4175	{
4176	pCodeBuf[off++] = 0x83;
4177	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4178	pCodeBuf[off++] = (uint8_t)uImm;
4179	}
4180	else
4181	{
4182	pCodeBuf[off++] = 0x81;
4183	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4184	pCodeBuf[off++] = RT_BYTE1(uImm);
4185	pCodeBuf[off++] = RT_BYTE2(uImm);
4186	pCodeBuf[off++] = RT_BYTE3(uImm);
4187	pCodeBuf[off++] = RT_BYTE4(uImm);
4188	}
4189
4190	#elif defined(RT_ARCH_ARM64)
4191	uint32_t uImmR = 0;
4192	uint32_t uImmNandS = 0;
4193	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4194	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4195	else
4196	# ifdef IEM_WITH_THROW_CATCH
4197	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4198	# else
4199	AssertReleaseFailedStmt(off = UINT32_MAX);
4200	# endif
4201
4202	#else
4203	# error "Port me"
4204	#endif
4205	return off;
4206	}
4207
4208
4209	/*********************************************************************************************************************************
4210	* Shifting *
4211	*********************************************************************************************************************************/
4212
4213	/**
4214	* Emits code for shifting a GPR a fixed number of bits to the left.
4215	*/
4216	DECL_FORCE_INLINE(uint32_t)
4217	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4218	{
4219	Assert(cShift > 0 && cShift < 64);
4220
4221	#if defined(RT_ARCH_AMD64)
4222	/* shl dst, cShift */
4223	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4224	if (cShift != 1)
4225	{
4226	pCodeBuf[off++] = 0xc1;
4227	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4228	pCodeBuf[off++] = cShift;
4229	}
4230	else
4231	{
4232	pCodeBuf[off++] = 0xd1;
4233	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4234	}
4235
4236	#elif defined(RT_ARCH_ARM64)
4237	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
4238
4239	#else
4240	# error "Port me"
4241	#endif
4242	return off;
4243	}
4244
4245
4246	/**
4247	* Emits code for shifting a GPR a fixed number of bits to the left.
4248	*/
4249	DECL_INLINE_THROW(uint32_t)
4250	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4251	{
4252	#if defined(RT_ARCH_AMD64)
4253	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4254	#elif defined(RT_ARCH_ARM64)
4255	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4256	#else
4257	# error "Port me"
4258	#endif
4259	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4260	return off;
4261	}
4262
4263
4264	/**
4265	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4266	*/
4267	DECL_FORCE_INLINE(uint32_t)
4268	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4269	{
4270	Assert(cShift > 0 && cShift < 32);
4271
4272	#if defined(RT_ARCH_AMD64)
4273	/* shl dst, cShift */
4274	if (iGprDst >= 8)
4275	pCodeBuf[off++] = X86_OP_REX_B;
4276	if (cShift != 1)
4277	{
4278	pCodeBuf[off++] = 0xc1;
4279	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4280	pCodeBuf[off++] = cShift;
4281	}
4282	else
4283	{
4284	pCodeBuf[off++] = 0xd1;
4285	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4286	}
4287
4288	#elif defined(RT_ARCH_ARM64)
4289	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
4290
4291	#else
4292	# error "Port me"
4293	#endif
4294	return off;
4295	}
4296
4297
4298	/**
4299	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4300	*/
4301	DECL_INLINE_THROW(uint32_t)
4302	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4303	{
4304	#if defined(RT_ARCH_AMD64)
4305	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4306	#elif defined(RT_ARCH_ARM64)
4307	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4308	#else
4309	# error "Port me"
4310	#endif
4311	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4312	return off;
4313	}
4314
4315
4316	/**
4317	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4318	*/
4319	DECL_FORCE_INLINE(uint32_t)
4320	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4321	{
4322	Assert(cShift > 0 && cShift < 64);
4323
4324	#if defined(RT_ARCH_AMD64)
4325	/* shr dst, cShift */
4326	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4327	if (cShift != 1)
4328	{
4329	pCodeBuf[off++] = 0xc1;
4330	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4331	pCodeBuf[off++] = cShift;
4332	}
4333	else
4334	{
4335	pCodeBuf[off++] = 0xd1;
4336	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4337	}
4338
4339	#elif defined(RT_ARCH_ARM64)
4340	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
4341
4342	#else
4343	# error "Port me"
4344	#endif
4345	return off;
4346	}
4347
4348
4349	/**
4350	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4351	*/
4352	DECL_INLINE_THROW(uint32_t)
4353	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4354	{
4355	#if defined(RT_ARCH_AMD64)
4356	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4357	#elif defined(RT_ARCH_ARM64)
4358	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4359	#else
4360	# error "Port me"
4361	#endif
4362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4363	return off;
4364	}
4365
4366
4367	/**
4368	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4369	* right.
4370	*/
4371	DECL_FORCE_INLINE(uint32_t)
4372	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4373	{
4374	Assert(cShift > 0 && cShift < 32);
4375
4376	#if defined(RT_ARCH_AMD64)
4377	/* shr dst, cShift */
4378	if (iGprDst >= 8)
4379	pCodeBuf[off++] = X86_OP_REX_B;
4380	if (cShift != 1)
4381	{
4382	pCodeBuf[off++] = 0xc1;
4383	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4384	pCodeBuf[off++] = cShift;
4385	}
4386	else
4387	{
4388	pCodeBuf[off++] = 0xd1;
4389	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4390	}
4391
4392	#elif defined(RT_ARCH_ARM64)
4393	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
4394
4395	#else
4396	# error "Port me"
4397	#endif
4398	return off;
4399	}
4400
4401
4402	/**
4403	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4404	* right.
4405	*/
4406	DECL_INLINE_THROW(uint32_t)
4407	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4408	{
4409	#if defined(RT_ARCH_AMD64)
4410	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4411	#elif defined(RT_ARCH_ARM64)
4412	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4413	#else
4414	# error "Port me"
4415	#endif
4416	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4417	return off;
4418	}
4419
4420
4421	/**
4422	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4423	* right and assigning it to a different GPR.
4424	*/
4425	DECL_INLINE_THROW(uint32_t)
4426	iemNativeEmitGpr32EqGprShiftRightImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint8_t cShift)
4427	{
4428	Assert(cShift > 0); Assert(cShift < 32);
4429	#if defined(RT_ARCH_AMD64)
4430	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4431	off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, iGprDst, cShift);
4432
4433	#elif defined(RT_ARCH_ARM64)
4434	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprSrc, cShift, false /64Bit/);
4435
4436	#else
4437	# error "Port me"
4438	#endif
4439	return off;
4440	}
4441
4442
4443	/**
4444	* Emits code for rotating a GPR a fixed number of bits to the left.
4445	*/
4446	DECL_FORCE_INLINE(uint32_t)
4447	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4448	{
4449	Assert(cShift > 0 && cShift < 64);
4450
4451	#if defined(RT_ARCH_AMD64)
4452	/* rol dst, cShift */
4453	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4454	if (cShift != 1)
4455	{
4456	pCodeBuf[off++] = 0xc1;
4457	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4458	pCodeBuf[off++] = cShift;
4459	}
4460	else
4461	{
4462	pCodeBuf[off++] = 0xd1;
4463	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4464	}
4465
4466	#elif defined(RT_ARCH_ARM64)
4467	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
4468
4469	#else
4470	# error "Port me"
4471	#endif
4472	return off;
4473	}
4474
4475
4476
4477	/*********************************************************************************************************************************
4478	* Compare and Testing *
4479	*********************************************************************************************************************************/
4480
4481
4482	#ifdef RT_ARCH_ARM64
4483	/**
4484	* Emits an ARM64 compare instruction.
4485	*/
4486	DECL_INLINE_THROW(uint32_t)
4487	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
4488	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
4489	{
4490	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4491	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
4492	f64Bit, true /fSetFlags/, cShift, enmShift);
4493	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4494	return off;
4495	}
4496	#endif
4497
4498
4499	/**
4500	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4501	* with conditional instruction.
4502	*/
4503	DECL_FORCE_INLINE(uint32_t)
4504	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4505	{
4506	#ifdef RT_ARCH_AMD64
4507	/* cmp Gv, Ev */
4508	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4509	pCodeBuf[off++] = 0x3b;
4510	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4511
4512	#elif defined(RT_ARCH_ARM64)
4513	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
4514
4515	#else
4516	# error "Port me!"
4517	#endif
4518	return off;
4519	}
4520
4521
4522	/**
4523	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4524	* with conditional instruction.
4525	*/
4526	DECL_INLINE_THROW(uint32_t)
4527	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4528	{
4529	#ifdef RT_ARCH_AMD64
4530	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4531	#elif defined(RT_ARCH_ARM64)
4532	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4533	#else
4534	# error "Port me!"
4535	#endif
4536	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4537	return off;
4538	}
4539
4540
4541	/**
4542	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4543	* with conditional instruction.
4544	*/
4545	DECL_FORCE_INLINE(uint32_t)
4546	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4547	{
4548	#ifdef RT_ARCH_AMD64
4549	/* cmp Gv, Ev */
4550	if (iGprLeft >= 8 \|\| iGprRight >= 8)
4551	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4552	pCodeBuf[off++] = 0x3b;
4553	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4554
4555	#elif defined(RT_ARCH_ARM64)
4556	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
4557
4558	#else
4559	# error "Port me!"
4560	#endif
4561	return off;
4562	}
4563
4564
4565	/**
4566	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4567	* with conditional instruction.
4568	*/
4569	DECL_INLINE_THROW(uint32_t)
4570	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4571	{
4572	#ifdef RT_ARCH_AMD64
4573	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4574	#elif defined(RT_ARCH_ARM64)
4575	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4576	#else
4577	# error "Port me!"
4578	#endif
4579	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4580	return off;
4581	}
4582
4583
4584	/**
4585	* Emits a compare of a 64-bit GPR with a constant value, settings status
4586	* flags/whatever for use with conditional instruction.
4587	*/
4588	DECL_INLINE_THROW(uint32_t)
4589	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
4590	{
4591	#ifdef RT_ARCH_AMD64
4592	if (uImm <= UINT32_C(0xff))
4593	{
4594	/* cmp Ev, Ib */
4595	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
4596	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4597	pbCodeBuf[off++] = 0x83;
4598	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4599	pbCodeBuf[off++] = (uint8_t)uImm;
4600	}
4601	else if ((int64_t)uImm == (int32_t)uImm)
4602	{
4603	/* cmp Ev, imm */
4604	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
4605	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4606	pbCodeBuf[off++] = 0x81;
4607	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4608	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4609	pbCodeBuf[off++] = RT_BYTE1(uImm);
4610	pbCodeBuf[off++] = RT_BYTE2(uImm);
4611	pbCodeBuf[off++] = RT_BYTE3(uImm);
4612	pbCodeBuf[off++] = RT_BYTE4(uImm);
4613	}
4614	else
4615	{
4616	/* Use temporary register for the immediate. */
4617	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4618	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4619	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4620	}
4621
4622	#elif defined(RT_ARCH_ARM64)
4623	/** @todo guess there are clevere things we can do here... */
4624	if (uImm < _4K)
4625	{
4626	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4627	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4628	true /64Bit/, true /fSetFlags/);
4629	}
4630	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4631	{
4632	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4633	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
4634	true /64Bit/, true /fSetFlags/, true /fShift12/);
4635	}
4636	else
4637	{
4638	/* Use temporary register for the immediate. */
4639	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4640	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4641	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4642	}
4643
4644	#else
4645	# error "Port me!"
4646	#endif
4647
4648	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4649	return off;
4650	}
4651
4652
4653	/**
4654	* Emits a compare of a 32-bit GPR with a constant value, settings status
4655	* flags/whatever for use with conditional instruction.
4656	*
4657	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
4658	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
4659	* bits all zero). Will release assert or throw exception if the caller
4660	* violates this restriction.
4661	*/
4662	DECL_FORCE_INLINE_THROW(uint32_t)
4663	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4664	{
4665	#ifdef RT_ARCH_AMD64
4666	if (iGprLeft >= 8)
4667	pCodeBuf[off++] = X86_OP_REX_B;
4668	if (uImm <= UINT32_C(0x7f))
4669	{
4670	/* cmp Ev, Ib */
4671	pCodeBuf[off++] = 0x83;
4672	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4673	pCodeBuf[off++] = (uint8_t)uImm;
4674	}
4675	else
4676	{
4677	/* cmp Ev, imm */
4678	pCodeBuf[off++] = 0x81;
4679	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4680	pCodeBuf[off++] = RT_BYTE1(uImm);
4681	pCodeBuf[off++] = RT_BYTE2(uImm);
4682	pCodeBuf[off++] = RT_BYTE3(uImm);
4683	pCodeBuf[off++] = RT_BYTE4(uImm);
4684	}
4685
4686	#elif defined(RT_ARCH_ARM64)
4687	/** @todo guess there are clevere things we can do here... */
4688	if (uImm < _4K)
4689	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4690	false /64Bit/, true /fSetFlags/);
4691	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4692	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4693	false /64Bit/, true /fSetFlags/, true /fShift12/);
4694	else
4695	# ifdef IEM_WITH_THROW_CATCH
4696	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4697	# else
4698	AssertReleaseFailedStmt(off = UINT32_MAX);
4699	# endif
4700
4701	#else
4702	# error "Port me!"
4703	#endif
4704	return off;
4705	}
4706
4707
4708	/**
4709	* Emits a compare of a 32-bit GPR with a constant value, settings status
4710	* flags/whatever for use with conditional instruction.
4711	*/
4712	DECL_INLINE_THROW(uint32_t)
4713	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4714	{
4715	#ifdef RT_ARCH_AMD64
4716	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
4717
4718	#elif defined(RT_ARCH_ARM64)
4719	/** @todo guess there are clevere things we can do here... */
4720	if (uImm < _4K)
4721	{
4722	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4723	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4724	false /64Bit/, true /fSetFlags/);
4725	}
4726	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4727	{
4728	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4729	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4730	false /64Bit/, true /fSetFlags/, true /fShift12/);
4731	}
4732	else
4733	{
4734	/* Use temporary register for the immediate. */
4735	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4736	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
4737	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4738	}
4739
4740	#else
4741	# error "Port me!"
4742	#endif
4743
4744	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4745	return off;
4746	}
4747
4748
4749
4750	/*********************************************************************************************************************************
4751	* Branching *
4752	*********************************************************************************************************************************/
4753
4754	/**
4755	* Emits a JMP rel32 / B imm19 to the given label.
4756	*/
4757	DECL_FORCE_INLINE_THROW(uint32_t)
4758	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
4759	{
4760	Assert(idxLabel < pReNative->cLabels);
4761
4762	#ifdef RT_ARCH_AMD64
4763	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4764	{
4765	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
4766	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
4767	{
4768	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
4769	pCodeBuf[off++] = (uint8_t)offRel;
4770	}
4771	else
4772	{
4773	offRel -= 3;
4774	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4775	pCodeBuf[off++] = RT_BYTE1(offRel);
4776	pCodeBuf[off++] = RT_BYTE2(offRel);
4777	pCodeBuf[off++] = RT_BYTE3(offRel);
4778	pCodeBuf[off++] = RT_BYTE4(offRel);
4779	}
4780	}
4781	else
4782	{
4783	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4784	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4785	pCodeBuf[off++] = 0xfe;
4786	pCodeBuf[off++] = 0xff;
4787	pCodeBuf[off++] = 0xff;
4788	pCodeBuf[off++] = 0xff;
4789	}
4790	pCodeBuf[off++] = 0xcc; /* int3 poison */
4791
4792	#elif defined(RT_ARCH_ARM64)
4793	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4794	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
4795	else
4796	{
4797	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
4798	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
4799	}
4800
4801	#else
4802	# error "Port me!"
4803	#endif
4804	return off;
4805	}
4806
4807
4808	/**
4809	* Emits a JMP rel32 / B imm19 to the given label.
4810	*/
4811	DECL_INLINE_THROW(uint32_t)
4812	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4813	{
4814	#ifdef RT_ARCH_AMD64
4815	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
4816	#elif defined(RT_ARCH_ARM64)
4817	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
4818	#else
4819	# error "Port me!"
4820	#endif
4821	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4822	return off;
4823	}
4824
4825
4826	/**
4827	* Emits a JMP rel32 / B imm19 to a new undefined label.
4828	*/
4829	DECL_INLINE_THROW(uint32_t)
4830	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4831	{
4832	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4833	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
4834	}
4835
4836	/** Condition type. */
4837	#ifdef RT_ARCH_AMD64
4838	typedef enum IEMNATIVEINSTRCOND : uint8_t
4839	{
4840	kIemNativeInstrCond_o = 0,
4841	kIemNativeInstrCond_no,
4842	kIemNativeInstrCond_c,
4843	kIemNativeInstrCond_nc,
4844	kIemNativeInstrCond_e,
4845	kIemNativeInstrCond_ne,
4846	kIemNativeInstrCond_be,
4847	kIemNativeInstrCond_nbe,
4848	kIemNativeInstrCond_s,
4849	kIemNativeInstrCond_ns,
4850	kIemNativeInstrCond_p,
4851	kIemNativeInstrCond_np,
4852	kIemNativeInstrCond_l,
4853	kIemNativeInstrCond_nl,
4854	kIemNativeInstrCond_le,
4855	kIemNativeInstrCond_nle
4856	} IEMNATIVEINSTRCOND;
4857	#elif defined(RT_ARCH_ARM64)
4858	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
4859	# define kIemNativeInstrCond_o todo_conditional_codes
4860	# define kIemNativeInstrCond_no todo_conditional_codes
4861	# define kIemNativeInstrCond_c todo_conditional_codes
4862	# define kIemNativeInstrCond_nc todo_conditional_codes
4863	# define kIemNativeInstrCond_e kArmv8InstrCond_Eq
4864	# define kIemNativeInstrCond_ne kArmv8InstrCond_Ne
4865	# define kIemNativeInstrCond_be kArmv8InstrCond_Ls
4866	# define kIemNativeInstrCond_nbe kArmv8InstrCond_Hi
4867	# define kIemNativeInstrCond_s todo_conditional_codes
4868	# define kIemNativeInstrCond_ns todo_conditional_codes
4869	# define kIemNativeInstrCond_p todo_conditional_codes
4870	# define kIemNativeInstrCond_np todo_conditional_codes
4871	# define kIemNativeInstrCond_l kArmv8InstrCond_Lt
4872	# define kIemNativeInstrCond_nl kArmv8InstrCond_Ge
4873	# define kIemNativeInstrCond_le kArmv8InstrCond_Le
4874	# define kIemNativeInstrCond_nle kArmv8InstrCond_Gt
4875	#else
4876	# error "Port me!"
4877	#endif
4878
4879
4880	/**
4881	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
4882	*/
4883	DECL_FORCE_INLINE_THROW(uint32_t)
4884	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
4885	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
4886	{
4887	Assert(idxLabel < pReNative->cLabels);
4888
4889	uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
4890	#ifdef RT_ARCH_AMD64
4891	if (offLabel >= off)
4892	{
4893	/* jcc rel32 */
4894	pCodeBuf[off++] = 0x0f;
4895	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4896	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
4897	pCodeBuf[off++] = 0x00;
4898	pCodeBuf[off++] = 0x00;
4899	pCodeBuf[off++] = 0x00;
4900	pCodeBuf[off++] = 0x00;
4901	}
4902	else
4903	{
4904	int32_t offDisp = offLabel - (off + 2);
4905	if ((int8_t)offDisp == offDisp)
4906	{
4907	/* jcc rel8 */
4908	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
4909	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4910	}
4911	else
4912	{
4913	/* jcc rel32 */
4914	offDisp -= 4;
4915	pCodeBuf[off++] = 0x0f;
4916	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
4917	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
4918	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
4919	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
4920	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
4921	}
4922	}
4923
4924	#elif defined(RT_ARCH_ARM64)
4925	if (offLabel >= off)
4926	{
4927	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4928	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
4929	}
4930	else
4931	{
4932	Assert(off - offLabel <= 0x3ffffU);
4933	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
4934	}
4935
4936	#else
4937	# error "Port me!"
4938	#endif
4939	return off;
4940	}
4941
4942
4943	/**
4944	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
4945	*/
4946	DECL_INLINE_THROW(uint32_t)
4947	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
4948	{
4949	#ifdef RT_ARCH_AMD64
4950	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
4951	#elif defined(RT_ARCH_ARM64)
4952	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
4953	#else
4954	# error "Port me!"
4955	#endif
4956	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4957	return off;
4958	}
4959
4960
4961	/**
4962	* Emits a Jcc rel32 / B.cc imm19 to a new label.
4963	*/
4964	DECL_INLINE_THROW(uint32_t)
4965	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4966	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
4967	{
4968	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4969	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
4970	}
4971
4972
4973	/**
4974	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
4975	*/
4976	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
4977	{
4978	#ifdef RT_ARCH_AMD64
4979	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
4980	#elif defined(RT_ARCH_ARM64)
4981	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
4982	#else
4983	# error "Port me!"
4984	#endif
4985	}
4986
4987	/**
4988	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
4989	*/
4990	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4991	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4992	{
4993	#ifdef RT_ARCH_AMD64
4994	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
4995	#elif defined(RT_ARCH_ARM64)
4996	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
4997	#else
4998	# error "Port me!"
4999	#endif
5000	}
5001
5002
5003	/**
5004	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
5005	*/
5006	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5007	{
5008	#ifdef RT_ARCH_AMD64
5009	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
5010	#elif defined(RT_ARCH_ARM64)
5011	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
5012	#else
5013	# error "Port me!"
5014	#endif
5015	}
5016
5017	/**
5018	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
5019	*/
5020	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5021	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5022	{
5023	#ifdef RT_ARCH_AMD64
5024	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
5025	#elif defined(RT_ARCH_ARM64)
5026	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
5027	#else
5028	# error "Port me!"
5029	#endif
5030	}
5031
5032
5033	/**
5034	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
5035	*/
5036	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5037	{
5038	#ifdef RT_ARCH_AMD64
5039	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
5040	#elif defined(RT_ARCH_ARM64)
5041	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
5042	#else
5043	# error "Port me!"
5044	#endif
5045	}
5046
5047	/**
5048	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
5049	*/
5050	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5051	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5052	{
5053	#ifdef RT_ARCH_AMD64
5054	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
5055	#elif defined(RT_ARCH_ARM64)
5056	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
5057	#else
5058	# error "Port me!"
5059	#endif
5060	}
5061
5062
5063	/**
5064	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
5065	*/
5066	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5067	{
5068	#ifdef RT_ARCH_AMD64
5069	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
5070	#elif defined(RT_ARCH_ARM64)
5071	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
5072	#else
5073	# error "Port me!"
5074	#endif
5075	}
5076
5077	/**
5078	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
5079	*/
5080	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5081	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5082	{
5083	#ifdef RT_ARCH_AMD64
5084	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
5085	#elif defined(RT_ARCH_ARM64)
5086	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
5087	#else
5088	# error "Port me!"
5089	#endif
5090	}
5091
5092
5093	/**
5094	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
5095	*/
5096	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5097	{
5098	#ifdef RT_ARCH_AMD64
5099	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
5100	#elif defined(RT_ARCH_ARM64)
5101	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
5102	#else
5103	# error "Port me!"
5104	#endif
5105	}
5106
5107	/**
5108	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
5109	*/
5110	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5111	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5112	{
5113	#ifdef RT_ARCH_AMD64
5114	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
5115	#elif defined(RT_ARCH_ARM64)
5116	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
5117	#else
5118	# error "Port me!"
5119	#endif
5120	}
5121
5122
5123	/**
5124	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5125	*
5126	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5127	*
5128	* Only use hardcoded jumps forward when emitting for exactly one
5129	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5130	* the right target address on all platforms!
5131	*
5132	* Please also note that on x86 it is necessary pass off + 256 or higher
5133	* for @a offTarget one believe the intervening code is more than 127
5134	* bytes long.
5135	*/
5136	DECL_FORCE_INLINE(uint32_t)
5137	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5138	{
5139	#ifdef RT_ARCH_AMD64
5140	/* jcc rel8 / rel32 */
5141	int32_t offDisp = (int32_t)(offTarget - (off + 2));
5142	if (offDisp < 128 && offDisp >= -128)
5143	{
5144	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
5145	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5146	}
5147	else
5148	{
5149	offDisp -= 4;
5150	pCodeBuf[off++] = 0x0f;
5151	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5152	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5153	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5154	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5155	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5156	}
5157
5158	#elif defined(RT_ARCH_ARM64)
5159	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
5160
5161	#else
5162	# error "Port me!"
5163	#endif
5164	return off;
5165	}
5166
5167
5168	/**
5169	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5170	*
5171	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5172	*
5173	* Only use hardcoded jumps forward when emitting for exactly one
5174	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5175	* the right target address on all platforms!
5176	*
5177	* Please also note that on x86 it is necessary pass off + 256 or higher
5178	* for @a offTarget one believe the intervening code is more than 127
5179	* bytes long.
5180	*/
5181	DECL_INLINE_THROW(uint32_t)
5182	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5183	{
5184	#ifdef RT_ARCH_AMD64
5185	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
5186	#elif defined(RT_ARCH_ARM64)
5187	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
5188	#else
5189	# error "Port me!"
5190	#endif
5191	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5192	return off;
5193	}
5194
5195
5196	/**
5197	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
5198	*
5199	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5200	*/
5201	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5202	{
5203	#ifdef RT_ARCH_AMD64
5204	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
5205	#elif defined(RT_ARCH_ARM64)
5206	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
5207	#else
5208	# error "Port me!"
5209	#endif
5210	}
5211
5212
5213	/**
5214	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
5215	*
5216	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5217	*/
5218	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5219	{
5220	#ifdef RT_ARCH_AMD64
5221	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
5222	#elif defined(RT_ARCH_ARM64)
5223	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
5224	#else
5225	# error "Port me!"
5226	#endif
5227	}
5228
5229
5230	/**
5231	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
5232	*
5233	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5234	*/
5235	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5236	{
5237	#ifdef RT_ARCH_AMD64
5238	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
5239	#elif defined(RT_ARCH_ARM64)
5240	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
5241	#else
5242	# error "Port me!"
5243	#endif
5244	}
5245
5246
5247	/**
5248	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
5249	*
5250	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5251	*/
5252	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5253	{
5254	#ifdef RT_ARCH_AMD64
5255	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
5256	#elif defined(RT_ARCH_ARM64)
5257	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
5258	#else
5259	# error "Port me!"
5260	#endif
5261	}
5262
5263
5264	/**
5265	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5266	*
5267	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5268	*/
5269	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
5270	{
5271	#ifdef RT_ARCH_AMD64
5272	/* jmp rel8 or rel32 */
5273	int32_t offDisp = offTarget - (off + 2);
5274	if (offDisp < 128 && offDisp >= -128)
5275	{
5276	pCodeBuf[off++] = 0xeb;
5277	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5278	}
5279	else
5280	{
5281	offDisp -= 3;
5282	pCodeBuf[off++] = 0xe9;
5283	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5284	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5285	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5286	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5287	}
5288
5289	#elif defined(RT_ARCH_ARM64)
5290	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
5291
5292	#else
5293	# error "Port me!"
5294	#endif
5295	return off;
5296	}
5297
5298
5299	/**
5300	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5301	*
5302	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5303	*/
5304	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5305	{
5306	#ifdef RT_ARCH_AMD64
5307	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
5308	#elif defined(RT_ARCH_ARM64)
5309	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
5310	#else
5311	# error "Port me!"
5312	#endif
5313	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5314	return off;
5315	}
5316
5317
5318	/**
5319	* Fixes up a conditional jump to a fixed label.
5320	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
5321	* iemNativeEmitJzToFixed, ...
5322	*/
5323	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
5324	{
5325	#ifdef RT_ARCH_AMD64
5326	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
5327	uint8_t const bOpcode = pbCodeBuf[offFixup];
5328	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
5329	{
5330	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
5331	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
5332	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
5333	}
5334	else
5335	{
5336	if (bOpcode != 0x0f)
5337	Assert(bOpcode == 0xe9);
5338	else
5339	{
5340	offFixup += 1;
5341	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
5342	}
5343	uint32_t const offRel32 = offTarget - (offFixup + 5);
5344	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
5345	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
5346	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
5347	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
5348	}
5349
5350	#elif defined(RT_ARCH_ARM64)
5351	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
5352	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
5353	{
5354	/* B.COND + BC.COND */
5355	int32_t const offDisp = offTarget - offFixup;
5356	Assert(offDisp >= -262144 && offDisp < 262144);
5357	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
5358	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
5359	}
5360	else
5361	{
5362	/* B imm26 */
5363	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
5364	int32_t const offDisp = offTarget - offFixup;
5365	Assert(offDisp >= -33554432 && offDisp < 33554432);
5366	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
5367	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
5368	}
5369
5370	#else
5371	# error "Port me!"
5372	#endif
5373	}
5374
5375
5376	/**
5377	* Internal helper, don't call directly.
5378	*/
5379	DECL_INLINE_THROW(uint32_t)
5380	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5381	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
5382	{
5383	Assert(iBitNo < 64);
5384	#ifdef RT_ARCH_AMD64
5385	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5386	if (iBitNo < 8)
5387	{
5388	/* test Eb, imm8 */
5389	if (iGprSrc >= 4)
5390	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5391	pbCodeBuf[off++] = 0xf6;
5392	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5393	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
5394	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5395	}
5396	else
5397	{
5398	/* bt Ev, imm8 */
5399	if (iBitNo >= 32)
5400	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
5401	else if (iGprSrc >= 8)
5402	pbCodeBuf[off++] = X86_OP_REX_B;
5403	pbCodeBuf[off++] = 0x0f;
5404	pbCodeBuf[off++] = 0xba;
5405	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
5406	pbCodeBuf[off++] = iBitNo;
5407	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
5408	}
5409
5410	#elif defined(RT_ARCH_ARM64)
5411	/* Use the TBNZ instruction here. */
5412	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5413	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
5414	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
5415
5416	#else
5417	# error "Port me!"
5418	#endif
5419	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5420	return off;
5421	}
5422
5423
5424	/**
5425	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
5426	* @a iGprSrc.
5427	*
5428	* @note On ARM64 the range is only +/-8191 instructions.
5429	*/
5430	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5431	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5432	{
5433	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
5434	}
5435
5436
5437	/**
5438	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
5439	* _set_ in @a iGprSrc.
5440	*
5441	* @note On ARM64 the range is only +/-8191 instructions.
5442	*/
5443	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5444	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5445	{
5446	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
5447	}
5448
5449
5450	/**
5451	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
5452	* flags accordingly.
5453	*/
5454	DECL_INLINE_THROW(uint32_t)
5455	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
5456	{
5457	Assert(fBits != 0);
5458	#ifdef RT_ARCH_AMD64
5459
5460	if (fBits >= UINT32_MAX)
5461	{
5462	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5463
5464	/* test Ev,Gv */
5465	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5466	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
5467	pbCodeBuf[off++] = 0x85;
5468	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
5469
5470	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5471	}
5472	else if (fBits <= UINT32_MAX)
5473	{
5474	/* test Eb, imm8 or test Ev, imm32 */
5475	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
5476	if (fBits <= UINT8_MAX)
5477	{
5478	if (iGprSrc >= 4)
5479	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5480	pbCodeBuf[off++] = 0xf6;
5481	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5482	pbCodeBuf[off++] = (uint8_t)fBits;
5483	}
5484	else
5485	{
5486	if (iGprSrc >= 8)
5487	pbCodeBuf[off++] = X86_OP_REX_B;
5488	pbCodeBuf[off++] = 0xf7;
5489	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5490	pbCodeBuf[off++] = RT_BYTE1(fBits);
5491	pbCodeBuf[off++] = RT_BYTE2(fBits);
5492	pbCodeBuf[off++] = RT_BYTE3(fBits);
5493	pbCodeBuf[off++] = RT_BYTE4(fBits);
5494	}
5495	}
5496	/** @todo implement me. */
5497	else
5498	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
5499
5500	#elif defined(RT_ARCH_ARM64)
5501	uint32_t uImmR = 0;
5502	uint32_t uImmNandS = 0;
5503	if (Armv8A64ConvertMask64ToImmRImmS(fBits, &uImmNandS, &uImmR))
5504	{
5505	/* ands xzr, iGprSrc, #fBits */
5506	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5507	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR);
5508	}
5509	else
5510	{
5511	/* ands xzr, iGprSrc, iTmpReg */
5512	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5513	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5514	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
5515	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5516	}
5517
5518	#else
5519	# error "Port me!"
5520	#endif
5521	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5522	return off;
5523	}
5524
5525
5526	/**
5527	* Emits a test for any of the bits from @a fBits in the lower 32 bits of
5528	* @a iGprSrc, setting CPU flags accordingly.
5529	*
5530	* @note For ARM64 this only supports @a fBits values that can be expressed
5531	* using the two 6-bit immediates of the ANDS instruction. The caller
5532	* must make sure this is possible!
5533	*/
5534	DECL_FORCE_INLINE_THROW(uint32_t)
5535	iemNativeEmitTestAnyBitsInGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint32_t fBits)
5536	{
5537	Assert(fBits != 0);
5538
5539	#ifdef RT_ARCH_AMD64
5540	if (fBits <= UINT8_MAX)
5541	{
5542	/* test Eb, imm8 */
5543	if (iGprSrc >= 4)
5544	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5545	pCodeBuf[off++] = 0xf6;
5546	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5547	pCodeBuf[off++] = (uint8_t)fBits;
5548	}
5549	else
5550	{
5551	/* test Ev, imm32 */
5552	if (iGprSrc >= 8)
5553	pCodeBuf[off++] = X86_OP_REX_B;
5554	pCodeBuf[off++] = 0xf7;
5555	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5556	pCodeBuf[off++] = RT_BYTE1(fBits);
5557	pCodeBuf[off++] = RT_BYTE2(fBits);
5558	pCodeBuf[off++] = RT_BYTE3(fBits);
5559	pCodeBuf[off++] = RT_BYTE4(fBits);
5560	}
5561
5562	#elif defined(RT_ARCH_ARM64)
5563	/* ands xzr, src, #fBits */
5564	uint32_t uImmR = 0;
5565	uint32_t uImmNandS = 0;
5566	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5567	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5568	else
5569	# ifdef IEM_WITH_THROW_CATCH
5570	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5571	# else
5572	AssertReleaseFailedStmt(off = UINT32_MAX);
5573	# endif
5574
5575	#else
5576	# error "Port me!"
5577	#endif
5578	return off;
5579	}
5580
5581
5582
5583	/**
5584	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5585	* @a iGprSrc, setting CPU flags accordingly.
5586	*
5587	* @note For ARM64 this only supports @a fBits values that can be expressed
5588	* using the two 6-bit immediates of the ANDS instruction. The caller
5589	* must make sure this is possible!
5590	*/
5591	DECL_FORCE_INLINE_THROW(uint32_t)
5592	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5593	{
5594	Assert(fBits != 0);
5595
5596	#ifdef RT_ARCH_AMD64
5597	/* test Eb, imm8 */
5598	if (iGprSrc >= 4)
5599	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5600	pCodeBuf[off++] = 0xf6;
5601	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5602	pCodeBuf[off++] = fBits;
5603
5604	#elif defined(RT_ARCH_ARM64)
5605	/* ands xzr, src, #fBits */
5606	uint32_t uImmR = 0;
5607	uint32_t uImmNandS = 0;
5608	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5609	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5610	else
5611	# ifdef IEM_WITH_THROW_CATCH
5612	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5613	# else
5614	AssertReleaseFailedStmt(off = UINT32_MAX);
5615	# endif
5616
5617	#else
5618	# error "Port me!"
5619	#endif
5620	return off;
5621	}
5622
5623
5624	/**
5625	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5626	* @a iGprSrc, setting CPU flags accordingly.
5627	*/
5628	DECL_INLINE_THROW(uint32_t)
5629	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5630	{
5631	Assert(fBits != 0);
5632
5633	#ifdef RT_ARCH_AMD64
5634	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
5635
5636	#elif defined(RT_ARCH_ARM64)
5637	/* ands xzr, src, [tmp\|#imm] */
5638	uint32_t uImmR = 0;
5639	uint32_t uImmNandS = 0;
5640	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5641	{
5642	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5643	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5644	}
5645	else
5646	{
5647	/* Use temporary register for the 64-bit immediate. */
5648	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5649	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5650	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
5651	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5652	}
5653
5654	#else
5655	# error "Port me!"
5656	#endif
5657	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5658	return off;
5659	}
5660
5661
5662	/**
5663	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
5664	* are set in @a iGprSrc.
5665	*/
5666	DECL_INLINE_THROW(uint32_t)
5667	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5668	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5669	{
5670	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5671
5672	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5673	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5674
5675	return off;
5676	}
5677
5678
5679	/**
5680	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
5681	* are set in @a iGprSrc.
5682	*/
5683	DECL_INLINE_THROW(uint32_t)
5684	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5685	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5686	{
5687	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5688
5689	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5690	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
5691
5692	return off;
5693	}
5694
5695
5696	/**
5697	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5698	*
5699	* The operand size is given by @a f64Bit.
5700	*/
5701	DECL_FORCE_INLINE_THROW(uint32_t)
5702	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5703	uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5704	{
5705	Assert(idxLabel < pReNative->cLabels);
5706
5707	#ifdef RT_ARCH_AMD64
5708	/* test reg32,reg32 / test reg64,reg64 */
5709	if (f64Bit)
5710	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
5711	else if (iGprSrc >= 8)
5712	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
5713	pCodeBuf[off++] = 0x85;
5714	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
5715
5716	/* jnz idxLabel */
5717	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
5718	fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5719
5720	#elif defined(RT_ARCH_ARM64)
5721	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
5722	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(pReNative->paLabels[idxLabel].off - off),
5723	iGprSrc, f64Bit);
5724	else
5725	{
5726	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5727	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, 0, iGprSrc, f64Bit);
5728	}
5729
5730	#else
5731	# error "Port me!"
5732	#endif
5733	return off;
5734	}
5735
5736
5737	/**
5738	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5739	*
5740	* The operand size is given by @a f64Bit.
5741	*/
5742	DECL_FORCE_INLINE_THROW(uint32_t)
5743	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5744	bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5745	{
5746	#ifdef RT_ARCH_AMD64
5747	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
5748	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5749	#elif defined(RT_ARCH_ARM64)
5750	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1),
5751	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5752	#else
5753	# error "Port me!"
5754	#endif
5755	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5756	return off;
5757	}
5758
5759
5760	/**
5761	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5762	*
5763	* The operand size is given by @a f64Bit.
5764	*/
5765	DECL_FORCE_INLINE_THROW(uint32_t)
5766	iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5767	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5768	{
5769	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5770	f64Bit, false /fJmpIfNotZero/, idxLabel);
5771	}
5772
5773
5774	/**
5775	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5776	*
5777	* The operand size is given by @a f64Bit.
5778	*/
5779	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5780	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5781	{
5782	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /fJmpIfNotZero/, idxLabel);
5783	}
5784
5785
5786	/**
5787	* Emits code that jumps to a new label if @a iGprSrc is zero.
5788	*
5789	* The operand size is given by @a f64Bit.
5790	*/
5791	DECL_INLINE_THROW(uint32_t)
5792	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5793	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5794	{
5795	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5796	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5797	}
5798
5799
5800	/**
5801	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5802	*
5803	* The operand size is given by @a f64Bit.
5804	*/
5805	DECL_FORCE_INLINE_THROW(uint32_t)
5806	iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5807	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5808	{
5809	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5810	f64Bit, true /fJmpIfNotZero/, idxLabel);
5811	}
5812
5813
5814	/**
5815	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5816	*
5817	* The operand size is given by @a f64Bit.
5818	*/
5819	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5820	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5821	{
5822	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /fJmpIfNotZero/, idxLabel);
5823	}
5824
5825
5826	/**
5827	* Emits code that jumps to a new label if @a iGprSrc is not zero.
5828	*
5829	* The operand size is given by @a f64Bit.
5830	*/
5831	DECL_INLINE_THROW(uint32_t)
5832	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
5833	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5834	{
5835	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5836	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
5837	}
5838
5839
5840	/**
5841	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
5842	* differs.
5843	*/
5844	DECL_INLINE_THROW(uint32_t)
5845	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5846	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
5847	{
5848	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
5849	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5850	return off;
5851	}
5852
5853
5854	/**
5855	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
5856	*/
5857	DECL_INLINE_THROW(uint32_t)
5858	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5859	uint8_t iGprLeft, uint8_t iGprRight,
5860	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5861	{
5862	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5863	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
5864	}
5865
5866
5867	/**
5868	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
5869	*/
5870	DECL_INLINE_THROW(uint32_t)
5871	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5872	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
5873	{
5874	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
5875	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5876	return off;
5877	}
5878
5879
5880	/**
5881	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
5882	*/
5883	DECL_INLINE_THROW(uint32_t)
5884	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5885	uint8_t iGprSrc, uint64_t uImm,
5886	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5887	{
5888	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5889	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
5890	}
5891
5892
5893	/**
5894	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
5895	* @a uImm.
5896	*/
5897	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5898	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
5899	{
5900	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
5901	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5902	return off;
5903	}
5904
5905
5906	/**
5907	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
5908	* @a uImm.
5909	*/
5910	DECL_INLINE_THROW(uint32_t)
5911	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5912	uint8_t iGprSrc, uint32_t uImm,
5913	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5914	{
5915	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5916	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
5917	}
5918
5919
5920	/*********************************************************************************************************************************
5921	* Calls. *
5922	*********************************************************************************************************************************/
5923
5924	/**
5925	* Emits a call to a 64-bit address.
5926	*/
5927	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
5928	{
5929	#ifdef RT_ARCH_AMD64
5930	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
5931
5932	/* call rax */
5933	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
5934	pbCodeBuf[off++] = 0xff;
5935	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
5936
5937	#elif defined(RT_ARCH_ARM64)
5938	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
5939
5940	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5941	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
5942
5943	#else
5944	# error "port me"
5945	#endif
5946	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5947	return off;
5948	}
5949
5950
5951	/**
5952	* Emits code to load a stack variable into an argument GPR.
5953	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
5954	*/
5955	DECL_FORCE_INLINE_THROW(uint32_t)
5956	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
5957	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = UINT32_MAX,
5958	bool fSpilledVarsInVolatileRegs = false)
5959	{
5960	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
5961	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
5962	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
5963
5964	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
5965	if ( idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs)
5966	&& ( (RT_BIT_32(idxRegVar) & (~IEMNATIVE_CALL_VOLATILE_GREG_MASK \| fHstVolatileRegsAllowed))
5967	\|\| !fSpilledVarsInVolatileRegs ))
5968	{
5969	AssertStmt( !(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK)
5970	\|\| (RT_BIT_32(idxRegVar) & fHstVolatileRegsAllowed),
5971	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_13));
5972	if (!offAddend)
5973	{
5974	if (idxRegArg != idxRegVar)
5975	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
5976	}
5977	else
5978	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
5979	}
5980	else
5981	{
5982	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
5983	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
5984	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
5985	if (offAddend)
5986	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
5987	}
5988	return off;
5989	}
5990
5991
5992	/**
5993	* Emits code to load a stack or immediate variable value into an argument GPR,
5994	* optional with a addend.
5995	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
5996	*/
5997	DECL_FORCE_INLINE_THROW(uint32_t)
5998	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
5999	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = 0,
6000	bool fSpilledVarsInVolatileRegs = false)
6001	{
6002	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6003	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
6004	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
6005	else
6006	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend,
6007	fHstVolatileRegsAllowed, fSpilledVarsInVolatileRegs);
6008	return off;
6009	}
6010
6011
6012	/**
6013	* Emits code to load the variable address into an argument GRP.
6014	*
6015	* This only works for uninitialized and stack variables.
6016	*/
6017	DECL_FORCE_INLINE_THROW(uint32_t)
6018	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6019	bool fFlushShadows)
6020	{
6021	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6022	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
6023	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
6024	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6025
6026	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
6027	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
6028
6029	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
6030	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
6031	{
6032	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
6033	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
6034	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6035	}
6036	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
6037	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6038
6039	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
6040	}
6041
6042
6043	/** @} */
6044
6045	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
6046

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102790

Download in other formats: