IEMN8veRecompilerEmit.h@ 102856

Last change on this file since 102856 was 102850, checked in by vboxsync, 11 months ago
VMM/IEM: Implemented the first of two code TLB lookups. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 214.3 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102850 2024-01-12 00:47:47Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	pCodeBuf[off++] = 0xcc;
88	RT_NOREF(uInfo); /** @todo use multibyte nop for info? */
89
90	#elif defined(RT_ARCH_ARM64)
91	pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
92
93	#else
94	# error "error"
95	#endif
96	return off;
97	}
98
99
100	/**
101	* Emit a breakpoint instruction.
102	*/
103	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
104	{
105	#ifdef RT_ARCH_AMD64
106	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
107	#elif defined(RT_ARCH_ARM64)
108	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
109	#else
110	# error "error"
111	#endif
112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
113	return off;
114	}
115
116
117	/*********************************************************************************************************************************
118	* Loads, Stores and Related Stuff. *
119	*********************************************************************************************************************************/
120
121	#ifdef RT_ARCH_AMD64
122	/**
123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
124	*/
125	DECL_FORCE_INLINE(uint32_t)
126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
127	{
128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
129	{
130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
133	}
134	else if (offDisp == (int8_t)offDisp)
135	{
136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
139	pbCodeBuf[off++] = (uint8_t)offDisp;
140	}
141	else
142	{
143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
150	}
151	return off;
152	}
153	#endif /* RT_ARCH_AMD64 */
154
155	/**
156	* Emits setting a GPR to zero.
157	*/
158	DECL_INLINE_THROW(uint32_t)
159	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
160	{
161	#ifdef RT_ARCH_AMD64
162	/* xor gpr32, gpr32 */
163	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
164	if (iGpr >= 8)
165	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
166	pbCodeBuf[off++] = 0x33;
167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
168
169	#elif defined(RT_ARCH_ARM64)
170	/* mov gpr, #0x0 */
171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
172	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
173
174	#else
175	# error "port me"
176	#endif
177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
178	return off;
179	}
180
181
182	/**
183	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
184	* buffer space.
185	*
186	* Max buffer consumption:
187	* - AMD64: 10 instruction bytes.
188	* - ARM64: 4 instruction words (16 bytes).
189	*/
190	DECL_FORCE_INLINE(uint32_t)
191	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
192	{
193	#ifdef RT_ARCH_AMD64
194	if (uImm64 == 0)
195	{
196	/* xor gpr, gpr */
197	if (iGpr >= 8)
198	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
199	pCodeBuf[off++] = 0x33;
200	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
201	}
202	else if (uImm64 <= UINT32_MAX)
203	{
204	/* mov gpr, imm32 */
205	if (iGpr >= 8)
206	pCodeBuf[off++] = X86_OP_REX_B;
207	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else if (uImm64 == (uint64_t)(int32_t)uImm64)
214	{
215	/* mov gpr, sx(imm32) */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xc7;
221	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
222	pCodeBuf[off++] = RT_BYTE1(uImm64);
223	pCodeBuf[off++] = RT_BYTE2(uImm64);
224	pCodeBuf[off++] = RT_BYTE3(uImm64);
225	pCodeBuf[off++] = RT_BYTE4(uImm64);
226	}
227	else
228	{
229	/* mov gpr, imm64 */
230	if (iGpr < 8)
231	pCodeBuf[off++] = X86_OP_REX_W;
232	else
233	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
234	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
235	pCodeBuf[off++] = RT_BYTE1(uImm64);
236	pCodeBuf[off++] = RT_BYTE2(uImm64);
237	pCodeBuf[off++] = RT_BYTE3(uImm64);
238	pCodeBuf[off++] = RT_BYTE4(uImm64);
239	pCodeBuf[off++] = RT_BYTE5(uImm64);
240	pCodeBuf[off++] = RT_BYTE6(uImm64);
241	pCodeBuf[off++] = RT_BYTE7(uImm64);
242	pCodeBuf[off++] = RT_BYTE8(uImm64);
243	}
244
245	#elif defined(RT_ARCH_ARM64)
246	/*
247	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
248	* supply remaining bits using 'movk grp, imm16, lsl #x'.
249	*
250	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
251	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
252	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
253	* after the first non-zero immediate component so we switch to movk for
254	* the remainder.
255	*/
256	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
257	+ !((uImm64 >> 16) & UINT16_MAX)
258	+ !((uImm64 >> 32) & UINT16_MAX)
259	+ !((uImm64 >> 48) & UINT16_MAX);
260	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
261	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
262	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
263	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
264	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
265	if (cFfffHalfWords <= cZeroHalfWords)
266	{
267	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
268
269	/* movz gpr, imm16 */
270	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
271	if (uImmPart \|\| cZeroHalfWords == 4)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #16 */
277	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
278	if (uImmPart)
279	{
280	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
281	fMovBase \|= RT_BIT_32(29);
282	}
283	/* mov[z/k] gpr, imm16, lsl #32 */
284	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
285	if (uImmPart)
286	{
287	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
288	fMovBase \|= RT_BIT_32(29);
289	}
290	/* mov[z/k] gpr, imm16, lsl #48 */
291	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
292	if (uImmPart)
293	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
294	}
295	else
296	{
297	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
298
299	/* find the first half-word that isn't UINT16_MAX. */
300	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
301	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
302	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
303
304	/* movn gpr, imm16, lsl #iHwNotFfff16 /
305	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
306	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
307	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
308	/* movk gpr, imm16 */
309	if (iHwNotFfff != 0)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #16 */
316	if (iHwNotFfff != 1)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
321	}
322	/* movk gpr, imm16, lsl #32 */
323	if (iHwNotFfff != 2)
324	{
325	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
326	if (uImmPart != UINT32_C(0xffff))
327	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
328	}
329	/* movk gpr, imm16, lsl #48 */
330	if (iHwNotFfff != 3)
331	{
332	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
333	if (uImmPart != UINT32_C(0xffff))
334	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
335	}
336	}
337
338	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
339	* clang 12.x does that, only to use the 'x' version for the
340	* addressing in the following ldr). */
341
342	#else
343	# error "port me"
344	#endif
345	return off;
346	}
347
348
349	/**
350	* Emits loading a constant into a 64-bit GPR
351	*/
352	DECL_INLINE_THROW(uint32_t)
353	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
354	{
355	#ifdef RT_ARCH_AMD64
356	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
357	#elif defined(RT_ARCH_ARM64)
358	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
359	#else
360	# error "port me"
361	#endif
362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
363	return off;
364	}
365
366
367	/**
368	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
369	* buffer space.
370	*
371	* Max buffer consumption:
372	* - AMD64: 6 instruction bytes.
373	* - ARM64: 2 instruction words (8 bytes).
374	*
375	* @note The top 32 bits will be cleared.
376	*/
377	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
378	{
379	#ifdef RT_ARCH_AMD64
380	if (uImm32 == 0)
381	{
382	/* xor gpr, gpr */
383	if (iGpr >= 8)
384	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
385	pCodeBuf[off++] = 0x33;
386	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
387	}
388	else
389	{
390	/* mov gpr, imm32 */
391	if (iGpr >= 8)
392	pCodeBuf[off++] = X86_OP_REX_B;
393	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
394	pCodeBuf[off++] = RT_BYTE1(uImm32);
395	pCodeBuf[off++] = RT_BYTE2(uImm32);
396	pCodeBuf[off++] = RT_BYTE3(uImm32);
397	pCodeBuf[off++] = RT_BYTE4(uImm32);
398	}
399
400	#elif defined(RT_ARCH_ARM64)
401	if ((uImm32 >> 16) == 0)
402	/* movz gpr, imm16 */
403	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
404	else if ((uImm32 & UINT32_C(0xffff)) == 0)
405	/* movz gpr, imm16, lsl #16 */
406	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
407	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
408	/* movn gpr, imm16, lsl #16 */
409	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
410	else if ((uImm32 >> 16) == UINT32_C(0xffff))
411	/* movn gpr, imm16 */
412	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
413	else
414	{
415	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
416	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
417	}
418
419	#else
420	# error "port me"
421	#endif
422	return off;
423	}
424
425
426	/**
427	* Emits loading a constant into a 32-bit GPR.
428	* @note The top 32 bits will be cleared.
429	*/
430	DECL_INLINE_THROW(uint32_t)
431	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
432	{
433	#ifdef RT_ARCH_AMD64
434	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
435	#elif defined(RT_ARCH_ARM64)
436	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
437	#else
438	# error "port me"
439	#endif
440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
441	return off;
442	}
443
444
445	/**
446	* Emits loading a constant into a 8-bit GPR
447	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
448	* only the ARM64 version does that.
449	*/
450	DECL_INLINE_THROW(uint32_t)
451	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
452	{
453	#ifdef RT_ARCH_AMD64
454	/* mov gpr, imm8 */
455	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
456	if (iGpr >= 8)
457	pbCodeBuf[off++] = X86_OP_REX_B;
458	else if (iGpr >= 4)
459	pbCodeBuf[off++] = X86_OP_REX;
460	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
461	pbCodeBuf[off++] = RT_BYTE1(uImm8);
462
463	#elif defined(RT_ARCH_ARM64)
464	/* movz gpr, imm16, lsl #0 */
465	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
466	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
467
468	#else
469	# error "port me"
470	#endif
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472	return off;
473	}
474
475
476	#ifdef RT_ARCH_AMD64
477	/**
478	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
479	*/
480	DECL_FORCE_INLINE(uint32_t)
481	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
482	{
483	if (offVCpu < 128)
484	{
485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
486	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
487	}
488	else
489	{
490	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
491	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
492	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
493	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
494	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
495	}
496	return off;
497	}
498
499	#elif defined(RT_ARCH_ARM64)
500
501	/**
502	* Common bit of iemNativeEmitLoadGprFromVCpuU64Ex and friends.
503	*
504	* @note Loads can use @a iGprReg for large offsets, stores requires a temporary
505	* registers (@a iGprTmp).
506	* @note DON'T try this with prefetch.
507	*/
508	DECL_FORCE_INLINE_THROW(uint32_t)
509	iemNativeEmitGprByVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu,
510	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
511	{
512	/*
513	* There are a couple of ldr variants that takes an immediate offset, so
514	* try use those if we can, otherwise we have to use the temporary register
515	* help with the addressing.
516	*/
517	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
518	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
519	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
520	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
521	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
522	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
523	else if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) \|\| iGprTmp != UINT8_MAX)
524	{
525	/* The offset is too large, so we must load it into a register and use
526	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
527	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
528	if (iGprTmp == UINT8_MAX)
529	iGprTmp = iGprReg;
530	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, offVCpu);
531	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, iGprTmp);
532	}
533	else
534	# ifdef IEM_WITH_THROW_CATCH
535	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
536	# else
537	AssertReleaseFailedStmt(off = UINT32_MAX);
538	# endif
539
540	return off;
541	}
542
543	/**
544	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
545	*/
546	DECL_FORCE_INLINE_THROW(uint32_t)
547	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
548	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
549	{
550	/*
551	* There are a couple of ldr variants that takes an immediate offset, so
552	* try use those if we can, otherwise we have to use the temporary register
553	* help with the addressing.
554	*/
555	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
556	{
557	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
558	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
559	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
560	}
561	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
562	{
563	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
564	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
565	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
566	}
567	else
568	{
569	/* The offset is too large, so we must load it into a register and use
570	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
571	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
572	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
574	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
575	IEMNATIVE_REG_FIXED_TMP0);
576	}
577	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
578	return off;
579	}
580
581	#endif /* RT_ARCH_ARM64 */
582
583
584	/**
585	* Emits a 64-bit GPR load of a VCpu value.
586	*/
587	DECL_FORCE_INLINE_THROW(uint32_t)
588	iemNativeEmitLoadGprFromVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
589	{
590	#ifdef RT_ARCH_AMD64
591	/* mov reg64, mem64 */
592	if (iGpr < 8)
593	pCodeBuf[off++] = X86_OP_REX_W;
594	else
595	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
596	pCodeBuf[off++] = 0x8b;
597	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
598
599	#elif defined(RT_ARCH_ARM64)
600	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
601
602	#else
603	# error "port me"
604	#endif
605	return off;
606	}
607
608
609	/**
610	* Emits a 64-bit GPR load of a VCpu value.
611	*/
612	DECL_INLINE_THROW(uint32_t)
613	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
614	{
615	#ifdef RT_ARCH_AMD64
616	off = iemNativeEmitLoadGprFromVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
617	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
618
619	#elif defined(RT_ARCH_ARM64)
620	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
621
622	#else
623	# error "port me"
624	#endif
625	return off;
626	}
627
628
629	/**
630	* Emits a 32-bit GPR load of a VCpu value.
631	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
632	*/
633	DECL_INLINE_THROW(uint32_t)
634	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
635	{
636	#ifdef RT_ARCH_AMD64
637	/* mov reg32, mem32 */
638	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
639	if (iGpr >= 8)
640	pbCodeBuf[off++] = X86_OP_REX_R;
641	pbCodeBuf[off++] = 0x8b;
642	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
644
645	#elif defined(RT_ARCH_ARM64)
646	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
647
648	#else
649	# error "port me"
650	#endif
651	return off;
652	}
653
654
655	/**
656	* Emits a 16-bit GPR load of a VCpu value.
657	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
658	*/
659	DECL_INLINE_THROW(uint32_t)
660	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
661	{
662	#ifdef RT_ARCH_AMD64
663	/* movzx reg32, mem16 */
664	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
665	if (iGpr >= 8)
666	pbCodeBuf[off++] = X86_OP_REX_R;
667	pbCodeBuf[off++] = 0x0f;
668	pbCodeBuf[off++] = 0xb7;
669	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
670	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
671
672	#elif defined(RT_ARCH_ARM64)
673	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
674
675	#else
676	# error "port me"
677	#endif
678	return off;
679	}
680
681
682	/**
683	* Emits a 8-bit GPR load of a VCpu value.
684	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
685	*/
686	DECL_INLINE_THROW(uint32_t)
687	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
688	{
689	#ifdef RT_ARCH_AMD64
690	/* movzx reg32, mem8 */
691	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
692	if (iGpr >= 8)
693	pbCodeBuf[off++] = X86_OP_REX_R;
694	pbCodeBuf[off++] = 0x0f;
695	pbCodeBuf[off++] = 0xb6;
696	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
697	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
698
699	#elif defined(RT_ARCH_ARM64)
700	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
701
702	#else
703	# error "port me"
704	#endif
705	return off;
706	}
707
708
709	/**
710	* Emits a store of a GPR value to a 64-bit VCpu field.
711	*/
712	DECL_FORCE_INLINE_THROW(uint32_t)
713	iemNativeEmitStoreGprToVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu,
714	uint8_t iGprTmp = UINT8_MAX)
715	{
716	#ifdef RT_ARCH_AMD64
717	/* mov mem64, reg64 */
718	if (iGpr < 8)
719	pCodeBuf[off++] = X86_OP_REX_W;
720	else
721	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
722	pCodeBuf[off++] = 0x89;
723	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
724	RT_NOREF(iGprTmp);
725
726	#elif defined(RT_ARCH_ARM64)
727	off = iemNativeEmitGprByVCpuLdStEx(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
728
729	#else
730	# error "port me"
731	#endif
732	return off;
733	}
734
735
736	/**
737	* Emits a store of a GPR value to a 64-bit VCpu field.
738	*/
739	DECL_INLINE_THROW(uint32_t)
740	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
741	{
742	#ifdef RT_ARCH_AMD64
743	off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
744	#elif defined(RT_ARCH_ARM64)
745	off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGpr, offVCpu,
746	IEMNATIVE_REG_FIXED_TMP0);
747	#else
748	# error "port me"
749	#endif
750	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
751	return off;
752	}
753
754
755	/**
756	* Emits a store of a GPR value to a 32-bit VCpu field.
757	*/
758	DECL_INLINE_THROW(uint32_t)
759	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
760	{
761	#ifdef RT_ARCH_AMD64
762	/* mov mem32, reg32 */
763	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
764	if (iGpr >= 8)
765	pbCodeBuf[off++] = X86_OP_REX_R;
766	pbCodeBuf[off++] = 0x89;
767	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
768	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
769
770	#elif defined(RT_ARCH_ARM64)
771	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
772
773	#else
774	# error "port me"
775	#endif
776	return off;
777	}
778
779
780	/**
781	* Emits a store of a GPR value to a 16-bit VCpu field.
782	*/
783	DECL_INLINE_THROW(uint32_t)
784	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
785	{
786	#ifdef RT_ARCH_AMD64
787	/* mov mem16, reg16 */
788	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
789	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
790	if (iGpr >= 8)
791	pbCodeBuf[off++] = X86_OP_REX_R;
792	pbCodeBuf[off++] = 0x89;
793	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
794	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
795
796	#elif defined(RT_ARCH_ARM64)
797	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
798
799	#else
800	# error "port me"
801	#endif
802	return off;
803	}
804
805
806	/**
807	* Emits a store of a GPR value to a 8-bit VCpu field.
808	*/
809	DECL_INLINE_THROW(uint32_t)
810	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
811	{
812	#ifdef RT_ARCH_AMD64
813	/* mov mem8, reg8 */
814	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
815	if (iGpr >= 8)
816	pbCodeBuf[off++] = X86_OP_REX_R;
817	pbCodeBuf[off++] = 0x88;
818	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
819	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
820
821	#elif defined(RT_ARCH_ARM64)
822	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
823
824	#else
825	# error "port me"
826	#endif
827	return off;
828	}
829
830
831	/**
832	* Emits a store of an immediate value to a 16-bit VCpu field.
833	*
834	* @note ARM64: A idxTmp1 is always required! The idxTmp2 depends on whehter the
835	* offset can be encoded as an immediate or not. The @a offVCpu immediate
836	* range is 0..8190 bytes from VMCPU and the same from CPUMCPU.
837	*/
838	DECL_FORCE_INLINE_THROW(uint32_t)
839	iemNativeEmitStoreImmToVCpuU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint32_t offVCpu,
840	uint8_t idxTmp1 = UINT8_MAX, uint8_t idxTmp2 = UINT8_MAX)
841	{
842	#ifdef RT_ARCH_AMD64
843	/* mov mem16, imm16 */
844	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
845	pCodeBuf[off++] = 0xc7;
846	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
847	pCodeBuf[off++] = RT_BYTE1(uImm);
848	pCodeBuf[off++] = RT_BYTE2(uImm);
849	RT_NOREF(idxTmp1, idxTmp2);
850
851	#elif defined(RT_ARCH_ARM64)
852	if (idxTmp1 != UINT8_MAX)
853	{
854	pCodeBuf[off++] = Armv8A64MkInstrMovZ(idxTmp1, uImm);
855	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, idxTmp1, offVCpu, kArmv8A64InstrLdStType_St_Half,
856	sizeof(uint16_t), idxTmp2);
857	}
858	else
859	# ifdef IEM_WITH_THROW_CATCH
860	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
861	# else
862	AssertReleaseFailedStmt(off = UINT32_MAX);
863	# endif
864
865	#else
866	# error "port me"
867	#endif
868	return off;
869	}
870
871
872	/**
873	* Emits a store of an immediate value to a 8-bit VCpu field.
874	*/
875	DECL_INLINE_THROW(uint32_t)
876	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
877	{
878	#ifdef RT_ARCH_AMD64
879	/* mov mem8, imm8 */
880	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
881	pbCodeBuf[off++] = 0xc6;
882	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
883	pbCodeBuf[off++] = bImm;
884	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
885
886	#elif defined(RT_ARCH_ARM64)
887	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
888	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
889	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
890	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
891
892	#else
893	# error "port me"
894	#endif
895	return off;
896	}
897
898
899	/**
900	* Emits a load effective address to a GRP of a VCpu field.
901	*/
902	DECL_INLINE_THROW(uint32_t)
903	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
904	{
905	#ifdef RT_ARCH_AMD64
906	/* lea gprdst, [rbx + offDisp] */
907	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
908	if (iGprDst < 8)
909	pbCodeBuf[off++] = X86_OP_REX_W;
910	else
911	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
912	pbCodeBuf[off++] = 0x8d;
913	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
914
915	#elif defined(RT_ARCH_ARM64)
916	if (offVCpu < (unsigned)_4K)
917	{
918	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
919	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
920	}
921	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
922	{
923	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
924	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
925	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
926	}
927	else
928	{
929	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
930	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
931	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
932	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
933	}
934
935	#else
936	# error "port me"
937	#endif
938	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
939	return off;
940	}
941
942
943	/** This is just as a typesafe alternative to RT_UOFFSETOF. */
944	DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromStamCounterPtr(PVMCPU pVCpu, PSTAMCOUNTER pStamCounter)
945	{
946	uintptr_t const off = (uintptr_t)pStamCounter - (uintptr_t)pVCpu;
947	Assert(off < sizeof(VMCPU));
948	return off;
949	}
950
951
952	/** This is just as a typesafe alternative to RT_UOFFSETOF. */
953	DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromU64Ptr(PVMCPU pVCpu, uint64_t *pu64)
954	{
955	uintptr_t const off = (uintptr_t)pu64 - (uintptr_t)pVCpu;
956	Assert(off < sizeof(VMCPU));
957	return off;
958	}
959
960
961	/**
962	* Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
963	*
964	* @note The two temp registers are not required for AMD64. ARM64 always
965	* requires the first, and the 2nd is needed if the offset cannot be
966	* encoded as an immediate.
967	*/
968	DECL_FORCE_INLINE(uint32_t)
969	iemNativeEmitIncStamCounterInVCpuEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
970	{
971	#ifdef RT_ARCH_AMD64
972	/* inc qword [pVCpu + off] */
973	pCodeBuf[off++] = X86_OP_REX_W;
974	pCodeBuf[off++] = 0xff;
975	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
976	RT_NOREF(idxTmp1, idxTmp2);
977
978	#elif defined(RT_ARCH_ARM64)
979	/* Determine how we're to access pVCpu first. */
980	uint32_t const cbData = sizeof(STAMCOUNTER);
981	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
982	{
983	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
984	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1,
985	IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
986	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, idxTmp1, 1);
987	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1,
988	IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
989	}
990	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
991	{
992	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
993	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
994	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, idxTmp1, 1);
995	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
996	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
997	}
998	else
999	{
1000	/* The offset is too large, so we must load it into a register and use
1001	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1002	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmp2, offVCpu);
1003	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
1004	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, idxTmp1, 1);
1005	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
1006	}
1007
1008	#else
1009	# error "port me"
1010	#endif
1011	return off;
1012	}
1013
1014
1015	/**
1016	* Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
1017	*
1018	* @note The two temp registers are not required for AMD64. ARM64 always
1019	* requires the first, and the 2nd is needed if the offset cannot be
1020	* encoded as an immediate.
1021	*/
1022	DECL_FORCE_INLINE(uint32_t)
1023	iemNativeEmitIncStamCounterInVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
1024	{
1025	#ifdef RT_ARCH_AMD64
1026	off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, idxTmp1, idxTmp2, offVCpu);
1027	#elif defined(RT_ARCH_ARM64)
1028	off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 4+3), off, idxTmp1, idxTmp2, offVCpu);
1029	#else
1030	# error "port me"
1031	#endif
1032	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1033	return off;
1034	}
1035
1036
1037	/**
1038	* Emits a gprdst = gprsrc load.
1039	*/
1040	DECL_FORCE_INLINE(uint32_t)
1041	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1042	{
1043	#ifdef RT_ARCH_AMD64
1044	/* mov gprdst, gprsrc */
1045	if ((iGprDst \| iGprSrc) >= 8)
1046	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
1047	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
1048	: X86_OP_REX_W \| X86_OP_REX_R;
1049	else
1050	pCodeBuf[off++] = X86_OP_REX_W;
1051	pCodeBuf[off++] = 0x8b;
1052	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1053
1054	#elif defined(RT_ARCH_ARM64)
1055	/* mov dst, src; alias for: orr dst, xzr, src */
1056	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
1057
1058	#else
1059	# error "port me"
1060	#endif
1061	return off;
1062	}
1063
1064
1065	/**
1066	* Emits a gprdst = gprsrc load.
1067	*/
1068	DECL_INLINE_THROW(uint32_t)
1069	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1070	{
1071	#ifdef RT_ARCH_AMD64
1072	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
1073	#elif defined(RT_ARCH_ARM64)
1074	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1075	#else
1076	# error "port me"
1077	#endif
1078	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1079	return off;
1080	}
1081
1082
1083	/**
1084	* Emits a gprdst = gprsrc[31:0] load.
1085	* @note Bits 63 thru 32 are cleared.
1086	*/
1087	DECL_FORCE_INLINE(uint32_t)
1088	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1089	{
1090	#ifdef RT_ARCH_AMD64
1091	/* mov gprdst, gprsrc */
1092	if ((iGprDst \| iGprSrc) >= 8)
1093	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1094	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1095	: X86_OP_REX_R;
1096	pCodeBuf[off++] = 0x8b;
1097	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1098
1099	#elif defined(RT_ARCH_ARM64)
1100	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
1101	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
1102
1103	#else
1104	# error "port me"
1105	#endif
1106	return off;
1107	}
1108
1109
1110	/**
1111	* Emits a gprdst = gprsrc[31:0] load.
1112	* @note Bits 63 thru 32 are cleared.
1113	*/
1114	DECL_INLINE_THROW(uint32_t)
1115	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1116	{
1117	#ifdef RT_ARCH_AMD64
1118	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
1119	#elif defined(RT_ARCH_ARM64)
1120	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1121	#else
1122	# error "port me"
1123	#endif
1124	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1125	return off;
1126	}
1127
1128
1129	/**
1130	* Emits a gprdst = gprsrc[15:0] load.
1131	* @note Bits 63 thru 15 are cleared.
1132	*/
1133	DECL_INLINE_THROW(uint32_t)
1134	iemNativeEmitLoadGprFromGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1135	{
1136	#ifdef RT_ARCH_AMD64
1137	/* movzx Gv,Ew */
1138	if ((iGprDst \| iGprSrc) >= 8)
1139	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1140	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1141	: X86_OP_REX_R;
1142	pCodeBuf[off++] = 0x0f;
1143	pCodeBuf[off++] = 0xb7;
1144	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1145
1146	#elif defined(RT_ARCH_ARM64)
1147	/* and gprdst, gprsrc, #0xffff */
1148	# if 1
1149	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
1150	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
1151	# else
1152	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
1153	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1154	# endif
1155
1156	#else
1157	# error "port me"
1158	#endif
1159	return off;
1160	}
1161
1162
1163	/**
1164	* Emits a gprdst = gprsrc[15:0] load.
1165	* @note Bits 63 thru 15 are cleared.
1166	*/
1167	DECL_INLINE_THROW(uint32_t)
1168	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1169	{
1170	#ifdef RT_ARCH_AMD64
1171	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1172	#elif defined(RT_ARCH_ARM64)
1173	off = iemNativeEmitLoadGprFromGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1174	#else
1175	# error "port me"
1176	#endif
1177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1178	return off;
1179	}
1180
1181
1182	/**
1183	* Emits a gprdst = gprsrc[7:0] load.
1184	* @note Bits 63 thru 8 are cleared.
1185	*/
1186	DECL_FORCE_INLINE(uint32_t)
1187	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1188	{
1189	#ifdef RT_ARCH_AMD64
1190	/* movzx Gv,Eb */
1191	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1192	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1193	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1194	: X86_OP_REX_R;
1195	else if (iGprSrc >= 4)
1196	pCodeBuf[off++] = X86_OP_REX;
1197	pCodeBuf[off++] = 0x0f;
1198	pCodeBuf[off++] = 0xb6;
1199	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1200
1201	#elif defined(RT_ARCH_ARM64)
1202	/* and gprdst, gprsrc, #0xff */
1203	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1204	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
1205
1206	#else
1207	# error "port me"
1208	#endif
1209	return off;
1210	}
1211
1212
1213	/**
1214	* Emits a gprdst = gprsrc[7:0] load.
1215	* @note Bits 63 thru 8 are cleared.
1216	*/
1217	DECL_INLINE_THROW(uint32_t)
1218	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1219	{
1220	#ifdef RT_ARCH_AMD64
1221	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
1222	#elif defined(RT_ARCH_ARM64)
1223	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
1224	#else
1225	# error "port me"
1226	#endif
1227	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1228	return off;
1229	}
1230
1231
1232	/**
1233	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1234	* @note Bits 63 thru 8 are cleared.
1235	*/
1236	DECL_INLINE_THROW(uint32_t)
1237	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1238	{
1239	#ifdef RT_ARCH_AMD64
1240	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1241
1242	/* movzx Gv,Ew */
1243	if ((iGprDst \| iGprSrc) >= 8)
1244	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1245	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1246	: X86_OP_REX_R;
1247	pbCodeBuf[off++] = 0x0f;
1248	pbCodeBuf[off++] = 0xb7;
1249	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1250
1251	/* shr Ev,8 */
1252	if (iGprDst >= 8)
1253	pbCodeBuf[off++] = X86_OP_REX_B;
1254	pbCodeBuf[off++] = 0xc1;
1255	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1256	pbCodeBuf[off++] = 8;
1257
1258	#elif defined(RT_ARCH_ARM64)
1259	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1260	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1261	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1262
1263	#else
1264	# error "port me"
1265	#endif
1266	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1267	return off;
1268	}
1269
1270
1271	/**
1272	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1273	*/
1274	DECL_INLINE_THROW(uint32_t)
1275	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1276	{
1277	#ifdef RT_ARCH_AMD64
1278	/* movsxd r64, r/m32 */
1279	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1280	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1281	pbCodeBuf[off++] = 0x63;
1282	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1283
1284	#elif defined(RT_ARCH_ARM64)
1285	/* sxtw dst, src */
1286	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1287	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1288
1289	#else
1290	# error "port me"
1291	#endif
1292	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1293	return off;
1294	}
1295
1296
1297	/**
1298	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1299	*/
1300	DECL_INLINE_THROW(uint32_t)
1301	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1302	{
1303	#ifdef RT_ARCH_AMD64
1304	/* movsx r64, r/m16 */
1305	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1306	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1307	pbCodeBuf[off++] = 0x0f;
1308	pbCodeBuf[off++] = 0xbf;
1309	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1310
1311	#elif defined(RT_ARCH_ARM64)
1312	/* sxth dst, src */
1313	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1314	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1315
1316	#else
1317	# error "port me"
1318	#endif
1319	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1320	return off;
1321	}
1322
1323
1324	/**
1325	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1326	*/
1327	DECL_INLINE_THROW(uint32_t)
1328	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1329	{
1330	#ifdef RT_ARCH_AMD64
1331	/* movsx r64, r/m16 */
1332	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1333	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1334	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1335	pbCodeBuf[off++] = 0x0f;
1336	pbCodeBuf[off++] = 0xbf;
1337	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1338
1339	#elif defined(RT_ARCH_ARM64)
1340	/* sxth dst32, src */
1341	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1342	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1343
1344	#else
1345	# error "port me"
1346	#endif
1347	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1348	return off;
1349	}
1350
1351
1352	/**
1353	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1354	*/
1355	DECL_INLINE_THROW(uint32_t)
1356	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1357	{
1358	#ifdef RT_ARCH_AMD64
1359	/* movsx r64, r/m8 */
1360	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1361	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1362	pbCodeBuf[off++] = 0x0f;
1363	pbCodeBuf[off++] = 0xbe;
1364	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1365
1366	#elif defined(RT_ARCH_ARM64)
1367	/* sxtb dst, src */
1368	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1369	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1370
1371	#else
1372	# error "port me"
1373	#endif
1374	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1375	return off;
1376	}
1377
1378
1379	/**
1380	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1381	* @note Bits 63 thru 32 are cleared.
1382	*/
1383	DECL_INLINE_THROW(uint32_t)
1384	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1385	{
1386	#ifdef RT_ARCH_AMD64
1387	/* movsx r32, r/m8 */
1388	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1389	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1390	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1391	else if (iGprSrc >= 4)
1392	pbCodeBuf[off++] = X86_OP_REX;
1393	pbCodeBuf[off++] = 0x0f;
1394	pbCodeBuf[off++] = 0xbe;
1395	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1396
1397	#elif defined(RT_ARCH_ARM64)
1398	/* sxtb dst32, src32 */
1399	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1400	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1401
1402	#else
1403	# error "port me"
1404	#endif
1405	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1406	return off;
1407	}
1408
1409
1410	/**
1411	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1412	* @note Bits 63 thru 16 are cleared.
1413	*/
1414	DECL_INLINE_THROW(uint32_t)
1415	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1416	{
1417	#ifdef RT_ARCH_AMD64
1418	/* movsx r16, r/m8 */
1419	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1420	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1421	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1422	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1423	else if (iGprSrc >= 4)
1424	pbCodeBuf[off++] = X86_OP_REX;
1425	pbCodeBuf[off++] = 0x0f;
1426	pbCodeBuf[off++] = 0xbe;
1427	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1428
1429	/* movzx r32, r/m16 */
1430	if (iGprDst >= 8)
1431	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1432	pbCodeBuf[off++] = 0x0f;
1433	pbCodeBuf[off++] = 0xb7;
1434	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1435
1436	#elif defined(RT_ARCH_ARM64)
1437	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1438	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1439	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1440	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1441	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1442
1443	#else
1444	# error "port me"
1445	#endif
1446	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1447	return off;
1448	}
1449
1450
1451	/**
1452	* Emits a gprdst = gprsrc + addend load.
1453	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1454	*/
1455	#ifdef RT_ARCH_AMD64
1456	DECL_INLINE_THROW(uint32_t)
1457	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1458	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1459	{
1460	Assert(iAddend != 0);
1461
1462	/* lea gprdst, [gprsrc + iAddend] */
1463	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1464	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1465	pbCodeBuf[off++] = 0x8d;
1466	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1467	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1468	return off;
1469	}
1470
1471	#elif defined(RT_ARCH_ARM64)
1472	DECL_INLINE_THROW(uint32_t)
1473	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1474	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1475	{
1476	if ((uint32_t)iAddend < 4096)
1477	{
1478	/* add dst, src, uimm12 */
1479	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1480	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1481	}
1482	else if ((uint32_t)-iAddend < 4096)
1483	{
1484	/* sub dst, src, uimm12 */
1485	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1486	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1487	}
1488	else
1489	{
1490	Assert(iGprSrc != iGprDst);
1491	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1492	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1493	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1494	}
1495	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1496	return off;
1497	}
1498	#else
1499	# error "port me"
1500	#endif
1501
1502	/**
1503	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1504	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1505	*/
1506	#ifdef RT_ARCH_AMD64
1507	DECL_INLINE_THROW(uint32_t)
1508	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1509	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1510	#else
1511	DECL_INLINE_THROW(uint32_t)
1512	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1513	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1514	#endif
1515	{
1516	if (iAddend != 0)
1517	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1518	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1519	}
1520
1521
1522	/**
1523	* Emits a gprdst = gprsrc32 + addend load.
1524	* @note Bits 63 thru 32 are cleared.
1525	*/
1526	DECL_INLINE_THROW(uint32_t)
1527	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1528	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1529	{
1530	Assert(iAddend != 0);
1531
1532	#ifdef RT_ARCH_AMD64
1533	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1534	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1535	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1536	if ((iGprDst \| iGprSrc) >= 8)
1537	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1538	pbCodeBuf[off++] = 0x8d;
1539	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1540
1541	#elif defined(RT_ARCH_ARM64)
1542	if ((uint32_t)iAddend < 4096)
1543	{
1544	/* add dst, src, uimm12 */
1545	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1546	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1547	}
1548	else if ((uint32_t)-iAddend < 4096)
1549	{
1550	/* sub dst, src, uimm12 */
1551	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1552	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1553	}
1554	else
1555	{
1556	Assert(iGprSrc != iGprDst);
1557	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1558	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1559	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1560	}
1561
1562	#else
1563	# error "port me"
1564	#endif
1565	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1566	return off;
1567	}
1568
1569
1570	/**
1571	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1572	*/
1573	DECL_INLINE_THROW(uint32_t)
1574	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1575	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1576	{
1577	if (iAddend != 0)
1578	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1579	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1580	}
1581
1582
1583	/**
1584	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1585	* destination.
1586	*/
1587	DECL_FORCE_INLINE(uint32_t)
1588	iemNativeEmitGprMergeInGpr16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1589	{
1590	#ifdef RT_ARCH_AMD64
1591	/* mov reg16, r/m16 */
1592	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1593	if (idxDst >= 8 \|\| idxSrc >= 8)
1594	pCodeBuf[off++] = (idxDst < 8 ? 0 : X86_OP_REX_R) \| (idxSrc < 8 ? 0 : X86_OP_REX_B);
1595	pCodeBuf[off++] = 0x8b;
1596	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxDst & 7, idxSrc & 7);
1597
1598	#elif defined(RT_ARCH_ARM64)
1599	/* bfi w1, w2, 0, 16 - moves bits 15:0 from idxSrc to idxDst bits 15:0. */
1600	pCodeBuf[off++] = Armv8A64MkInstrBfi(idxDst, idxSrc, 0, 16);
1601
1602	#else
1603	# error "Port me!"
1604	#endif
1605	return off;
1606	}
1607
1608
1609	/**
1610	* Emits a gprdst[15:0] = gprsrc[15:0], preserving all other bits in the
1611	* destination.
1612	*/
1613	DECL_INLINE_THROW(uint32_t)
1614	iemNativeEmitGprMergeInGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDst, uint8_t idxSrc)
1615	{
1616	#ifdef RT_ARCH_AMD64
1617	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, idxDst, idxSrc);
1618	#elif defined(RT_ARCH_ARM64)
1619	off = iemNativeEmitGprMergeInGpr16Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, idxDst, idxSrc);
1620	#else
1621	# error "Port me!"
1622	#endif
1623	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1624	return off;
1625	}
1626
1627
1628	#ifdef RT_ARCH_AMD64
1629	/**
1630	* Common bit of iemNativeEmitLoadGprByBp and friends.
1631	*/
1632	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1633	PIEMRECOMPILERSTATE pReNativeAssert)
1634	{
1635	if (offDisp < 128 && offDisp >= -128)
1636	{
1637	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1638	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1639	}
1640	else
1641	{
1642	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1643	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1644	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1645	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1646	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1647	}
1648	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1649	return off;
1650	}
1651	#elif defined(RT_ARCH_ARM64)
1652	/**
1653	* Common bit of iemNativeEmitLoadGprByBp and friends.
1654	*/
1655	DECL_FORCE_INLINE_THROW(uint32_t)
1656	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1657	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1658	{
1659	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1660	{
1661	/* str w/ unsigned imm12 (scaled) */
1662	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1663	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1664	}
1665	else if (offDisp >= -256 && offDisp <= 256)
1666	{
1667	/* stur w/ signed imm9 (unscaled) */
1668	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1669	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1670	}
1671	else
1672	{
1673	/* Use temporary indexing register. */
1674	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1675	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1676	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1677	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1678	}
1679	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1680	return off;
1681	}
1682	#endif
1683
1684
1685	/**
1686	* Emits a 64-bit GRP load instruction with an BP relative source address.
1687	*/
1688	DECL_INLINE_THROW(uint32_t)
1689	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1690	{
1691	#ifdef RT_ARCH_AMD64
1692	/* mov gprdst, qword [rbp + offDisp] */
1693	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1694	if (iGprDst < 8)
1695	pbCodeBuf[off++] = X86_OP_REX_W;
1696	else
1697	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1698	pbCodeBuf[off++] = 0x8b;
1699	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1700
1701	#elif defined(RT_ARCH_ARM64)
1702	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1703
1704	#else
1705	# error "port me"
1706	#endif
1707	}
1708
1709
1710	/**
1711	* Emits a 32-bit GRP load instruction with an BP relative source address.
1712	* @note Bits 63 thru 32 of the GPR will be cleared.
1713	*/
1714	DECL_INLINE_THROW(uint32_t)
1715	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1716	{
1717	#ifdef RT_ARCH_AMD64
1718	/* mov gprdst, dword [rbp + offDisp] */
1719	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1720	if (iGprDst >= 8)
1721	pbCodeBuf[off++] = X86_OP_REX_R;
1722	pbCodeBuf[off++] = 0x8b;
1723	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1724
1725	#elif defined(RT_ARCH_ARM64)
1726	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1727
1728	#else
1729	# error "port me"
1730	#endif
1731	}
1732
1733
1734	/**
1735	* Emits a 16-bit GRP load instruction with an BP relative source address.
1736	* @note Bits 63 thru 16 of the GPR will be cleared.
1737	*/
1738	DECL_INLINE_THROW(uint32_t)
1739	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1740	{
1741	#ifdef RT_ARCH_AMD64
1742	/* movzx gprdst, word [rbp + offDisp] */
1743	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1744	if (iGprDst >= 8)
1745	pbCodeBuf[off++] = X86_OP_REX_R;
1746	pbCodeBuf[off++] = 0x0f;
1747	pbCodeBuf[off++] = 0xb7;
1748	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1749
1750	#elif defined(RT_ARCH_ARM64)
1751	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1752
1753	#else
1754	# error "port me"
1755	#endif
1756	}
1757
1758
1759	/**
1760	* Emits a 8-bit GRP load instruction with an BP relative source address.
1761	* @note Bits 63 thru 8 of the GPR will be cleared.
1762	*/
1763	DECL_INLINE_THROW(uint32_t)
1764	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1765	{
1766	#ifdef RT_ARCH_AMD64
1767	/* movzx gprdst, byte [rbp + offDisp] */
1768	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1769	if (iGprDst >= 8)
1770	pbCodeBuf[off++] = X86_OP_REX_R;
1771	pbCodeBuf[off++] = 0x0f;
1772	pbCodeBuf[off++] = 0xb6;
1773	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1774
1775	#elif defined(RT_ARCH_ARM64)
1776	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1777
1778	#else
1779	# error "port me"
1780	#endif
1781	}
1782
1783
1784	/**
1785	* Emits a load effective address to a GRP with an BP relative source address.
1786	*/
1787	DECL_INLINE_THROW(uint32_t)
1788	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1789	{
1790	#ifdef RT_ARCH_AMD64
1791	/* lea gprdst, [rbp + offDisp] */
1792	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1793	if (iGprDst < 8)
1794	pbCodeBuf[off++] = X86_OP_REX_W;
1795	else
1796	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1797	pbCodeBuf[off++] = 0x8d;
1798	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1799
1800	#elif defined(RT_ARCH_ARM64)
1801	if ((uint32_t)offDisp < (unsigned)_4K)
1802	{
1803	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1804	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1805	}
1806	else if ((uint32_t)-offDisp < (unsigned)_4K)
1807	{
1808	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1809	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1810	}
1811	else
1812	{
1813	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1814	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1815	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1816	if (offDisp >= 0)
1817	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1818	else
1819	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1820	}
1821
1822	#else
1823	# error "port me"
1824	#endif
1825
1826	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1827	return off;
1828	}
1829
1830
1831	/**
1832	* Emits a 64-bit GPR store with an BP relative destination address.
1833	*
1834	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1835	*/
1836	DECL_INLINE_THROW(uint32_t)
1837	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1838	{
1839	#ifdef RT_ARCH_AMD64
1840	/* mov qword [rbp + offDisp], gprdst */
1841	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1842	if (iGprSrc < 8)
1843	pbCodeBuf[off++] = X86_OP_REX_W;
1844	else
1845	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1846	pbCodeBuf[off++] = 0x89;
1847	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1848
1849	#elif defined(RT_ARCH_ARM64)
1850	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1851	{
1852	/* str w/ unsigned imm12 (scaled) */
1853	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1854	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1855	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1856	}
1857	else if (offDisp >= -256 && offDisp <= 256)
1858	{
1859	/* stur w/ signed imm9 (unscaled) */
1860	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1861	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1862	}
1863	else if ((uint32_t)-offDisp < (unsigned)_4K)
1864	{
1865	/* Use temporary indexing register w/ sub uimm12. */
1866	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1867	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1868	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1869	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1870	}
1871	else
1872	{
1873	/* Use temporary indexing register. */
1874	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1875	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1876	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1877	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1878	}
1879	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1880	return off;
1881
1882	#else
1883	# error "Port me!"
1884	#endif
1885	}
1886
1887
1888	/**
1889	* Emits a 64-bit immediate store with an BP relative destination address.
1890	*
1891	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1892	*/
1893	DECL_INLINE_THROW(uint32_t)
1894	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1895	{
1896	#ifdef RT_ARCH_AMD64
1897	if ((int64_t)uImm64 == (int32_t)uImm64)
1898	{
1899	/* mov qword [rbp + offDisp], imm32 - sign extended */
1900	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1901	pbCodeBuf[off++] = X86_OP_REX_W;
1902	pbCodeBuf[off++] = 0xc7;
1903	if (offDisp < 128 && offDisp >= -128)
1904	{
1905	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1906	pbCodeBuf[off++] = (uint8_t)offDisp;
1907	}
1908	else
1909	{
1910	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1911	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1912	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1913	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1914	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1915	}
1916	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1917	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1918	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1919	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1920	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1921	return off;
1922	}
1923	#endif
1924
1925	/* Load tmp0, imm64; Store tmp to bp+disp. */
1926	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1927	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1928	}
1929
1930	#if defined(RT_ARCH_ARM64)
1931
1932	/**
1933	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1934	*
1935	* @note Odd and large @a offDisp values requires a temporary, unless it's a
1936	* load and @a iGprReg differs from @a iGprBase. Will assert / throw if
1937	* caller does not heed this.
1938	*
1939	* @note DON'T try this with prefetch.
1940	*/
1941	DECL_FORCE_INLINE_THROW(uint32_t)
1942	iemNativeEmitGprByGprLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp,
1943	ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
1944	{
1945	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1946	{
1947	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1948	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1949	}
1950	else if ( ( !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation)
1951	&& iGprReg != iGprBase)
1952	\|\| iGprTmp != UINT8_MAX)
1953	{
1954	/* The offset is too large, so we must load it into a register and use
1955	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1956	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1957	if (iGprTmp == UINT8_MAX)
1958	iGprTmp = iGprReg;
1959	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (int64_t)offDisp);
1960	pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, iGprTmp);
1961	}
1962	else
1963	# ifdef IEM_WITH_THROW_CATCH
1964	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
1965	# else
1966	AssertReleaseFailedStmt(off = UINT32_MAX);
1967	# endif
1968	return off;
1969	}
1970
1971	/**
1972	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1973	*/
1974	DECL_FORCE_INLINE_THROW(uint32_t)
1975	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1976	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1977	{
1978	/*
1979	* There are a couple of ldr variants that takes an immediate offset, so
1980	* try use those if we can, otherwise we have to use the temporary register
1981	* help with the addressing.
1982	*/
1983	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1984	{
1985	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1986	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1987	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1988	}
1989	else
1990	{
1991	/* The offset is too large, so we must load it into a register and use
1992	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1993	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1994	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (int64_t)offDisp);
1995
1996	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1997	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1998
1999	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
2000	}
2001	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2002	return off;
2003	}
2004
2005	#endif /* RT_ARCH_ARM64 */
2006
2007	/**
2008	* Emits a 64-bit GPR load via a GPR base address with a displacement.
2009	*
2010	* @note ARM64: Misaligned @a offDisp values and values not in the
2011	* -0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2012	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2013	* does not heed this.
2014	*/
2015	DECL_FORCE_INLINE_THROW(uint32_t)
2016	iemNativeEmitLoadGprByGprU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2017	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2018	{
2019	#ifdef RT_ARCH_AMD64
2020	/* mov reg64, mem64 */
2021	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2022	pCodeBuf[off++] = 0x8b;
2023	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2024	RT_NOREF(iGprTmp);
2025
2026	#elif defined(RT_ARCH_ARM64)
2027	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2028	kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t), iGprTmp);
2029
2030	#else
2031	# error "port me"
2032	#endif
2033	return off;
2034	}
2035
2036
2037	/**
2038	* Emits a 64-bit GPR load via a GPR base address with a displacement.
2039	*/
2040	DECL_INLINE_THROW(uint32_t)
2041	iemNativeEmitLoadGprByGprU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
2042	{
2043	#ifdef RT_ARCH_AMD64
2044	off = iemNativeEmitLoadGprByGprU64Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
2045	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2046
2047	#elif defined(RT_ARCH_ARM64)
2048	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
2049
2050	#else
2051	# error "port me"
2052	#endif
2053	return off;
2054	}
2055
2056
2057	/**
2058	* Emits a 32-bit GPR load via a GPR base address with a displacement.
2059	*
2060	* @note ARM64: Misaligned @a offDisp values and values not in the
2061	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
2062	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2063	* caller does not heed this.
2064	*
2065	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2066	*/
2067	DECL_FORCE_INLINE_THROW(uint32_t)
2068	iemNativeEmitLoadGprByGprU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2069	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2070	{
2071	#ifdef RT_ARCH_AMD64
2072	/* mov reg32, mem32 */
2073	if (iGprDst >= 8 \|\| iGprBase >= 8)
2074	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2075	pCodeBuf[off++] = 0x8b;
2076	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2077	RT_NOREF(iGprTmp);
2078
2079	#elif defined(RT_ARCH_ARM64)
2080	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2081	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t), iGprTmp);
2082
2083	#else
2084	# error "port me"
2085	#endif
2086	return off;
2087	}
2088
2089
2090	/**
2091	* Emits a 32-bit GPR load via a GPR base address with a displacement.
2092	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2093	*/
2094	DECL_INLINE_THROW(uint32_t)
2095	iemNativeEmitLoadGprByGprU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
2096	{
2097	#ifdef RT_ARCH_AMD64
2098	off = iemNativeEmitLoadGprByGprU32Ex(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
2099	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2100
2101	#elif defined(RT_ARCH_ARM64)
2102	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
2103
2104	#else
2105	# error "port me"
2106	#endif
2107	return off;
2108	}
2109
2110
2111	/**
2112	* Emits a 32-bit GPR load via a GPR base address with a displacement,
2113	* sign-extending the value to 64 bits.
2114	*
2115	* @note ARM64: Misaligned @a offDisp values and values not in the
2116	* -0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp)
2117	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2118	* caller does not heed this.
2119	*/
2120	DECL_FORCE_INLINE_THROW(uint32_t)
2121	iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2122	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2123	{
2124	#ifdef RT_ARCH_AMD64
2125	/* movsxd reg64, mem32 */
2126	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2127	pCodeBuf[off++] = 0x63;
2128	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2129	RT_NOREF(iGprTmp);
2130
2131	#elif defined(RT_ARCH_ARM64)
2132	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2133	kArmv8A64InstrLdStType_Ld_SignWord64, sizeof(uint32_t), iGprTmp);
2134
2135	#else
2136	# error "port me"
2137	#endif
2138	return off;
2139	}
2140
2141
2142	/**
2143	* Emits a 16-bit GPR load via a GPR base address with a displacement.
2144	*
2145	* @note ARM64: Misaligned @a offDisp values and values not in the
2146	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2147	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2148	* caller does not heed this.
2149	*
2150	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2151	*/
2152	DECL_FORCE_INLINE_THROW(uint32_t)
2153	iemNativeEmitLoadGprByGprU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2154	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2155	{
2156	#ifdef RT_ARCH_AMD64
2157	/* movzx reg32, mem16 */
2158	if (iGprDst >= 8 \|\| iGprBase >= 8)
2159	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2160	pCodeBuf[off++] = 0x0f;
2161	pCodeBuf[off++] = 0xb7;
2162	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2163	RT_NOREF(iGprTmp);
2164
2165	#elif defined(RT_ARCH_ARM64)
2166	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2167	kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t), iGprTmp);
2168
2169	#else
2170	# error "port me"
2171	#endif
2172	return off;
2173	}
2174
2175
2176	/**
2177	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2178	* sign-extending the value to 64 bits.
2179	*
2180	* @note ARM64: Misaligned @a offDisp values and values not in the
2181	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2182	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2183	* caller does not heed this.
2184	*/
2185	DECL_FORCE_INLINE_THROW(uint32_t)
2186	iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2187	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2188	{
2189	#ifdef RT_ARCH_AMD64
2190	/* movsx reg64, mem16 */
2191	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2192	pCodeBuf[off++] = 0x0f;
2193	pCodeBuf[off++] = 0xbf;
2194	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2195	RT_NOREF(iGprTmp);
2196
2197	#elif defined(RT_ARCH_ARM64)
2198	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2199	kArmv8A64InstrLdStType_Ld_SignHalf64, sizeof(uint16_t), iGprTmp);
2200
2201	#else
2202	# error "port me"
2203	#endif
2204	return off;
2205	}
2206
2207
2208	/**
2209	* Emits a 16-bit GPR load via a GPR base address with a displacement,
2210	* sign-extending the value to 32 bits.
2211	*
2212	* @note ARM64: Misaligned @a offDisp values and values not in the
2213	* -0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp)
2214	* if @a iGprReg and @a iGprBase are the same. Will assert / throw if
2215	* caller does not heed this.
2216	*
2217	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2218	*/
2219	DECL_FORCE_INLINE_THROW(uint32_t)
2220	iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2221	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2222	{
2223	#ifdef RT_ARCH_AMD64
2224	/* movsx reg32, mem16 */
2225	if (iGprDst >= 8 \|\| iGprBase >= 8)
2226	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2227	pCodeBuf[off++] = 0x0f;
2228	pCodeBuf[off++] = 0xbf;
2229	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2230	RT_NOREF(iGprTmp);
2231
2232	#elif defined(RT_ARCH_ARM64)
2233	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2234	kArmv8A64InstrLdStType_Ld_SignHalf32, sizeof(uint16_t), iGprTmp);
2235
2236	#else
2237	# error "port me"
2238	#endif
2239	return off;
2240	}
2241
2242
2243	/**
2244	* Emits a 8-bit GPR load via a GPR base address with a displacement.
2245	*
2246	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2247	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2248	* same. Will assert / throw if caller does not heed this.
2249	*
2250	* @note Bits 63 thru 8 in @a iGprDst will be cleared.
2251	*/
2252	DECL_FORCE_INLINE_THROW(uint32_t)
2253	iemNativeEmitLoadGprByGprU8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2254	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2255	{
2256	#ifdef RT_ARCH_AMD64
2257	/* movzx reg32, mem8 */
2258	if (iGprDst >= 8 \|\| iGprBase >= 8)
2259	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2260	pCodeBuf[off++] = 0x0f;
2261	pCodeBuf[off++] = 0xb6;
2262	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2263	RT_NOREF(iGprTmp);
2264
2265	#elif defined(RT_ARCH_ARM64)
2266	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2267	kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t), iGprTmp);
2268
2269	#else
2270	# error "port me"
2271	#endif
2272	return off;
2273	}
2274
2275
2276	/**
2277	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2278	* sign-extending the value to 64 bits.
2279	*
2280	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2281	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2282	* same. Will assert / throw if caller does not heed this.
2283	*/
2284	DECL_FORCE_INLINE_THROW(uint32_t)
2285	iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2286	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2287	{
2288	#ifdef RT_ARCH_AMD64
2289	/* movsx reg64, mem8 */
2290	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2291	pCodeBuf[off++] = 0x0f;
2292	pCodeBuf[off++] = 0xbe;
2293	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2294	RT_NOREF(iGprTmp);
2295
2296	#elif defined(RT_ARCH_ARM64)
2297	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2298	kArmv8A64InstrLdStType_Ld_SignByte64, sizeof(uint8_t), iGprTmp);
2299
2300	#else
2301	# error "port me"
2302	#endif
2303	return off;
2304	}
2305
2306
2307	/**
2308	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2309	* sign-extending the value to 32 bits.
2310	*
2311	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2312	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2313	* same. Will assert / throw if caller does not heed this.
2314	*
2315	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
2316	*/
2317	DECL_FORCE_INLINE_THROW(uint32_t)
2318	iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2319	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2320	{
2321	#ifdef RT_ARCH_AMD64
2322	/* movsx reg32, mem8 */
2323	if (iGprDst >= 8 \|\| iGprBase >= 8)
2324	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2325	pCodeBuf[off++] = 0x0f;
2326	pCodeBuf[off++] = 0xbe;
2327	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2328	RT_NOREF(iGprTmp);
2329
2330	#elif defined(RT_ARCH_ARM64)
2331	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2332	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2333
2334	#else
2335	# error "port me"
2336	#endif
2337	return off;
2338	}
2339
2340
2341	/**
2342	* Emits a 8-bit GPR load via a GPR base address with a displacement,
2343	* sign-extending the value to 16 bits.
2344	*
2345	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2346	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2347	* same. Will assert / throw if caller does not heed this.
2348	*
2349	* @note Bits 63 thru 16 in @a iGprDst will be cleared.
2350	*/
2351	DECL_FORCE_INLINE_THROW(uint32_t)
2352	iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
2353	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2354	{
2355	#ifdef RT_ARCH_AMD64
2356	/* movsx reg32, mem8 */
2357	if (iGprDst >= 8 \|\| iGprBase >= 8)
2358	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2359	pCodeBuf[off++] = 0x0f;
2360	pCodeBuf[off++] = 0xbe;
2361	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
2362	# if 1 /** @todo use 'movzx reg32, reg16' instead of 'and reg32, 0ffffh' ? */
2363	/* and reg32, 0xffffh */
2364	if (iGprDst >= 8)
2365	pCodeBuf[off++] = X86_OP_REX_B;
2366	pCodeBuf[off++] = 0x81;
2367	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2368	pCodeBuf[off++] = 0xff;
2369	pCodeBuf[off++] = 0xff;
2370	pCodeBuf[off++] = 0;
2371	pCodeBuf[off++] = 0;
2372	# else
2373	/* movzx reg32, reg16 */
2374	if (iGprDst >= 8)
2375	pCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2376	pCodeBuf[off++] = 0x0f;
2377	pCodeBuf[off++] = 0xb7;
2378	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2379	# endif
2380	RT_NOREF(iGprTmp);
2381
2382	#elif defined(RT_ARCH_ARM64)
2383	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
2384	kArmv8A64InstrLdStType_Ld_SignByte32, sizeof(uint8_t), iGprTmp);
2385	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
2386	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /64Bit/);
2387
2388	#else
2389	# error "port me"
2390	#endif
2391	return off;
2392	}
2393
2394
2395	/**
2396	* Emits a 64-bit GPR store via a GPR base address with a displacement.
2397	*
2398	* @note ARM64: Misaligned @a offDisp values and values not in the
2399	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2400	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2401	* does not heed this.
2402	*/
2403	DECL_FORCE_INLINE_THROW(uint32_t)
2404	iemNativeEmitStoreGpr64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2405	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2406	{
2407	#ifdef RT_ARCH_AMD64
2408	/* mov mem64, reg64 */
2409	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2410	pCodeBuf[off++] = 0x89;
2411	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2412	RT_NOREF(iGprTmp);
2413
2414	#elif defined(RT_ARCH_ARM64)
2415	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2416	kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
2417
2418	#else
2419	# error "port me"
2420	#endif
2421	return off;
2422	}
2423
2424
2425	/**
2426	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2427	*
2428	* @note ARM64: Misaligned @a offDisp values and values not in the
2429	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2430	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2431	* does not heed this.
2432	*/
2433	DECL_FORCE_INLINE_THROW(uint32_t)
2434	iemNativeEmitStoreGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2435	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2436	{
2437	#ifdef RT_ARCH_AMD64
2438	/* mov mem32, reg32 */
2439	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2440	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2441	pCodeBuf[off++] = 0x89;
2442	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2443	RT_NOREF(iGprTmp);
2444
2445	#elif defined(RT_ARCH_ARM64)
2446	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2447	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2448
2449	#else
2450	# error "port me"
2451	#endif
2452	return off;
2453	}
2454
2455
2456	/**
2457	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2458	*
2459	* @note ARM64: Misaligned @a offDisp values and values not in the
2460	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2461	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2462	* does not heed this.
2463	*/
2464	DECL_FORCE_INLINE_THROW(uint32_t)
2465	iemNativeEmitStoreGpr16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2466	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2467	{
2468	#ifdef RT_ARCH_AMD64
2469	/* mov mem16, reg16 */
2470	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2471	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2472	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2473	pCodeBuf[off++] = 0x89;
2474	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2475	RT_NOREF(iGprTmp);
2476
2477	#elif defined(RT_ARCH_ARM64)
2478	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2479	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2480
2481	#else
2482	# error "port me"
2483	#endif
2484	return off;
2485	}
2486
2487
2488	/**
2489	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2490	*
2491	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2492	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2493	* same. Will assert / throw if caller does not heed this.
2494	*/
2495	DECL_FORCE_INLINE_THROW(uint32_t)
2496	iemNativeEmitStoreGpr8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iGprBase,
2497	int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2498	{
2499	#ifdef RT_ARCH_AMD64
2500	/* mov mem8, reg8 */
2501	if (iGprSrc >= 8 \|\| iGprBase >= 8)
2502	pCodeBuf[off++] = (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2503	else if (iGprSrc >= 4)
2504	pCodeBuf[off++] = X86_OP_REX;
2505	pCodeBuf[off++] = 0x88;
2506	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprSrc, iGprBase, offDisp);
2507	RT_NOREF(iGprTmp);
2508
2509	#elif defined(RT_ARCH_ARM64)
2510	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprSrc, iGprBase, offDisp,
2511	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2512
2513	#else
2514	# error "port me"
2515	#endif
2516	return off;
2517	}
2518
2519
2520	/**
2521	* Emits a 64-bit immediate store via a GPR base address with a displacement.
2522	*
2523	* @note This will always require @a iGprTmpImm on ARM (except for uImm=0), on
2524	* AMD64 it depends on the immediate value.
2525	*
2526	* @note ARM64: Misaligned @a offDisp values and values not in the
2527	* 0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
2528	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2529	* does not heed this.
2530	*/
2531	DECL_FORCE_INLINE_THROW(uint32_t)
2532	iemNativeEmitStoreImm64ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint64_t uImm, uint8_t iGprBase,
2533	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2534	{
2535	#ifdef RT_ARCH_AMD64
2536	if ((int32_t)uImm == (int64_t)uImm)
2537	{
2538	/* mov mem64, imm32 (sign-extended) */
2539	pCodeBuf[off++] = X86_OP_REX_W \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
2540	pCodeBuf[off++] = 0xc7;
2541	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2542	pCodeBuf[off++] = RT_BYTE1(uImm);
2543	pCodeBuf[off++] = RT_BYTE2(uImm);
2544	pCodeBuf[off++] = RT_BYTE3(uImm);
2545	pCodeBuf[off++] = RT_BYTE4(uImm);
2546	}
2547	else if (iGprImmTmp != UINT8_MAX \|\| iGprTmp != UINT8_MAX)
2548	{
2549	/* require temporary register. */
2550	if (iGprImmTmp == UINT8_MAX)
2551	iGprImmTmp = iGprTmp;
2552	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2553	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp);
2554	}
2555	else
2556	# ifdef IEM_WITH_THROW_CATCH
2557	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2558	# else
2559	AssertReleaseFailedStmt(off = UINT32_MAX);
2560	# endif
2561
2562	#elif defined(RT_ARCH_ARM64)
2563	if (uImm == 0)
2564	iGprImmTmp = ARMV8_A64_REG_XZR;
2565	else
2566	{
2567	Assert(iGprImmTmp < 31);
2568	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2569	}
2570	off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp, iGprTmp);
2571
2572	#else
2573	# error "port me"
2574	#endif
2575	return off;
2576	}
2577
2578
2579	/**
2580	* Emits a 32-bit GPR store via a GPR base address with a displacement.
2581	*
2582	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2583	*
2584	* @note ARM64: Misaligned @a offDisp values and values not in the
2585	* 0x3ffc...0x3ffc range will require a temporary register (@a iGprTmp) if
2586	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2587	* does not heed this.
2588	*/
2589	DECL_FORCE_INLINE_THROW(uint32_t)
2590	iemNativeEmitStoreImm32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uImm, uint8_t iGprBase,
2591	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2592	{
2593	#ifdef RT_ARCH_AMD64
2594	/* mov mem32, imm32 */
2595	if (iGprBase >= 8)
2596	pCodeBuf[off++] = X86_OP_REX_B;
2597	pCodeBuf[off++] = 0xc7;
2598	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2599	pCodeBuf[off++] = RT_BYTE1(uImm);
2600	pCodeBuf[off++] = RT_BYTE2(uImm);
2601	pCodeBuf[off++] = RT_BYTE3(uImm);
2602	pCodeBuf[off++] = RT_BYTE4(uImm);
2603	RT_NOREF(iGprImmTmp, iGprTmp);
2604
2605	#elif defined(RT_ARCH_ARM64)
2606	Assert(iGprImmTmp < 31);
2607	if (uImm == 0)
2608	iGprImmTmp = ARMV8_A64_REG_XZR;
2609	else
2610	{
2611	Assert(iGprImmTmp < 31);
2612	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprImmTmp, uImm);
2613	}
2614	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2615	kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t), iGprTmp);
2616
2617	#else
2618	# error "port me"
2619	#endif
2620	return off;
2621	}
2622
2623
2624	/**
2625	* Emits a 16-bit GPR store via a GPR base address with a displacement.
2626	*
2627	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2628	*
2629	* @note ARM64: Misaligned @a offDisp values and values not in the
2630	* 0x1ffe...0x1ffe range will require a temporary register (@a iGprTmp) if
2631	* @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
2632	* does not heed this.
2633	*/
2634	DECL_FORCE_INLINE_THROW(uint32_t)
2635	iemNativeEmitStoreImm16ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint8_t iGprBase,
2636	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2637	{
2638	#ifdef RT_ARCH_AMD64
2639	/* mov mem16, imm16 */
2640	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2641	if (iGprBase >= 8)
2642	pCodeBuf[off++] = X86_OP_REX_B;
2643	pCodeBuf[off++] = 0xc7;
2644	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2645	pCodeBuf[off++] = RT_BYTE1(uImm);
2646	pCodeBuf[off++] = RT_BYTE2(uImm);
2647	RT_NOREF(iGprImmTmp, iGprTmp);
2648
2649	#elif defined(RT_ARCH_ARM64)
2650	if (uImm == 0)
2651	iGprImmTmp = ARMV8_A64_REG_XZR;
2652	else
2653	{
2654	Assert(iGprImmTmp < 31);
2655	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2656	}
2657	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2658	kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t), iGprTmp);
2659
2660	#else
2661	# error "port me"
2662	#endif
2663	return off;
2664	}
2665
2666
2667	/**
2668	* Emits a 8-bit GPR store via a GPR base address with a displacement.
2669	*
2670	* @note This will always require @a iGprTmpImm on ARM64 (except for uImm=0).
2671	*
2672	* @note ARM64: @a offDisp values not in the 0xfff...0xfff range will require a
2673	* temporary register (@a iGprTmp) if @a iGprReg and @a iGprBase are the
2674	* same. Will assert / throw if caller does not heed this.
2675	*/
2676	DECL_FORCE_INLINE_THROW(uint32_t)
2677	iemNativeEmitStoreImm8ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t uImm, uint8_t iGprBase,
2678	uint8_t iGprImmTmp = UINT8_MAX, int32_t offDisp = 0, uint8_t iGprTmp = UINT8_MAX)
2679	{
2680	#ifdef RT_ARCH_AMD64
2681	/* mov mem8, imm8 */
2682	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
2683	if (iGprBase >= 8)
2684	pCodeBuf[off++] = X86_OP_REX_B;
2685	pCodeBuf[off++] = 0xc6;
2686	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, 0, iGprBase, offDisp);
2687	pCodeBuf[off++] = uImm;
2688	RT_NOREF(iGprImmTmp, iGprTmp);
2689
2690	#elif defined(RT_ARCH_ARM64)
2691	if (uImm == 0)
2692	iGprImmTmp = ARMV8_A64_REG_XZR;
2693	else
2694	{
2695	Assert(iGprImmTmp < 31);
2696	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGprImmTmp, uImm);
2697	}
2698	off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprImmTmp, iGprBase, offDisp,
2699	kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t), iGprTmp);
2700
2701	#else
2702	# error "port me"
2703	#endif
2704	return off;
2705	}
2706
2707
2708
2709	/*********************************************************************************************************************************
2710	* Subtraction and Additions *
2711	*********************************************************************************************************************************/
2712
2713	/**
2714	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
2715	* @note The AMD64 version sets flags.
2716	*/
2717	DECL_INLINE_THROW(uint32_t)
2718	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2719	{
2720	#if defined(RT_ARCH_AMD64)
2721	/* sub Gv,Ev */
2722	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2723	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2724	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2725	pbCodeBuf[off++] = 0x2b;
2726	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2727
2728	#elif defined(RT_ARCH_ARM64)
2729	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2730	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
2731
2732	#else
2733	# error "Port me"
2734	#endif
2735	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2736	return off;
2737	}
2738
2739
2740	/**
2741	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2742	* @note The AMD64 version sets flags.
2743	*/
2744	DECL_FORCE_INLINE(uint32_t)
2745	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2746	{
2747	#if defined(RT_ARCH_AMD64)
2748	/* sub Gv,Ev */
2749	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
2750	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
2751	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
2752	pCodeBuf[off++] = 0x2b;
2753	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
2754
2755	#elif defined(RT_ARCH_ARM64)
2756	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
2757
2758	#else
2759	# error "Port me"
2760	#endif
2761	return off;
2762	}
2763
2764
2765	/**
2766	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
2767	* @note The AMD64 version sets flags.
2768	*/
2769	DECL_INLINE_THROW(uint32_t)
2770	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
2771	{
2772	#if defined(RT_ARCH_AMD64)
2773	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
2774	#elif defined(RT_ARCH_ARM64)
2775	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
2776	#else
2777	# error "Port me"
2778	#endif
2779	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2780	return off;
2781	}
2782
2783
2784	/**
2785	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2786	*
2787	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2788	*
2789	* @note Larger constants will require a temporary register. Failing to specify
2790	* one when needed will trigger fatal assertion / throw.
2791	*/
2792	DECL_FORCE_INLINE_THROW(uint32_t)
2793	iemNativeEmitSubGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2794	uint8_t iGprTmp = UINT8_MAX)
2795	{
2796	#ifdef RT_ARCH_AMD64
2797	pCodeBuf[off++] = iGprDst >= 8 ? X86_OP_REX_W \| X86_OP_REX_B : X86_OP_REX_W;
2798	if (iSubtrahend == 1)
2799	{
2800	/* dec r/m64 */
2801	pCodeBuf[off++] = 0xff;
2802	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2803	}
2804	else if (iSubtrahend == -1)
2805	{
2806	/* inc r/m64 */
2807	pCodeBuf[off++] = 0xff;
2808	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2809	}
2810	else if ((int8_t)iSubtrahend == iSubtrahend)
2811	{
2812	/* sub r/m64, imm8 */
2813	pCodeBuf[off++] = 0x83;
2814	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2815	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2816	}
2817	else if ((int32_t)iSubtrahend == iSubtrahend)
2818	{
2819	/* sub r/m64, imm32 */
2820	pCodeBuf[off++] = 0x81;
2821	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2822	pCodeBuf[off++] = RT_BYTE1((uint64_t)iSubtrahend);
2823	pCodeBuf[off++] = RT_BYTE2((uint64_t)iSubtrahend);
2824	pCodeBuf[off++] = RT_BYTE3((uint64_t)iSubtrahend);
2825	pCodeBuf[off++] = RT_BYTE4((uint64_t)iSubtrahend);
2826	}
2827	else if (iGprTmp != UINT8_MAX)
2828	{
2829	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off - 1, iGprTmp, (uint64_t)iSubtrahend);
2830	/* sub r/m64, r64 */
2831	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B) \| (iGprTmp < 8 ? 0 : X86_OP_REX_R);
2832	pCodeBuf[off++] = 0x29;
2833	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprTmp & 7, iGprDst & 7);
2834	}
2835	else
2836	# ifdef IEM_WITH_THROW_CATCH
2837	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2838	# else
2839	AssertReleaseFailedStmt(off = UINT32_MAX);
2840	# endif
2841
2842	#elif defined(RT_ARCH_ARM64)
2843	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2844	if (uAbsSubtrahend < 4096)
2845	{
2846	if (iSubtrahend >= 0)
2847	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2848	else
2849	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend);
2850	}
2851	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2852	{
2853	if (iSubtrahend >= 0)
2854	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2855	true /f64Bit/, false /fSetFlags/, true /fShift/);
2856	else
2857	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2858	true /f64Bit/, false /fSetFlags/, true /fShift/);
2859	}
2860	else if (iGprTmp != UINT8_MAX)
2861	{
2862	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (uint64_t)iSubtrahend);
2863	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp);
2864	}
2865	else
2866	# ifdef IEM_WITH_THROW_CATCH
2867	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2868	# else
2869	AssertReleaseFailedStmt(off = UINT32_MAX);
2870	# endif
2871
2872	#else
2873	# error "Port me"
2874	#endif
2875	return off;
2876	}
2877
2878
2879	/**
2880	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
2881	*
2882	* @note Larger constants will require a temporary register. Failing to specify
2883	* one when needed will trigger fatal assertion / throw.
2884	*/
2885	DECL_INLINE_THROW(uint32_t)
2886	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iSubtrahend,
2887	uint8_t iGprTmp = UINT8_MAX)
2888
2889	{
2890	#ifdef RT_ARCH_AMD64
2891	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 13), off, iGprDst, iSubtrahend, iGprTmp);
2892	#elif defined(RT_ARCH_ARM64)
2893	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGprDst, iSubtrahend, iGprTmp);
2894	#else
2895	# error "Port me"
2896	#endif
2897	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2898	return off;
2899	}
2900
2901
2902	/**
2903	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2904	*
2905	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
2906	*
2907	* @note ARM64: Larger constants will require a temporary register. Failing to
2908	* specify one when needed will trigger fatal assertion / throw.
2909	*/
2910	DECL_FORCE_INLINE_THROW(uint32_t)
2911	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2912	uint8_t iGprTmp = UINT8_MAX)
2913	{
2914	#ifdef RT_ARCH_AMD64
2915	if (iGprDst >= 8)
2916	pCodeBuf[off++] = X86_OP_REX_B;
2917	if (iSubtrahend == 1)
2918	{
2919	/* dec r/m32 */
2920	pCodeBuf[off++] = 0xff;
2921	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
2922	}
2923	else if (iSubtrahend == -1)
2924	{
2925	/* inc r/m32 */
2926	pCodeBuf[off++] = 0xff;
2927	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2928	}
2929	else if (iSubtrahend < 128 && iSubtrahend >= -128)
2930	{
2931	/* sub r/m32, imm8 */
2932	pCodeBuf[off++] = 0x83;
2933	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2934	pCodeBuf[off++] = (uint8_t)iSubtrahend;
2935	}
2936	else
2937	{
2938	/* sub r/m32, imm32 */
2939	pCodeBuf[off++] = 0x81;
2940	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2941	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
2942	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
2943	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
2944	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
2945	}
2946	RT_NOREF(iGprTmp);
2947
2948	#elif defined(RT_ARCH_ARM64)
2949	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
2950	if (uAbsSubtrahend < 4096)
2951	{
2952	if (iSubtrahend >= 0)
2953	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2954	else
2955	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
2956	}
2957	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
2958	{
2959	if (iSubtrahend >= 0)
2960	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2961	false /f64Bit/, false /fSetFlags/, true /fShift/);
2962	else
2963	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
2964	false /f64Bit/, false /fSetFlags/, true /fShift/);
2965	}
2966	else if (iGprTmp != UINT8_MAX)
2967	{
2968	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
2969	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
2970	}
2971	else
2972	# ifdef IEM_WITH_THROW_CATCH
2973	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2974	# else
2975	AssertReleaseFailedStmt(off = UINT32_MAX);
2976	# endif
2977
2978	#else
2979	# error "Port me"
2980	#endif
2981	return off;
2982	}
2983
2984
2985	/**
2986	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
2987	*
2988	* @note ARM64: Larger constants will require a temporary register. Failing to
2989	* specify one when needed will trigger fatal assertion / throw.
2990	*/
2991	DECL_INLINE_THROW(uint32_t)
2992	iemNativeEmitSubGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
2993	uint8_t iGprTmp = UINT8_MAX)
2994
2995	{
2996	#ifdef RT_ARCH_AMD64
2997	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iSubtrahend, iGprTmp);
2998	#elif defined(RT_ARCH_ARM64)
2999	off = iemNativeEmitSubGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iSubtrahend, iGprTmp);
3000	#else
3001	# error "Port me"
3002	#endif
3003	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3004	return off;
3005	}
3006
3007
3008	/**
3009	* Emits a 16-bit GPR subtract with a signed immediate subtrahend.
3010	*
3011	* This will optimize using DEC/INC/whatever and ARM64 will not set flags,
3012	* so not suitable as a base for conditional jumps.
3013	*
3014	* @note AMD64: Will only update the lower 16 bits of the register.
3015	* @note ARM64: Will update the entire register.
3016	* @note ARM64: Larger constants will require a temporary register. Failing to
3017	* specify one when needed will trigger fatal assertion / throw.
3018	*/
3019	DECL_FORCE_INLINE_THROW(uint32_t)
3020	iemNativeEmitSubGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iSubtrahend,
3021	uint8_t iGprTmp = UINT8_MAX)
3022	{
3023	#ifdef RT_ARCH_AMD64
3024	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
3025	if (iGprDst >= 8)
3026	pCodeBuf[off++] = X86_OP_REX_B;
3027	if (iSubtrahend == 1)
3028	{
3029	/* dec r/m16 */
3030	pCodeBuf[off++] = 0xff;
3031	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
3032	}
3033	else if (iSubtrahend == -1)
3034	{
3035	/* inc r/m16 */
3036	pCodeBuf[off++] = 0xff;
3037	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3038	}
3039	else if ((int8_t)iSubtrahend == iSubtrahend)
3040	{
3041	/* sub r/m16, imm8 */
3042	pCodeBuf[off++] = 0x83;
3043	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
3044	pCodeBuf[off++] = (uint8_t)iSubtrahend;
3045	}
3046	else
3047	{
3048	/* sub r/m16, imm16 */
3049	pCodeBuf[off++] = 0x81;
3050	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
3051	pCodeBuf[off++] = RT_BYTE1((uint16_t)iSubtrahend);
3052	pCodeBuf[off++] = RT_BYTE2((uint16_t)iSubtrahend);
3053	}
3054	RT_NOREF(iGprTmp);
3055
3056	#elif defined(RT_ARCH_ARM64)
3057	uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
3058	if (uAbsSubtrahend < 4096)
3059	{
3060	if (iSubtrahend >= 0)
3061	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
3062	else
3063	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /f64Bit/);
3064	}
3065	else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
3066	{
3067	if (iSubtrahend >= 0)
3068	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
3069	false /f64Bit/, false /fSetFlags/, true /fShift/);
3070	else
3071	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
3072	false /f64Bit/, false /fSetFlags/, true /fShift/);
3073	}
3074	else if (iGprTmp != UINT8_MAX)
3075	{
3076	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
3077	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
3078	}
3079	else
3080	# ifdef IEM_WITH_THROW_CATCH
3081	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3082	# else
3083	AssertReleaseFailedStmt(off = UINT32_MAX);
3084	# endif
3085	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
3086
3087	#else
3088	# error "Port me"
3089	#endif
3090	return off;
3091	}
3092
3093
3094	/**
3095	* Emits adding a 64-bit GPR to another, storing the result in the first.
3096	* @note The AMD64 version sets flags.
3097	*/
3098	DECL_FORCE_INLINE(uint32_t)
3099	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3100	{
3101	#if defined(RT_ARCH_AMD64)
3102	/* add Gv,Ev */
3103	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3104	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
3105	pCodeBuf[off++] = 0x03;
3106	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
3107
3108	#elif defined(RT_ARCH_ARM64)
3109	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
3110
3111	#else
3112	# error "Port me"
3113	#endif
3114	return off;
3115	}
3116
3117
3118	/**
3119	* Emits adding a 64-bit GPR to another, storing the result in the first.
3120	* @note The AMD64 version sets flags.
3121	*/
3122	DECL_INLINE_THROW(uint32_t)
3123	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3124	{
3125	#if defined(RT_ARCH_AMD64)
3126	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
3127	#elif defined(RT_ARCH_ARM64)
3128	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
3129	#else
3130	# error "Port me"
3131	#endif
3132	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3133	return off;
3134	}
3135
3136
3137	/**
3138	* Emits adding a 64-bit GPR to another, storing the result in the first.
3139	* @note The AMD64 version sets flags.
3140	*/
3141	DECL_FORCE_INLINE(uint32_t)
3142	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3143	{
3144	#if defined(RT_ARCH_AMD64)
3145	/* add Gv,Ev */
3146	if (iGprDst >= 8 \|\| iGprAddend >= 8)
3147	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
3148	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
3149	pCodeBuf[off++] = 0x03;
3150	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
3151
3152	#elif defined(RT_ARCH_ARM64)
3153	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
3154
3155	#else
3156	# error "Port me"
3157	#endif
3158	return off;
3159	}
3160
3161
3162	/**
3163	* Emits adding a 64-bit GPR to another, storing the result in the first.
3164	* @note The AMD64 version sets flags.
3165	*/
3166	DECL_INLINE_THROW(uint32_t)
3167	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
3168	{
3169	#if defined(RT_ARCH_AMD64)
3170	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
3171	#elif defined(RT_ARCH_ARM64)
3172	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
3173	#else
3174	# error "Port me"
3175	#endif
3176	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3177	return off;
3178	}
3179
3180
3181	/**
3182	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3183	*/
3184	DECL_INLINE_THROW(uint32_t)
3185	iemNativeEmitAddGprImm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3186	{
3187	#if defined(RT_ARCH_AMD64)
3188	/* add or inc */
3189	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3190	if (iImm8 != 1)
3191	{
3192	pCodeBuf[off++] = 0x83;
3193	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3194	pCodeBuf[off++] = (uint8_t)iImm8;
3195	}
3196	else
3197	{
3198	pCodeBuf[off++] = 0xff;
3199	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3200	}
3201
3202	#elif defined(RT_ARCH_ARM64)
3203	if (iImm8 >= 0)
3204	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint8_t)iImm8);
3205	else
3206	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint8_t)-iImm8);
3207
3208	#else
3209	# error "Port me"
3210	#endif
3211	return off;
3212	}
3213
3214
3215	/**
3216	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
3217	*/
3218	DECL_INLINE_THROW(uint32_t)
3219	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3220	{
3221	#if defined(RT_ARCH_AMD64)
3222	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3223	#elif defined(RT_ARCH_ARM64)
3224	off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3225	#else
3226	# error "Port me"
3227	#endif
3228	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3229	return off;
3230	}
3231
3232
3233	/**
3234	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3235	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3236	*/
3237	DECL_FORCE_INLINE(uint32_t)
3238	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3239	{
3240	#if defined(RT_ARCH_AMD64)
3241	/* add or inc */
3242	if (iGprDst >= 8)
3243	pCodeBuf[off++] = X86_OP_REX_B;
3244	if (iImm8 != 1)
3245	{
3246	pCodeBuf[off++] = 0x83;
3247	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3248	pCodeBuf[off++] = (uint8_t)iImm8;
3249	}
3250	else
3251	{
3252	pCodeBuf[off++] = 0xff;
3253	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3254	}
3255
3256	#elif defined(RT_ARCH_ARM64)
3257	if (iImm8 >= 0)
3258	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
3259	else
3260	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
3261
3262	#else
3263	# error "Port me"
3264	#endif
3265	return off;
3266	}
3267
3268
3269	/**
3270	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
3271	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3272	*/
3273	DECL_INLINE_THROW(uint32_t)
3274	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
3275	{
3276	#if defined(RT_ARCH_AMD64)
3277	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
3278	#elif defined(RT_ARCH_ARM64)
3279	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
3280	#else
3281	# error "Port me"
3282	#endif
3283	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3284	return off;
3285	}
3286
3287
3288	/**
3289	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3290	*
3291	* @note Will assert / throw if @a iGprTmp is not specified when needed.
3292	*/
3293	DECL_FORCE_INLINE_THROW(uint32_t)
3294	iemNativeEmitAddGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iAddend, uint8_t iGprTmp = UINT8_MAX)
3295	{
3296	#if defined(RT_ARCH_AMD64)
3297	if ((int8_t)iAddend == iAddend)
3298	return iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3299
3300	if ((int32_t)iAddend == iAddend)
3301	{
3302	/* add grp, imm32 */
3303	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3304	pCodeBuf[off++] = 0x81;
3305	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3306	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3307	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3308	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3309	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3310	}
3311	else if (iGprTmp != UINT8_MAX)
3312	{
3313	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3314
3315	/* add dst, tmpreg */
3316	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3317	\| (iGprTmp < 8 ? 0 : X86_OP_REX_B);
3318	pCodeBuf[off++] = 0x03;
3319	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprTmp & 7);
3320	}
3321	else
3322	# ifdef IEM_WITH_THROW_CATCH
3323	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3324	# else
3325	AssertReleaseFailedStmt(off = UINT32_MAX);
3326	# endif
3327
3328	#elif defined(RT_ARCH_ARM64)
3329	uint64_t const uAbsAddend = (uint64_t)RT_ABS(iAddend);
3330	if (uAbsAddend < 4096)
3331	{
3332	if (iAddend >= 0)
3333	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3334	else
3335	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
3336	}
3337	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3338	{
3339	if (iAddend >= 0)
3340	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3341	true /f64Bit/, true /fShift12/);
3342	else
3343	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
3344	true /f64Bit/, true /fShift12/);
3345	}
3346	else if (iGprTmp != UINT8_MAX)
3347	{
3348	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
3349	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp);
3350	}
3351	else
3352	# ifdef IEM_WITH_THROW_CATCH
3353	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3354	# else
3355	AssertReleaseFailedStmt(off = UINT32_MAX);
3356	# endif
3357
3358	#else
3359	# error "Port me"
3360	#endif
3361	return off;
3362	}
3363
3364
3365	/**
3366	* Emits a 64-bit GPR additions with a 64-bit signed addend.
3367	*/
3368	DECL_INLINE_THROW(uint32_t)
3369	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
3370	{
3371	#if defined(RT_ARCH_AMD64)
3372	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3373	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
3374
3375	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
3376	{
3377	/* add grp, imm32 */
3378	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3379	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3380	pbCodeBuf[off++] = 0x81;
3381	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3382	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3383	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3384	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3385	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3386	}
3387	else
3388	{
3389	/* Best to use a temporary register to deal with this in the simplest way: */
3390	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3391
3392	/* add dst, tmpreg */
3393	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3394	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
3395	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3396	pbCodeBuf[off++] = 0x03;
3397	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
3398
3399	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3400	}
3401
3402	#elif defined(RT_ARCH_ARM64)
3403	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3404	{
3405	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3406	if (iAddend >= 0)
3407	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
3408	else
3409	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
3410	}
3411	else
3412	{
3413	/* Use temporary register for the immediate. */
3414	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
3415
3416	/* add gprdst, gprdst, tmpreg */
3417	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3418	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
3419
3420	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3421	}
3422
3423	#else
3424	# error "Port me"
3425	#endif
3426	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3427	return off;
3428	}
3429
3430
3431	/**
3432	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3433	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3434	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
3435	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
3436	* the lower 12 bits always zero). The negative ranges are also allowed,
3437	* making it behave like a subtraction. If the constant does not conform,
3438	* bad stuff will happen.
3439	*/
3440	DECL_FORCE_INLINE_THROW(uint32_t)
3441	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3442	{
3443	#if defined(RT_ARCH_AMD64)
3444	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
3445	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
3446
3447	/* add grp, imm32 */
3448	if (iGprDst >= 8)
3449	pCodeBuf[off++] = X86_OP_REX_B;
3450	pCodeBuf[off++] = 0x81;
3451	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3452	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
3453	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
3454	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
3455	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
3456
3457	#elif defined(RT_ARCH_ARM64)
3458	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
3459	if (uAbsAddend <= 0xfff)
3460	{
3461	if (iAddend >= 0)
3462	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3463	else
3464	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3465	}
3466	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3467	{
3468	if (iAddend >= 0)
3469	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3470	false /f64Bit/, true /fShift12/);
3471	else
3472	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
3473	false /f64Bit/, true /fShift12/);
3474	}
3475	else
3476	# ifdef IEM_WITH_THROW_CATCH
3477	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3478	# else
3479	AssertReleaseFailedStmt(off = UINT32_MAX);
3480	# endif
3481
3482	#else
3483	# error "Port me"
3484	#endif
3485	return off;
3486	}
3487
3488
3489	/**
3490	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
3491	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
3492	*/
3493	DECL_INLINE_THROW(uint32_t)
3494	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
3495	{
3496	#if defined(RT_ARCH_AMD64)
3497	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
3498
3499	#elif defined(RT_ARCH_ARM64)
3500	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
3501	{
3502	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3503	if (iAddend >= 0)
3504	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
3505	else
3506	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
3507	}
3508	else
3509	{
3510	/* Use temporary register for the immediate. */
3511	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
3512
3513	/* add gprdst, gprdst, tmpreg */
3514	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3515	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
3516
3517	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3518	}
3519
3520	#else
3521	# error "Port me"
3522	#endif
3523	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3524	return off;
3525	}
3526
3527
3528	/**
3529	* Emits a 16-bit GPR add with a signed immediate addend.
3530	*
3531	* This will optimize using INC/DEC/whatever and ARM64 will not set flags,
3532	* so not suitable as a base for conditional jumps.
3533	*
3534	* @note AMD64: Will only update the lower 16 bits of the register.
3535	* @note ARM64: Will update the entire register.
3536	* @note ARM64: Larger constants will require a temporary register. Failing to
3537	* specify one when needed will trigger fatal assertion / throw.
3538	* @sa iemNativeEmitSubGpr16ImmEx
3539	*/
3540	DECL_FORCE_INLINE_THROW(uint32_t)
3541	iemNativeEmitAddGpr16ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int16_t iAddend,
3542	uint8_t iGprTmp = UINT8_MAX)
3543	{
3544	#ifdef RT_ARCH_AMD64
3545	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
3546	if (iGprDst >= 8)
3547	pCodeBuf[off++] = X86_OP_REX_B;
3548	if (iAddend == 1)
3549	{
3550	/* inc r/m16 */
3551	pCodeBuf[off++] = 0xff;
3552	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3553	}
3554	else if (iAddend == -1)
3555	{
3556	/* dec r/m16 */
3557	pCodeBuf[off++] = 0xff;
3558	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
3559	}
3560	else if ((int8_t)iAddend == iAddend)
3561	{
3562	/* add r/m16, imm8 */
3563	pCodeBuf[off++] = 0x83;
3564	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3565	pCodeBuf[off++] = (uint8_t)iAddend;
3566	}
3567	else
3568	{
3569	/* add r/m16, imm16 */
3570	pCodeBuf[off++] = 0x81;
3571	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
3572	pCodeBuf[off++] = RT_BYTE1((uint16_t)iAddend);
3573	pCodeBuf[off++] = RT_BYTE2((uint16_t)iAddend);
3574	}
3575	RT_NOREF(iGprTmp);
3576
3577	#elif defined(RT_ARCH_ARM64)
3578	uint32_t uAbsAddend = RT_ABS(iAddend);
3579	if (uAbsAddend < 4096)
3580	{
3581	if (iAddend >= 0)
3582	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3583	else
3584	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
3585	}
3586	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
3587	{
3588	if (iAddend >= 0)
3589	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3590	false /f64Bit/, false /fSetFlags/, true /fShift/);
3591	else
3592	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsAddend >> 12,
3593	false /f64Bit/, false /fSetFlags/, true /fShift/);
3594	}
3595	else if (iGprTmp != UINT8_MAX)
3596	{
3597	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iAddend);
3598	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp, false /f64Bit/);
3599	}
3600	else
3601	# ifdef IEM_WITH_THROW_CATCH
3602	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3603	# else
3604	AssertReleaseFailedStmt(off = UINT32_MAX);
3605	# endif
3606	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
3607
3608	#else
3609	# error "Port me"
3610	#endif
3611	return off;
3612	}
3613
3614
3615
3616	/**
3617	* Adds two 64-bit GPRs together, storing the result in a third register.
3618	*/
3619	DECL_FORCE_INLINE(uint32_t)
3620	iemNativeEmitGprEqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3621	{
3622	#ifdef RT_ARCH_AMD64
3623	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3624	{
3625	/** @todo consider LEA */
3626	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend1);
3627	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend2);
3628	}
3629	else
3630	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3631
3632	#elif defined(RT_ARCH_ARM64)
3633	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2);
3634
3635	#else
3636	# error "Port me!"
3637	#endif
3638	return off;
3639	}
3640
3641
3642
3643	/**
3644	* Adds two 32-bit GPRs together, storing the result in a third register.
3645	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3646	*/
3647	DECL_FORCE_INLINE(uint32_t)
3648	iemNativeEmitGpr32EqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
3649	{
3650	#ifdef RT_ARCH_AMD64
3651	if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
3652	{
3653	/** @todo consider LEA */
3654	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend1);
3655	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend2);
3656	}
3657	else
3658	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprDst != iGprAddend1 ? iGprAddend1 : iGprAddend2);
3659
3660	#elif defined(RT_ARCH_ARM64)
3661	pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2, false /f64Bit/);
3662
3663	#else
3664	# error "Port me!"
3665	#endif
3666	return off;
3667	}
3668
3669
3670	/**
3671	* Adds a 64-bit GPR and a 64-bit unsigned constant, storing the result in a
3672	* third register.
3673	*
3674	* @note The ARM64 version does not work for non-trivial constants if the
3675	* two registers are the same. Will assert / throw exception.
3676	*/
3677	DECL_FORCE_INLINE_THROW(uint32_t)
3678	iemNativeEmitGprEqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int64_t iImmAddend)
3679	{
3680	#ifdef RT_ARCH_AMD64
3681	/** @todo consider LEA */
3682	if ((int8_t)iImmAddend == iImmAddend)
3683	{
3684	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend);
3685	off = iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3686	}
3687	else
3688	{
3689	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3690	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3691	}
3692
3693	#elif defined(RT_ARCH_ARM64)
3694	uint64_t const uAbsImmAddend = RT_ABS(iImmAddend);
3695	if (uAbsImmAddend < 4096)
3696	{
3697	if (iImmAddend >= 0)
3698	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3699	else
3700	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend);
3701	}
3702	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3703	{
3704	if (iImmAddend >= 0)
3705	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3706	else
3707	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /f64Bit/, true /fShift12/);
3708	}
3709	else if (iGprDst != iGprAddend)
3710	{
3711	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, (uint64_t)iImmAddend);
3712	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3713	}
3714	else
3715	# ifdef IEM_WITH_THROW_CATCH
3716	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3717	# else
3718	AssertReleaseFailedStmt(off = UINT32_MAX);
3719	# endif
3720
3721	#else
3722	# error "Port me!"
3723	#endif
3724	return off;
3725	}
3726
3727
3728	/**
3729	* Adds a 32-bit GPR and a 32-bit unsigned constant, storing the result in a
3730	* third register.
3731	*
3732	* @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
3733	*
3734	* @note The ARM64 version does not work for non-trivial constants if the
3735	* two registers are the same. Will assert / throw exception.
3736	*/
3737	DECL_FORCE_INLINE_THROW(uint32_t)
3738	iemNativeEmitGpr32EqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
3739	{
3740	#ifdef RT_ARCH_AMD64
3741	/** @todo consider LEA */
3742	if ((int8_t)iImmAddend == iImmAddend)
3743	{
3744	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3745	off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
3746	}
3747	else
3748	{
3749	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, iImmAddend);
3750	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
3751	}
3752
3753	#elif defined(RT_ARCH_ARM64)
3754	uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
3755	if (uAbsImmAddend < 4096)
3756	{
3757	if (iImmAddend >= 0)
3758	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3759	else
3760	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /f64Bit/);
3761	}
3762	else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
3763	{
3764	if (iImmAddend >= 0)
3765	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3766	else
3767	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /f64Bit/, true /fShift12/);
3768	}
3769	else if (iGprDst != iGprAddend)
3770	{
3771	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
3772	off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend);
3773	}
3774	else
3775	# ifdef IEM_WITH_THROW_CATCH
3776	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3777	# else
3778	AssertReleaseFailedStmt(off = UINT32_MAX);
3779	# endif
3780
3781	#else
3782	# error "Port me!"
3783	#endif
3784	return off;
3785	}
3786
3787
3788	/*********************************************************************************************************************************
3789	* Unary Operations *
3790	*********************************************************************************************************************************/
3791
3792	/**
3793	* Emits code for two complement negation of a 64-bit GPR.
3794	*/
3795	DECL_FORCE_INLINE_THROW(uint32_t)
3796	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3797	{
3798	#if defined(RT_ARCH_AMD64)
3799	/* neg Ev */
3800	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
3801	pCodeBuf[off++] = 0xf7;
3802	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3803
3804	#elif defined(RT_ARCH_ARM64)
3805	/* sub dst, xzr, dst */
3806	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
3807
3808	#else
3809	# error "Port me"
3810	#endif
3811	return off;
3812	}
3813
3814
3815	/**
3816	* Emits code for two complement negation of a 64-bit GPR.
3817	*/
3818	DECL_INLINE_THROW(uint32_t)
3819	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3820	{
3821	#if defined(RT_ARCH_AMD64)
3822	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3823	#elif defined(RT_ARCH_ARM64)
3824	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3825	#else
3826	# error "Port me"
3827	#endif
3828	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3829	return off;
3830	}
3831
3832
3833	/**
3834	* Emits code for two complement negation of a 32-bit GPR.
3835	* @note bit 32 thru 63 are set to zero.
3836	*/
3837	DECL_FORCE_INLINE_THROW(uint32_t)
3838	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
3839	{
3840	#if defined(RT_ARCH_AMD64)
3841	/* neg Ev */
3842	if (iGprDst >= 8)
3843	pCodeBuf[off++] = X86_OP_REX_B;
3844	pCodeBuf[off++] = 0xf7;
3845	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
3846
3847	#elif defined(RT_ARCH_ARM64)
3848	/* sub dst, xzr, dst */
3849	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
3850
3851	#else
3852	# error "Port me"
3853	#endif
3854	return off;
3855	}
3856
3857
3858	/**
3859	* Emits code for two complement negation of a 32-bit GPR.
3860	* @note bit 32 thru 63 are set to zero.
3861	*/
3862	DECL_INLINE_THROW(uint32_t)
3863	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3864	{
3865	#if defined(RT_ARCH_AMD64)
3866	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
3867	#elif defined(RT_ARCH_ARM64)
3868	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
3869	#else
3870	# error "Port me"
3871	#endif
3872	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3873	return off;
3874	}
3875
3876
3877
3878	/*********************************************************************************************************************************
3879	* Bit Operations *
3880	*********************************************************************************************************************************/
3881
3882	/**
3883	* Emits code for clearing bits 16 thru 63 in the GPR.
3884	*/
3885	DECL_INLINE_THROW(uint32_t)
3886	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
3887	{
3888	#if defined(RT_ARCH_AMD64)
3889	/* movzx Gv,Ew */
3890	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3891	if (iGprDst >= 8)
3892	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
3893	pbCodeBuf[off++] = 0x0f;
3894	pbCodeBuf[off++] = 0xb7;
3895	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
3896
3897	#elif defined(RT_ARCH_ARM64)
3898	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3899	# if 1
3900	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
3901	# else
3902	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
3903	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
3904	# endif
3905	#else
3906	# error "Port me"
3907	#endif
3908	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3909	return off;
3910	}
3911
3912
3913	/**
3914	* Emits code for AND'ing two 64-bit GPRs.
3915	*
3916	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3917	* and ARM64 hosts.
3918	*/
3919	DECL_FORCE_INLINE(uint32_t)
3920	iemNativeEmitAndGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3921	{
3922	#if defined(RT_ARCH_AMD64)
3923	/* and Gv, Ev */
3924	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3925	pCodeBuf[off++] = 0x23;
3926	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3927	RT_NOREF(fSetFlags);
3928
3929	#elif defined(RT_ARCH_ARM64)
3930	if (!fSetFlags)
3931	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
3932	else
3933	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
3934
3935	#else
3936	# error "Port me"
3937	#endif
3938	return off;
3939	}
3940
3941
3942	/**
3943	* Emits code for AND'ing two 64-bit GPRs.
3944	*
3945	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
3946	* and ARM64 hosts.
3947	*/
3948	DECL_INLINE_THROW(uint32_t)
3949	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3950	{
3951	#if defined(RT_ARCH_AMD64)
3952	off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
3953	#elif defined(RT_ARCH_ARM64)
3954	off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
3955	#else
3956	# error "Port me"
3957	#endif
3958	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3959	return off;
3960	}
3961
3962
3963	/**
3964	* Emits code for AND'ing two 32-bit GPRs.
3965	*/
3966	DECL_FORCE_INLINE(uint32_t)
3967	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3968	{
3969	#if defined(RT_ARCH_AMD64)
3970	/* and Gv, Ev */
3971	if (iGprDst >= 8 \|\| iGprSrc >= 8)
3972	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3973	pCodeBuf[off++] = 0x23;
3974	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
3975	RT_NOREF(fSetFlags);
3976
3977	#elif defined(RT_ARCH_ARM64)
3978	if (!fSetFlags)
3979	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3980	else
3981	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
3982
3983	#else
3984	# error "Port me"
3985	#endif
3986	return off;
3987	}
3988
3989
3990	/**
3991	* Emits code for AND'ing two 32-bit GPRs.
3992	*/
3993	DECL_INLINE_THROW(uint32_t)
3994	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
3995	{
3996	#if defined(RT_ARCH_AMD64)
3997	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
3998	#elif defined(RT_ARCH_ARM64)
3999	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
4000	#else
4001	# error "Port me"
4002	#endif
4003	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4004	return off;
4005	}
4006
4007
4008	/**
4009	* Emits code for AND'ing a 64-bit GPRs with a constant.
4010	*
4011	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
4012	* and ARM64 hosts.
4013	*/
4014	DECL_INLINE_THROW(uint32_t)
4015	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
4016	{
4017	#if defined(RT_ARCH_AMD64)
4018	if ((int64_t)uImm == (int8_t)uImm)
4019	{
4020	/* and Ev, imm8 */
4021	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
4022	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
4023	pbCodeBuf[off++] = 0x83;
4024	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4025	pbCodeBuf[off++] = (uint8_t)uImm;
4026	}
4027	else if ((int64_t)uImm == (int32_t)uImm)
4028	{
4029	/* and Ev, imm32 */
4030	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
4031	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
4032	pbCodeBuf[off++] = 0x81;
4033	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4034	pbCodeBuf[off++] = RT_BYTE1(uImm);
4035	pbCodeBuf[off++] = RT_BYTE2(uImm);
4036	pbCodeBuf[off++] = RT_BYTE3(uImm);
4037	pbCodeBuf[off++] = RT_BYTE4(uImm);
4038	}
4039	else
4040	{
4041	/* Use temporary register for the 64-bit immediate. */
4042	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4043	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
4044	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4045	}
4046	RT_NOREF(fSetFlags);
4047
4048	#elif defined(RT_ARCH_ARM64)
4049	uint32_t uImmR = 0;
4050	uint32_t uImmNandS = 0;
4051	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
4052	{
4053	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4054	if (!fSetFlags)
4055	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
4056	else
4057	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
4058	}
4059	else
4060	{
4061	/* Use temporary register for the 64-bit immediate. */
4062	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4063	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
4064	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4065	}
4066
4067	#else
4068	# error "Port me"
4069	#endif
4070	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4071	return off;
4072	}
4073
4074
4075	/**
4076	* Emits code for AND'ing an 32-bit GPRs with a constant.
4077	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4078	* @note For ARM64 this only supports @a uImm values that can be expressed using
4079	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
4080	* make sure this is possible!
4081	*/
4082	DECL_FORCE_INLINE_THROW(uint32_t)
4083	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
4084	{
4085	#if defined(RT_ARCH_AMD64)
4086	/* and Ev, imm */
4087	if (iGprDst >= 8)
4088	pCodeBuf[off++] = X86_OP_REX_B;
4089	if ((int32_t)uImm == (int8_t)uImm)
4090	{
4091	pCodeBuf[off++] = 0x83;
4092	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4093	pCodeBuf[off++] = (uint8_t)uImm;
4094	}
4095	else
4096	{
4097	pCodeBuf[off++] = 0x81;
4098	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4099	pCodeBuf[off++] = RT_BYTE1(uImm);
4100	pCodeBuf[off++] = RT_BYTE2(uImm);
4101	pCodeBuf[off++] = RT_BYTE3(uImm);
4102	pCodeBuf[off++] = RT_BYTE4(uImm);
4103	}
4104	RT_NOREF(fSetFlags);
4105
4106	#elif defined(RT_ARCH_ARM64)
4107	uint32_t uImmR = 0;
4108	uint32_t uImmNandS = 0;
4109	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4110	{
4111	if (!fSetFlags)
4112	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4113	else
4114	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4115	}
4116	else
4117	# ifdef IEM_WITH_THROW_CATCH
4118	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4119	# else
4120	AssertReleaseFailedStmt(off = UINT32_MAX);
4121	# endif
4122
4123	#else
4124	# error "Port me"
4125	#endif
4126	return off;
4127	}
4128
4129
4130	/**
4131	* Emits code for AND'ing an 32-bit GPRs with a constant.
4132	*
4133	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4134	*/
4135	DECL_INLINE_THROW(uint32_t)
4136	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
4137	{
4138	#if defined(RT_ARCH_AMD64)
4139	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
4140
4141	#elif defined(RT_ARCH_ARM64)
4142	uint32_t uImmR = 0;
4143	uint32_t uImmNandS = 0;
4144	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4145	{
4146	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4147	if (!fSetFlags)
4148	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4149	else
4150	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4151	}
4152	else
4153	{
4154	/* Use temporary register for the 64-bit immediate. */
4155	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4156	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
4157	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4158	}
4159
4160	#else
4161	# error "Port me"
4162	#endif
4163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4164	return off;
4165	}
4166
4167
4168	/**
4169	* Emits code for AND'ing an 64-bit GPRs with a constant.
4170	*
4171	* @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
4172	* encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
4173	* the same.
4174	*/
4175	DECL_FORCE_INLINE_THROW(uint32_t)
4176	iemNativeEmitGprEqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint64_t uImm,
4177	bool fSetFlags = false)
4178	{
4179	#if defined(RT_ARCH_AMD64)
4180	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
4181	off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc);
4182	RT_NOREF(fSetFlags);
4183
4184	#elif defined(RT_ARCH_ARM64)
4185	uint32_t uImmR = 0;
4186	uint32_t uImmNandS = 0;
4187	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
4188	{
4189	if (!fSetFlags)
4190	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR);
4191	else
4192	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR);
4193	}
4194	else if (iGprDst != iGprSrc)
4195	{
4196	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
4197	off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
4198	}
4199	else
4200	# ifdef IEM_WITH_THROW_CATCH
4201	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4202	# else
4203	AssertReleaseFailedStmt(off = UINT32_MAX);
4204	# endif
4205
4206	#else
4207	# error "Port me"
4208	#endif
4209	return off;
4210	}
4211
4212	/**
4213	* Emits code for AND'ing an 32-bit GPRs with a constant.
4214	*
4215	* @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
4216	* encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
4217	* the same.
4218	*
4219	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4220	*/
4221	DECL_FORCE_INLINE_THROW(uint32_t)
4222	iemNativeEmitGpr32EqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint32_t uImm,
4223	bool fSetFlags = false)
4224	{
4225	#if defined(RT_ARCH_AMD64)
4226	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4227	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4228	RT_NOREF(fSetFlags);
4229
4230	#elif defined(RT_ARCH_ARM64)
4231	uint32_t uImmR = 0;
4232	uint32_t uImmNandS = 0;
4233	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4234	{
4235	if (!fSetFlags)
4236	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4237	else
4238	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4239	}
4240	else if (iGprDst != iGprSrc)
4241	{
4242	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
4243	off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
4244	}
4245	else
4246	# ifdef IEM_WITH_THROW_CATCH
4247	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4248	# else
4249	AssertReleaseFailedStmt(off = UINT32_MAX);
4250	# endif
4251
4252	#else
4253	# error "Port me"
4254	#endif
4255	return off;
4256	}
4257
4258
4259	/**
4260	* Emits code for OR'ing two 64-bit GPRs.
4261	*/
4262	DECL_FORCE_INLINE(uint32_t)
4263	iemNativeEmitOrGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4264	{
4265	#if defined(RT_ARCH_AMD64)
4266	/* or Gv, Ev */
4267	pCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4268	pCodeBuf[off++] = 0x0b;
4269	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4270
4271	#elif defined(RT_ARCH_ARM64)
4272	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc);
4273
4274	#else
4275	# error "Port me"
4276	#endif
4277	return off;
4278	}
4279
4280
4281	/**
4282	* Emits code for OR'ing two 32-bit GPRs.
4283	* @note Bits 63:32 of the destination GPR will be cleared.
4284	*/
4285	DECL_FORCE_INLINE(uint32_t)
4286	iemNativeEmitOrGpr32ByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4287	{
4288	#if defined(RT_ARCH_AMD64)
4289	/* or Gv, Ev */
4290	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4291	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4292	pCodeBuf[off++] = 0x0b;
4293	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4294
4295	#elif defined(RT_ARCH_ARM64)
4296	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4297
4298	#else
4299	# error "Port me"
4300	#endif
4301	return off;
4302	}
4303
4304
4305	/**
4306	* Emits code for OR'ing two 32-bit GPRs.
4307	* @note Bits 63:32 of the destination GPR will be cleared.
4308	*/
4309	DECL_INLINE_THROW(uint32_t)
4310	iemNativeEmitOrGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4311	{
4312	#if defined(RT_ARCH_AMD64)
4313	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
4314	#elif defined(RT_ARCH_ARM64)
4315	off = iemNativeEmitOrGpr32ByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
4316	#else
4317	# error "Port me"
4318	#endif
4319	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4320	return off;
4321	}
4322
4323
4324
4325	/**
4326	* Emits code for XOR'ing two 64-bit GPRs.
4327	*/
4328	DECL_INLINE_THROW(uint32_t)
4329	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4330	{
4331	#if defined(RT_ARCH_AMD64)
4332	/* and Gv, Ev */
4333	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4334	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4335	pbCodeBuf[off++] = 0x33;
4336	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4337
4338	#elif defined(RT_ARCH_ARM64)
4339	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4340	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
4341
4342	#else
4343	# error "Port me"
4344	#endif
4345	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4346	return off;
4347	}
4348
4349
4350	/**
4351	* Emits code for XOR'ing two 32-bit GPRs.
4352	*/
4353	DECL_INLINE_THROW(uint32_t)
4354	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
4355	{
4356	#if defined(RT_ARCH_AMD64)
4357	/* and Gv, Ev */
4358	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4359	if (iGprDst >= 8 \|\| iGprSrc >= 8)
4360	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
4361	pbCodeBuf[off++] = 0x33;
4362	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
4363
4364	#elif defined(RT_ARCH_ARM64)
4365	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4366	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
4367
4368	#else
4369	# error "Port me"
4370	#endif
4371	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4372	return off;
4373	}
4374
4375
4376	/**
4377	* Emits code for XOR'ing an 32-bit GPRs with a constant.
4378	* @note Bits 32 thru 63 in the destination will be zero after the operation.
4379	* @note For ARM64 this only supports @a uImm values that can be expressed using
4380	* the two 6-bit immediates of the EOR instructions. The caller must make
4381	* sure this is possible!
4382	*/
4383	DECL_FORCE_INLINE_THROW(uint32_t)
4384	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
4385	{
4386	#if defined(RT_ARCH_AMD64)
4387	/* and Ev, imm */
4388	if (iGprDst >= 8)
4389	pCodeBuf[off++] = X86_OP_REX_B;
4390	if ((int32_t)uImm == (int8_t)uImm)
4391	{
4392	pCodeBuf[off++] = 0x83;
4393	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4394	pCodeBuf[off++] = (uint8_t)uImm;
4395	}
4396	else
4397	{
4398	pCodeBuf[off++] = 0x81;
4399	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
4400	pCodeBuf[off++] = RT_BYTE1(uImm);
4401	pCodeBuf[off++] = RT_BYTE2(uImm);
4402	pCodeBuf[off++] = RT_BYTE3(uImm);
4403	pCodeBuf[off++] = RT_BYTE4(uImm);
4404	}
4405
4406	#elif defined(RT_ARCH_ARM64)
4407	uint32_t uImmR = 0;
4408	uint32_t uImmNandS = 0;
4409	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
4410	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
4411	else
4412	# ifdef IEM_WITH_THROW_CATCH
4413	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4414	# else
4415	AssertReleaseFailedStmt(off = UINT32_MAX);
4416	# endif
4417
4418	#else
4419	# error "Port me"
4420	#endif
4421	return off;
4422	}
4423
4424
4425	/*********************************************************************************************************************************
4426	* Shifting *
4427	*********************************************************************************************************************************/
4428
4429	/**
4430	* Emits code for shifting a GPR a fixed number of bits to the left.
4431	*/
4432	DECL_FORCE_INLINE(uint32_t)
4433	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4434	{
4435	Assert(cShift > 0 && cShift < 64);
4436
4437	#if defined(RT_ARCH_AMD64)
4438	/* shl dst, cShift */
4439	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4440	if (cShift != 1)
4441	{
4442	pCodeBuf[off++] = 0xc1;
4443	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4444	pCodeBuf[off++] = cShift;
4445	}
4446	else
4447	{
4448	pCodeBuf[off++] = 0xd1;
4449	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4450	}
4451
4452	#elif defined(RT_ARCH_ARM64)
4453	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
4454
4455	#else
4456	# error "Port me"
4457	#endif
4458	return off;
4459	}
4460
4461
4462	/**
4463	* Emits code for shifting a GPR a fixed number of bits to the left.
4464	*/
4465	DECL_INLINE_THROW(uint32_t)
4466	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4467	{
4468	#if defined(RT_ARCH_AMD64)
4469	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4470	#elif defined(RT_ARCH_ARM64)
4471	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4472	#else
4473	# error "Port me"
4474	#endif
4475	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4476	return off;
4477	}
4478
4479
4480	/**
4481	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4482	*/
4483	DECL_FORCE_INLINE(uint32_t)
4484	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4485	{
4486	Assert(cShift > 0 && cShift < 32);
4487
4488	#if defined(RT_ARCH_AMD64)
4489	/* shl dst, cShift */
4490	if (iGprDst >= 8)
4491	pCodeBuf[off++] = X86_OP_REX_B;
4492	if (cShift != 1)
4493	{
4494	pCodeBuf[off++] = 0xc1;
4495	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4496	pCodeBuf[off++] = cShift;
4497	}
4498	else
4499	{
4500	pCodeBuf[off++] = 0xd1;
4501	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
4502	}
4503
4504	#elif defined(RT_ARCH_ARM64)
4505	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
4506
4507	#else
4508	# error "Port me"
4509	#endif
4510	return off;
4511	}
4512
4513
4514	/**
4515	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
4516	*/
4517	DECL_INLINE_THROW(uint32_t)
4518	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4519	{
4520	#if defined(RT_ARCH_AMD64)
4521	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4522	#elif defined(RT_ARCH_ARM64)
4523	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4524	#else
4525	# error "Port me"
4526	#endif
4527	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4528	return off;
4529	}
4530
4531
4532	/**
4533	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4534	*/
4535	DECL_FORCE_INLINE(uint32_t)
4536	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4537	{
4538	Assert(cShift > 0 && cShift < 64);
4539
4540	#if defined(RT_ARCH_AMD64)
4541	/* shr dst, cShift */
4542	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4543	if (cShift != 1)
4544	{
4545	pCodeBuf[off++] = 0xc1;
4546	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4547	pCodeBuf[off++] = cShift;
4548	}
4549	else
4550	{
4551	pCodeBuf[off++] = 0xd1;
4552	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4553	}
4554
4555	#elif defined(RT_ARCH_ARM64)
4556	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
4557
4558	#else
4559	# error "Port me"
4560	#endif
4561	return off;
4562	}
4563
4564
4565	/**
4566	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
4567	*/
4568	DECL_INLINE_THROW(uint32_t)
4569	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4570	{
4571	#if defined(RT_ARCH_AMD64)
4572	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4573	#elif defined(RT_ARCH_ARM64)
4574	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4575	#else
4576	# error "Port me"
4577	#endif
4578	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4579	return off;
4580	}
4581
4582
4583	/**
4584	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4585	* right.
4586	*/
4587	DECL_FORCE_INLINE(uint32_t)
4588	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4589	{
4590	Assert(cShift > 0 && cShift < 32);
4591
4592	#if defined(RT_ARCH_AMD64)
4593	/* shr dst, cShift */
4594	if (iGprDst >= 8)
4595	pCodeBuf[off++] = X86_OP_REX_B;
4596	if (cShift != 1)
4597	{
4598	pCodeBuf[off++] = 0xc1;
4599	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4600	pCodeBuf[off++] = cShift;
4601	}
4602	else
4603	{
4604	pCodeBuf[off++] = 0xd1;
4605	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
4606	}
4607
4608	#elif defined(RT_ARCH_ARM64)
4609	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
4610
4611	#else
4612	# error "Port me"
4613	#endif
4614	return off;
4615	}
4616
4617
4618	/**
4619	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4620	* right.
4621	*/
4622	DECL_INLINE_THROW(uint32_t)
4623	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4624	{
4625	#if defined(RT_ARCH_AMD64)
4626	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
4627	#elif defined(RT_ARCH_ARM64)
4628	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
4629	#else
4630	# error "Port me"
4631	#endif
4632	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4633	return off;
4634	}
4635
4636
4637	/**
4638	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
4639	* right and assigning it to a different GPR.
4640	*/
4641	DECL_INLINE_THROW(uint32_t)
4642	iemNativeEmitGpr32EqGprShiftRightImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint8_t cShift)
4643	{
4644	Assert(cShift > 0); Assert(cShift < 32);
4645	#if defined(RT_ARCH_AMD64)
4646	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
4647	off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, iGprDst, cShift);
4648
4649	#elif defined(RT_ARCH_ARM64)
4650	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprSrc, cShift, false /64Bit/);
4651
4652	#else
4653	# error "Port me"
4654	#endif
4655	return off;
4656	}
4657
4658
4659	/**
4660	* Emits code for rotating a GPR a fixed number of bits to the left.
4661	*/
4662	DECL_FORCE_INLINE(uint32_t)
4663	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
4664	{
4665	Assert(cShift > 0 && cShift < 64);
4666
4667	#if defined(RT_ARCH_AMD64)
4668	/* rol dst, cShift */
4669	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
4670	if (cShift != 1)
4671	{
4672	pCodeBuf[off++] = 0xc1;
4673	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4674	pCodeBuf[off++] = cShift;
4675	}
4676	else
4677	{
4678	pCodeBuf[off++] = 0xd1;
4679	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
4680	}
4681
4682	#elif defined(RT_ARCH_ARM64)
4683	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
4684
4685	#else
4686	# error "Port me"
4687	#endif
4688	return off;
4689	}
4690
4691
4692
4693	/*********************************************************************************************************************************
4694	* Compare and Testing *
4695	*********************************************************************************************************************************/
4696
4697
4698	#ifdef RT_ARCH_ARM64
4699	/**
4700	* Emits an ARM64 compare instruction.
4701	*/
4702	DECL_INLINE_THROW(uint32_t)
4703	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
4704	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
4705	{
4706	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4707	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
4708	f64Bit, true /fSetFlags/, cShift, enmShift);
4709	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4710	return off;
4711	}
4712	#endif
4713
4714
4715	/**
4716	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4717	* with conditional instruction.
4718	*/
4719	DECL_FORCE_INLINE(uint32_t)
4720	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4721	{
4722	#ifdef RT_ARCH_AMD64
4723	/* cmp Gv, Ev */
4724	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4725	pCodeBuf[off++] = 0x3b;
4726	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4727
4728	#elif defined(RT_ARCH_ARM64)
4729	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
4730
4731	#else
4732	# error "Port me!"
4733	#endif
4734	return off;
4735	}
4736
4737
4738	/**
4739	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
4740	* with conditional instruction.
4741	*/
4742	DECL_INLINE_THROW(uint32_t)
4743	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4744	{
4745	#ifdef RT_ARCH_AMD64
4746	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4747	#elif defined(RT_ARCH_ARM64)
4748	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4749	#else
4750	# error "Port me!"
4751	#endif
4752	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4753	return off;
4754	}
4755
4756
4757	/**
4758	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4759	* with conditional instruction.
4760	*/
4761	DECL_FORCE_INLINE(uint32_t)
4762	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4763	{
4764	#ifdef RT_ARCH_AMD64
4765	/* cmp Gv, Ev */
4766	if (iGprLeft >= 8 \|\| iGprRight >= 8)
4767	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
4768	pCodeBuf[off++] = 0x3b;
4769	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
4770
4771	#elif defined(RT_ARCH_ARM64)
4772	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
4773
4774	#else
4775	# error "Port me!"
4776	#endif
4777	return off;
4778	}
4779
4780
4781	/**
4782	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
4783	* with conditional instruction.
4784	*/
4785	DECL_INLINE_THROW(uint32_t)
4786	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
4787	{
4788	#ifdef RT_ARCH_AMD64
4789	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
4790	#elif defined(RT_ARCH_ARM64)
4791	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
4792	#else
4793	# error "Port me!"
4794	#endif
4795	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4796	return off;
4797	}
4798
4799
4800	/**
4801	* Emits a compare of a 64-bit GPR with a constant value, settings status
4802	* flags/whatever for use with conditional instruction.
4803	*/
4804	DECL_INLINE_THROW(uint32_t)
4805	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
4806	{
4807	#ifdef RT_ARCH_AMD64
4808	if (uImm <= UINT32_C(0xff))
4809	{
4810	/* cmp Ev, Ib */
4811	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
4812	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4813	pbCodeBuf[off++] = 0x83;
4814	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4815	pbCodeBuf[off++] = (uint8_t)uImm;
4816	}
4817	else if ((int64_t)uImm == (int32_t)uImm)
4818	{
4819	/* cmp Ev, imm */
4820	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
4821	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
4822	pbCodeBuf[off++] = 0x81;
4823	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4824	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4825	pbCodeBuf[off++] = RT_BYTE1(uImm);
4826	pbCodeBuf[off++] = RT_BYTE2(uImm);
4827	pbCodeBuf[off++] = RT_BYTE3(uImm);
4828	pbCodeBuf[off++] = RT_BYTE4(uImm);
4829	}
4830	else
4831	{
4832	/* Use temporary register for the immediate. */
4833	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4834	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4835	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4836	}
4837
4838	#elif defined(RT_ARCH_ARM64)
4839	/** @todo guess there are clevere things we can do here... */
4840	if (uImm < _4K)
4841	{
4842	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4843	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4844	true /64Bit/, true /fSetFlags/);
4845	}
4846	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4847	{
4848	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4849	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
4850	true /64Bit/, true /fSetFlags/, true /fShift12/);
4851	}
4852	else
4853	{
4854	/* Use temporary register for the immediate. */
4855	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4856	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
4857	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4858	}
4859
4860	#else
4861	# error "Port me!"
4862	#endif
4863
4864	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4865	return off;
4866	}
4867
4868
4869	/**
4870	* Emits a compare of a 32-bit GPR with a constant value, settings status
4871	* flags/whatever for use with conditional instruction.
4872	*
4873	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
4874	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
4875	* bits all zero). Will release assert or throw exception if the caller
4876	* violates this restriction.
4877	*/
4878	DECL_FORCE_INLINE_THROW(uint32_t)
4879	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4880	{
4881	#ifdef RT_ARCH_AMD64
4882	if (iGprLeft >= 8)
4883	pCodeBuf[off++] = X86_OP_REX_B;
4884	if (uImm <= UINT32_C(0x7f))
4885	{
4886	/* cmp Ev, Ib */
4887	pCodeBuf[off++] = 0x83;
4888	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4889	pCodeBuf[off++] = (uint8_t)uImm;
4890	}
4891	else
4892	{
4893	/* cmp Ev, imm */
4894	pCodeBuf[off++] = 0x81;
4895	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
4896	pCodeBuf[off++] = RT_BYTE1(uImm);
4897	pCodeBuf[off++] = RT_BYTE2(uImm);
4898	pCodeBuf[off++] = RT_BYTE3(uImm);
4899	pCodeBuf[off++] = RT_BYTE4(uImm);
4900	}
4901
4902	#elif defined(RT_ARCH_ARM64)
4903	/** @todo guess there are clevere things we can do here... */
4904	if (uImm < _4K)
4905	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4906	false /64Bit/, true /fSetFlags/);
4907	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4908	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4909	false /64Bit/, true /fSetFlags/, true /fShift12/);
4910	else
4911	# ifdef IEM_WITH_THROW_CATCH
4912	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4913	# else
4914	AssertReleaseFailedStmt(off = UINT32_MAX);
4915	# endif
4916
4917	#else
4918	# error "Port me!"
4919	#endif
4920	return off;
4921	}
4922
4923
4924	/**
4925	* Emits a compare of a 32-bit GPR with a constant value, settings status
4926	* flags/whatever for use with conditional instruction.
4927	*/
4928	DECL_INLINE_THROW(uint32_t)
4929	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
4930	{
4931	#ifdef RT_ARCH_AMD64
4932	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
4933
4934	#elif defined(RT_ARCH_ARM64)
4935	/** @todo guess there are clevere things we can do here... */
4936	if (uImm < _4K)
4937	{
4938	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4939	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4940	false /64Bit/, true /fSetFlags/);
4941	}
4942	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
4943	{
4944	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4945	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
4946	false /64Bit/, true /fSetFlags/, true /fShift12/);
4947	}
4948	else
4949	{
4950	/* Use temporary register for the immediate. */
4951	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
4952	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
4953	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4954	}
4955
4956	#else
4957	# error "Port me!"
4958	#endif
4959
4960	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4961	return off;
4962	}
4963
4964
4965
4966	/*********************************************************************************************************************************
4967	* Branching *
4968	*********************************************************************************************************************************/
4969
4970	/**
4971	* Emits a JMP rel32 / B imm19 to the given label.
4972	*/
4973	DECL_FORCE_INLINE_THROW(uint32_t)
4974	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
4975	{
4976	Assert(idxLabel < pReNative->cLabels);
4977
4978	#ifdef RT_ARCH_AMD64
4979	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
4980	{
4981	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
4982	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
4983	{
4984	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
4985	pCodeBuf[off++] = (uint8_t)offRel;
4986	}
4987	else
4988	{
4989	offRel -= 3;
4990	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
4991	pCodeBuf[off++] = RT_BYTE1(offRel);
4992	pCodeBuf[off++] = RT_BYTE2(offRel);
4993	pCodeBuf[off++] = RT_BYTE3(offRel);
4994	pCodeBuf[off++] = RT_BYTE4(offRel);
4995	}
4996	}
4997	else
4998	{
4999	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
5000	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
5001	pCodeBuf[off++] = 0xfe;
5002	pCodeBuf[off++] = 0xff;
5003	pCodeBuf[off++] = 0xff;
5004	pCodeBuf[off++] = 0xff;
5005	}
5006	pCodeBuf[off++] = 0xcc; /* int3 poison */
5007
5008	#elif defined(RT_ARCH_ARM64)
5009	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
5010	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
5011	else
5012	{
5013	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
5014	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
5015	}
5016
5017	#else
5018	# error "Port me!"
5019	#endif
5020	return off;
5021	}
5022
5023
5024	/**
5025	* Emits a JMP rel32 / B imm19 to the given label.
5026	*/
5027	DECL_INLINE_THROW(uint32_t)
5028	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5029	{
5030	#ifdef RT_ARCH_AMD64
5031	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
5032	#elif defined(RT_ARCH_ARM64)
5033	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
5034	#else
5035	# error "Port me!"
5036	#endif
5037	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5038	return off;
5039	}
5040
5041
5042	/**
5043	* Emits a JMP rel32 / B imm19 to a new undefined label.
5044	*/
5045	DECL_INLINE_THROW(uint32_t)
5046	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5047	{
5048	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5049	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
5050	}
5051
5052	/** Condition type. */
5053	#ifdef RT_ARCH_AMD64
5054	typedef enum IEMNATIVEINSTRCOND : uint8_t
5055	{
5056	kIemNativeInstrCond_o = 0,
5057	kIemNativeInstrCond_no,
5058	kIemNativeInstrCond_c,
5059	kIemNativeInstrCond_nc,
5060	kIemNativeInstrCond_e,
5061	kIemNativeInstrCond_ne,
5062	kIemNativeInstrCond_be,
5063	kIemNativeInstrCond_nbe,
5064	kIemNativeInstrCond_s,
5065	kIemNativeInstrCond_ns,
5066	kIemNativeInstrCond_p,
5067	kIemNativeInstrCond_np,
5068	kIemNativeInstrCond_l,
5069	kIemNativeInstrCond_nl,
5070	kIemNativeInstrCond_le,
5071	kIemNativeInstrCond_nle
5072	} IEMNATIVEINSTRCOND;
5073	#elif defined(RT_ARCH_ARM64)
5074	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
5075	# define kIemNativeInstrCond_o todo_conditional_codes
5076	# define kIemNativeInstrCond_no todo_conditional_codes
5077	# define kIemNativeInstrCond_c todo_conditional_codes
5078	# define kIemNativeInstrCond_nc todo_conditional_codes
5079	# define kIemNativeInstrCond_e kArmv8InstrCond_Eq
5080	# define kIemNativeInstrCond_ne kArmv8InstrCond_Ne
5081	# define kIemNativeInstrCond_be kArmv8InstrCond_Ls
5082	# define kIemNativeInstrCond_nbe kArmv8InstrCond_Hi
5083	# define kIemNativeInstrCond_s todo_conditional_codes
5084	# define kIemNativeInstrCond_ns todo_conditional_codes
5085	# define kIemNativeInstrCond_p todo_conditional_codes
5086	# define kIemNativeInstrCond_np todo_conditional_codes
5087	# define kIemNativeInstrCond_l kArmv8InstrCond_Lt
5088	# define kIemNativeInstrCond_nl kArmv8InstrCond_Ge
5089	# define kIemNativeInstrCond_le kArmv8InstrCond_Le
5090	# define kIemNativeInstrCond_nle kArmv8InstrCond_Gt
5091	#else
5092	# error "Port me!"
5093	#endif
5094
5095
5096	/**
5097	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
5098	*/
5099	DECL_FORCE_INLINE_THROW(uint32_t)
5100	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5101	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
5102	{
5103	Assert(idxLabel < pReNative->cLabels);
5104
5105	uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
5106	#ifdef RT_ARCH_AMD64
5107	if (offLabel >= off)
5108	{
5109	/* jcc rel32 */
5110	pCodeBuf[off++] = 0x0f;
5111	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5112	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
5113	pCodeBuf[off++] = 0x00;
5114	pCodeBuf[off++] = 0x00;
5115	pCodeBuf[off++] = 0x00;
5116	pCodeBuf[off++] = 0x00;
5117	}
5118	else
5119	{
5120	int32_t offDisp = offLabel - (off + 2);
5121	if ((int8_t)offDisp == offDisp)
5122	{
5123	/* jcc rel8 */
5124	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
5125	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5126	}
5127	else
5128	{
5129	/* jcc rel32 */
5130	offDisp -= 4;
5131	pCodeBuf[off++] = 0x0f;
5132	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5133	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5134	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5135	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5136	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5137	}
5138	}
5139
5140	#elif defined(RT_ARCH_ARM64)
5141	if (offLabel >= off)
5142	{
5143	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5144	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
5145	}
5146	else
5147	{
5148	Assert(off - offLabel <= 0x3ffffU);
5149	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
5150	}
5151
5152	#else
5153	# error "Port me!"
5154	#endif
5155	return off;
5156	}
5157
5158
5159	/**
5160	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
5161	*/
5162	DECL_INLINE_THROW(uint32_t)
5163	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
5164	{
5165	#ifdef RT_ARCH_AMD64
5166	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
5167	#elif defined(RT_ARCH_ARM64)
5168	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
5169	#else
5170	# error "Port me!"
5171	#endif
5172	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5173	return off;
5174	}
5175
5176
5177	/**
5178	* Emits a Jcc rel32 / B.cc imm19 to a new label.
5179	*/
5180	DECL_INLINE_THROW(uint32_t)
5181	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5182	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
5183	{
5184	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
5185	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
5186	}
5187
5188
5189	/**
5190	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
5191	*/
5192	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5193	{
5194	#ifdef RT_ARCH_AMD64
5195	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
5196	#elif defined(RT_ARCH_ARM64)
5197	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
5198	#else
5199	# error "Port me!"
5200	#endif
5201	}
5202
5203	/**
5204	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
5205	*/
5206	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5207	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5208	{
5209	#ifdef RT_ARCH_AMD64
5210	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
5211	#elif defined(RT_ARCH_ARM64)
5212	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
5213	#else
5214	# error "Port me!"
5215	#endif
5216	}
5217
5218
5219	/**
5220	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
5221	*/
5222	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5223	{
5224	#ifdef RT_ARCH_AMD64
5225	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
5226	#elif defined(RT_ARCH_ARM64)
5227	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
5228	#else
5229	# error "Port me!"
5230	#endif
5231	}
5232
5233	/**
5234	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
5235	*/
5236	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5237	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5238	{
5239	#ifdef RT_ARCH_AMD64
5240	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
5241	#elif defined(RT_ARCH_ARM64)
5242	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
5243	#else
5244	# error "Port me!"
5245	#endif
5246	}
5247
5248
5249	/**
5250	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
5251	*/
5252	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5253	{
5254	#ifdef RT_ARCH_AMD64
5255	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
5256	#elif defined(RT_ARCH_ARM64)
5257	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
5258	#else
5259	# error "Port me!"
5260	#endif
5261	}
5262
5263	/**
5264	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
5265	*/
5266	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5267	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5268	{
5269	#ifdef RT_ARCH_AMD64
5270	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
5271	#elif defined(RT_ARCH_ARM64)
5272	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
5273	#else
5274	# error "Port me!"
5275	#endif
5276	}
5277
5278
5279	/**
5280	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
5281	*/
5282	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5283	{
5284	#ifdef RT_ARCH_AMD64
5285	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
5286	#elif defined(RT_ARCH_ARM64)
5287	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
5288	#else
5289	# error "Port me!"
5290	#endif
5291	}
5292
5293	/**
5294	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
5295	*/
5296	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5297	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5298	{
5299	#ifdef RT_ARCH_AMD64
5300	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
5301	#elif defined(RT_ARCH_ARM64)
5302	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
5303	#else
5304	# error "Port me!"
5305	#endif
5306	}
5307
5308
5309	/**
5310	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
5311	*/
5312	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
5313	{
5314	#ifdef RT_ARCH_AMD64
5315	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
5316	#elif defined(RT_ARCH_ARM64)
5317	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
5318	#else
5319	# error "Port me!"
5320	#endif
5321	}
5322
5323	/**
5324	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
5325	*/
5326	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5327	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
5328	{
5329	#ifdef RT_ARCH_AMD64
5330	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
5331	#elif defined(RT_ARCH_ARM64)
5332	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
5333	#else
5334	# error "Port me!"
5335	#endif
5336	}
5337
5338
5339	/**
5340	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5341	*
5342	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5343	*
5344	* Only use hardcoded jumps forward when emitting for exactly one
5345	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5346	* the right target address on all platforms!
5347	*
5348	* Please also note that on x86 it is necessary pass off + 256 or higher
5349	* for @a offTarget one believe the intervening code is more than 127
5350	* bytes long.
5351	*/
5352	DECL_FORCE_INLINE(uint32_t)
5353	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5354	{
5355	#ifdef RT_ARCH_AMD64
5356	/* jcc rel8 / rel32 */
5357	int32_t offDisp = (int32_t)(offTarget - (off + 2));
5358	if (offDisp < 128 && offDisp >= -128)
5359	{
5360	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
5361	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5362	}
5363	else
5364	{
5365	offDisp -= 4;
5366	pCodeBuf[off++] = 0x0f;
5367	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
5368	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5369	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5370	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5371	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5372	}
5373
5374	#elif defined(RT_ARCH_ARM64)
5375	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
5376
5377	#else
5378	# error "Port me!"
5379	#endif
5380	return off;
5381	}
5382
5383
5384	/**
5385	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
5386	*
5387	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
5388	*
5389	* Only use hardcoded jumps forward when emitting for exactly one
5390	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
5391	* the right target address on all platforms!
5392	*
5393	* Please also note that on x86 it is necessary pass off + 256 or higher
5394	* for @a offTarget one believe the intervening code is more than 127
5395	* bytes long.
5396	*/
5397	DECL_INLINE_THROW(uint32_t)
5398	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
5399	{
5400	#ifdef RT_ARCH_AMD64
5401	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
5402	#elif defined(RT_ARCH_ARM64)
5403	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
5404	#else
5405	# error "Port me!"
5406	#endif
5407	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5408	return off;
5409	}
5410
5411
5412	/**
5413	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
5414	*
5415	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5416	*/
5417	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5418	{
5419	#ifdef RT_ARCH_AMD64
5420	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
5421	#elif defined(RT_ARCH_ARM64)
5422	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
5423	#else
5424	# error "Port me!"
5425	#endif
5426	}
5427
5428
5429	/**
5430	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
5431	*
5432	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5433	*/
5434	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5435	{
5436	#ifdef RT_ARCH_AMD64
5437	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
5438	#elif defined(RT_ARCH_ARM64)
5439	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
5440	#else
5441	# error "Port me!"
5442	#endif
5443	}
5444
5445
5446	/**
5447	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
5448	*
5449	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5450	*/
5451	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5452	{
5453	#ifdef RT_ARCH_AMD64
5454	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
5455	#elif defined(RT_ARCH_ARM64)
5456	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
5457	#else
5458	# error "Port me!"
5459	#endif
5460	}
5461
5462
5463	/**
5464	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
5465	*
5466	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5467	*/
5468	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5469	{
5470	#ifdef RT_ARCH_AMD64
5471	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
5472	#elif defined(RT_ARCH_ARM64)
5473	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
5474	#else
5475	# error "Port me!"
5476	#endif
5477	}
5478
5479
5480	/**
5481	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5482	*
5483	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5484	*/
5485	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
5486	{
5487	#ifdef RT_ARCH_AMD64
5488	/* jmp rel8 or rel32 */
5489	int32_t offDisp = offTarget - (off + 2);
5490	if (offDisp < 128 && offDisp >= -128)
5491	{
5492	pCodeBuf[off++] = 0xeb;
5493	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5494	}
5495	else
5496	{
5497	offDisp -= 3;
5498	pCodeBuf[off++] = 0xe9;
5499	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
5500	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
5501	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
5502	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
5503	}
5504
5505	#elif defined(RT_ARCH_ARM64)
5506	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
5507
5508	#else
5509	# error "Port me!"
5510	#endif
5511	return off;
5512	}
5513
5514
5515	/**
5516	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
5517	*
5518	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
5519	*/
5520	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
5521	{
5522	#ifdef RT_ARCH_AMD64
5523	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
5524	#elif defined(RT_ARCH_ARM64)
5525	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
5526	#else
5527	# error "Port me!"
5528	#endif
5529	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5530	return off;
5531	}
5532
5533
5534	/**
5535	* Fixes up a conditional jump to a fixed label.
5536	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
5537	* iemNativeEmitJzToFixed, ...
5538	*/
5539	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
5540	{
5541	#ifdef RT_ARCH_AMD64
5542	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
5543	uint8_t const bOpcode = pbCodeBuf[offFixup];
5544	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
5545	{
5546	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
5547	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
5548	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
5549	}
5550	else
5551	{
5552	if (bOpcode != 0x0f)
5553	Assert(bOpcode == 0xe9);
5554	else
5555	{
5556	offFixup += 1;
5557	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
5558	}
5559	uint32_t const offRel32 = offTarget - (offFixup + 5);
5560	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
5561	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
5562	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
5563	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
5564	}
5565
5566	#elif defined(RT_ARCH_ARM64)
5567	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
5568	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
5569	{
5570	/* B.COND + BC.COND */
5571	int32_t const offDisp = offTarget - offFixup;
5572	Assert(offDisp >= -262144 && offDisp < 262144);
5573	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
5574	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
5575	}
5576	else
5577	{
5578	/* B imm26 */
5579	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
5580	int32_t const offDisp = offTarget - offFixup;
5581	Assert(offDisp >= -33554432 && offDisp < 33554432);
5582	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
5583	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
5584	}
5585
5586	#else
5587	# error "Port me!"
5588	#endif
5589	}
5590
5591
5592	/**
5593	* Internal helper, don't call directly.
5594	*/
5595	DECL_INLINE_THROW(uint32_t)
5596	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5597	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
5598	{
5599	Assert(iBitNo < 64);
5600	#ifdef RT_ARCH_AMD64
5601	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5602	if (iBitNo < 8)
5603	{
5604	/* test Eb, imm8 */
5605	if (iGprSrc >= 4)
5606	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5607	pbCodeBuf[off++] = 0xf6;
5608	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5609	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
5610	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5611	}
5612	else
5613	{
5614	/* bt Ev, imm8 */
5615	if (iBitNo >= 32)
5616	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
5617	else if (iGprSrc >= 8)
5618	pbCodeBuf[off++] = X86_OP_REX_B;
5619	pbCodeBuf[off++] = 0x0f;
5620	pbCodeBuf[off++] = 0xba;
5621	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
5622	pbCodeBuf[off++] = iBitNo;
5623	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
5624	}
5625
5626	#elif defined(RT_ARCH_ARM64)
5627	/* Use the TBNZ instruction here. */
5628	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5629	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
5630	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
5631
5632	#else
5633	# error "Port me!"
5634	#endif
5635	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5636	return off;
5637	}
5638
5639
5640	/**
5641	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
5642	* @a iGprSrc.
5643	*
5644	* @note On ARM64 the range is only +/-8191 instructions.
5645	*/
5646	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5647	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5648	{
5649	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
5650	}
5651
5652
5653	/**
5654	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
5655	* _set_ in @a iGprSrc.
5656	*
5657	* @note On ARM64 the range is only +/-8191 instructions.
5658	*/
5659	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5660	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
5661	{
5662	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
5663	}
5664
5665
5666	/**
5667	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
5668	* flags accordingly.
5669	*/
5670	DECL_INLINE_THROW(uint32_t)
5671	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
5672	{
5673	Assert(fBits != 0);
5674	#ifdef RT_ARCH_AMD64
5675
5676	if (fBits >= UINT32_MAX)
5677	{
5678	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5679
5680	/* test Ev,Gv */
5681	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
5682	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
5683	pbCodeBuf[off++] = 0x85;
5684	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
5685
5686	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5687	}
5688	else if (fBits <= UINT32_MAX)
5689	{
5690	/* test Eb, imm8 or test Ev, imm32 */
5691	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
5692	if (fBits <= UINT8_MAX)
5693	{
5694	if (iGprSrc >= 4)
5695	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5696	pbCodeBuf[off++] = 0xf6;
5697	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5698	pbCodeBuf[off++] = (uint8_t)fBits;
5699	}
5700	else
5701	{
5702	if (iGprSrc >= 8)
5703	pbCodeBuf[off++] = X86_OP_REX_B;
5704	pbCodeBuf[off++] = 0xf7;
5705	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5706	pbCodeBuf[off++] = RT_BYTE1(fBits);
5707	pbCodeBuf[off++] = RT_BYTE2(fBits);
5708	pbCodeBuf[off++] = RT_BYTE3(fBits);
5709	pbCodeBuf[off++] = RT_BYTE4(fBits);
5710	}
5711	}
5712	/** @todo implement me. */
5713	else
5714	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
5715
5716	#elif defined(RT_ARCH_ARM64)
5717	uint32_t uImmR = 0;
5718	uint32_t uImmNandS = 0;
5719	if (Armv8A64ConvertMask64ToImmRImmS(fBits, &uImmNandS, &uImmR))
5720	{
5721	/* ands xzr, iGprSrc, #fBits */
5722	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5723	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR);
5724	}
5725	else
5726	{
5727	/* ands xzr, iGprSrc, iTmpReg */
5728	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5729	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5730	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
5731	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5732	}
5733
5734	#else
5735	# error "Port me!"
5736	#endif
5737	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5738	return off;
5739	}
5740
5741
5742	/**
5743	* Emits a test for any of the bits from @a fBits in the lower 32 bits of
5744	* @a iGprSrc, setting CPU flags accordingly.
5745	*
5746	* @note For ARM64 this only supports @a fBits values that can be expressed
5747	* using the two 6-bit immediates of the ANDS instruction. The caller
5748	* must make sure this is possible!
5749	*/
5750	DECL_FORCE_INLINE_THROW(uint32_t)
5751	iemNativeEmitTestAnyBitsInGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint32_t fBits)
5752	{
5753	Assert(fBits != 0);
5754
5755	#ifdef RT_ARCH_AMD64
5756	if (fBits <= UINT8_MAX)
5757	{
5758	/* test Eb, imm8 */
5759	if (iGprSrc >= 4)
5760	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5761	pCodeBuf[off++] = 0xf6;
5762	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5763	pCodeBuf[off++] = (uint8_t)fBits;
5764	}
5765	else
5766	{
5767	/* test Ev, imm32 */
5768	if (iGprSrc >= 8)
5769	pCodeBuf[off++] = X86_OP_REX_B;
5770	pCodeBuf[off++] = 0xf7;
5771	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5772	pCodeBuf[off++] = RT_BYTE1(fBits);
5773	pCodeBuf[off++] = RT_BYTE2(fBits);
5774	pCodeBuf[off++] = RT_BYTE3(fBits);
5775	pCodeBuf[off++] = RT_BYTE4(fBits);
5776	}
5777
5778	#elif defined(RT_ARCH_ARM64)
5779	/* ands xzr, src, #fBits */
5780	uint32_t uImmR = 0;
5781	uint32_t uImmNandS = 0;
5782	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5783	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5784	else
5785	# ifdef IEM_WITH_THROW_CATCH
5786	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5787	# else
5788	AssertReleaseFailedStmt(off = UINT32_MAX);
5789	# endif
5790
5791	#else
5792	# error "Port me!"
5793	#endif
5794	return off;
5795	}
5796
5797
5798
5799	/**
5800	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5801	* @a iGprSrc, setting CPU flags accordingly.
5802	*
5803	* @note For ARM64 this only supports @a fBits values that can be expressed
5804	* using the two 6-bit immediates of the ANDS instruction. The caller
5805	* must make sure this is possible!
5806	*/
5807	DECL_FORCE_INLINE_THROW(uint32_t)
5808	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5809	{
5810	Assert(fBits != 0);
5811
5812	#ifdef RT_ARCH_AMD64
5813	/* test Eb, imm8 */
5814	if (iGprSrc >= 4)
5815	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
5816	pCodeBuf[off++] = 0xf6;
5817	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
5818	pCodeBuf[off++] = fBits;
5819
5820	#elif defined(RT_ARCH_ARM64)
5821	/* ands xzr, src, #fBits */
5822	uint32_t uImmR = 0;
5823	uint32_t uImmNandS = 0;
5824	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5825	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5826	else
5827	# ifdef IEM_WITH_THROW_CATCH
5828	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
5829	# else
5830	AssertReleaseFailedStmt(off = UINT32_MAX);
5831	# endif
5832
5833	#else
5834	# error "Port me!"
5835	#endif
5836	return off;
5837	}
5838
5839
5840	/**
5841	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
5842	* @a iGprSrc, setting CPU flags accordingly.
5843	*/
5844	DECL_INLINE_THROW(uint32_t)
5845	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
5846	{
5847	Assert(fBits != 0);
5848
5849	#ifdef RT_ARCH_AMD64
5850	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
5851
5852	#elif defined(RT_ARCH_ARM64)
5853	/* ands xzr, src, [tmp\|#imm] */
5854	uint32_t uImmR = 0;
5855	uint32_t uImmNandS = 0;
5856	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
5857	{
5858	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5859	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
5860	}
5861	else
5862	{
5863	/* Use temporary register for the 64-bit immediate. */
5864	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
5865	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
5866	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
5867	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
5868	}
5869
5870	#else
5871	# error "Port me!"
5872	#endif
5873	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5874	return off;
5875	}
5876
5877
5878	/**
5879	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
5880	* are set in @a iGprSrc.
5881	*/
5882	DECL_INLINE_THROW(uint32_t)
5883	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5884	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5885	{
5886	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5887
5888	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5889	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
5890
5891	return off;
5892	}
5893
5894
5895	/**
5896	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
5897	* are set in @a iGprSrc.
5898	*/
5899	DECL_INLINE_THROW(uint32_t)
5900	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5901	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
5902	{
5903	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
5904
5905	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
5906	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
5907
5908	return off;
5909	}
5910
5911
5912	/**
5913	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5914	*
5915	* The operand size is given by @a f64Bit.
5916	*/
5917	DECL_FORCE_INLINE_THROW(uint32_t)
5918	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5919	uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5920	{
5921	Assert(idxLabel < pReNative->cLabels);
5922
5923	#ifdef RT_ARCH_AMD64
5924	/* test reg32,reg32 / test reg64,reg64 */
5925	if (f64Bit)
5926	pCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
5927	else if (iGprSrc >= 8)
5928	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
5929	pCodeBuf[off++] = 0x85;
5930	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
5931
5932	/* jnz idxLabel */
5933	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
5934	fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
5935
5936	#elif defined(RT_ARCH_ARM64)
5937	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
5938	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(pReNative->paLabels[idxLabel].off - off),
5939	iGprSrc, f64Bit);
5940	else
5941	{
5942	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
5943	pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, 0, iGprSrc, f64Bit);
5944	}
5945
5946	#else
5947	# error "Port me!"
5948	#endif
5949	return off;
5950	}
5951
5952
5953	/**
5954	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
5955	*
5956	* The operand size is given by @a f64Bit.
5957	*/
5958	DECL_FORCE_INLINE_THROW(uint32_t)
5959	iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
5960	bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
5961	{
5962	#ifdef RT_ARCH_AMD64
5963	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
5964	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5965	#elif defined(RT_ARCH_ARM64)
5966	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1),
5967	off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
5968	#else
5969	# error "Port me!"
5970	#endif
5971	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
5972	return off;
5973	}
5974
5975
5976	/**
5977	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5978	*
5979	* The operand size is given by @a f64Bit.
5980	*/
5981	DECL_FORCE_INLINE_THROW(uint32_t)
5982	iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
5983	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5984	{
5985	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
5986	f64Bit, false /fJmpIfNotZero/, idxLabel);
5987	}
5988
5989
5990	/**
5991	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
5992	*
5993	* The operand size is given by @a f64Bit.
5994	*/
5995	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
5996	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
5997	{
5998	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /fJmpIfNotZero/, idxLabel);
5999	}
6000
6001
6002	/**
6003	* Emits code that jumps to a new label if @a iGprSrc is zero.
6004	*
6005	* The operand size is given by @a f64Bit.
6006	*/
6007	DECL_INLINE_THROW(uint32_t)
6008	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
6009	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6010	{
6011	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6012	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
6013	}
6014
6015
6016	/**
6017	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
6018	*
6019	* The operand size is given by @a f64Bit.
6020	*/
6021	DECL_FORCE_INLINE_THROW(uint32_t)
6022	iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
6023	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
6024	{
6025	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
6026	f64Bit, true /fJmpIfNotZero/, idxLabel);
6027	}
6028
6029
6030	/**
6031	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
6032	*
6033	* The operand size is given by @a f64Bit.
6034	*/
6035	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6036	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
6037	{
6038	return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /fJmpIfNotZero/, idxLabel);
6039	}
6040
6041
6042	/**
6043	* Emits code that jumps to a new label if @a iGprSrc is not zero.
6044	*
6045	* The operand size is given by @a f64Bit.
6046	*/
6047	DECL_INLINE_THROW(uint32_t)
6048	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
6049	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6050	{
6051	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6052	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
6053	}
6054
6055
6056	/**
6057	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
6058	* differs.
6059	*/
6060	DECL_INLINE_THROW(uint32_t)
6061	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6062	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
6063	{
6064	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
6065	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
6066	return off;
6067	}
6068
6069
6070	/**
6071	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
6072	*/
6073	DECL_INLINE_THROW(uint32_t)
6074	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6075	uint8_t iGprLeft, uint8_t iGprRight,
6076	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6077	{
6078	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6079	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
6080	}
6081
6082
6083	/**
6084	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
6085	*/
6086	DECL_INLINE_THROW(uint32_t)
6087	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6088	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
6089	{
6090	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
6091	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
6092	return off;
6093	}
6094
6095
6096	/**
6097	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
6098	*/
6099	DECL_INLINE_THROW(uint32_t)
6100	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6101	uint8_t iGprSrc, uint64_t uImm,
6102	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6103	{
6104	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6105	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
6106	}
6107
6108
6109	/**
6110	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
6111	* @a uImm.
6112	*/
6113	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6114	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
6115	{
6116	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
6117	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
6118	return off;
6119	}
6120
6121
6122	/**
6123	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
6124	* @a uImm.
6125	*/
6126	DECL_INLINE_THROW(uint32_t)
6127	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
6128	uint8_t iGprSrc, uint32_t uImm,
6129	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
6130	{
6131	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
6132	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
6133	}
6134
6135
6136	/*********************************************************************************************************************************
6137	* Calls. *
6138	*********************************************************************************************************************************/
6139
6140	/**
6141	* Emits a call to a 64-bit address.
6142	*/
6143	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
6144	{
6145	#ifdef RT_ARCH_AMD64
6146	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
6147
6148	/* call rax */
6149	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
6150	pbCodeBuf[off++] = 0xff;
6151	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
6152
6153	#elif defined(RT_ARCH_ARM64)
6154	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
6155
6156	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
6157	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
6158
6159	#else
6160	# error "port me"
6161	#endif
6162	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
6163	return off;
6164	}
6165
6166
6167	/**
6168	* Emits code to load a stack variable into an argument GPR.
6169	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
6170	*/
6171	DECL_FORCE_INLINE_THROW(uint32_t)
6172	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6173	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = UINT32_MAX,
6174	bool fSpilledVarsInVolatileRegs = false)
6175	{
6176	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6177	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
6178	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6179
6180	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
6181	if ( idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs)
6182	&& ( (RT_BIT_32(idxRegVar) & (~IEMNATIVE_CALL_VOLATILE_GREG_MASK \| fHstVolatileRegsAllowed))
6183	\|\| !fSpilledVarsInVolatileRegs ))
6184	{
6185	AssertStmt( !(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK)
6186	\|\| (RT_BIT_32(idxRegVar) & fHstVolatileRegsAllowed),
6187	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_13));
6188	if (!offAddend)
6189	{
6190	if (idxRegArg != idxRegVar)
6191	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
6192	}
6193	else
6194	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
6195	}
6196	else
6197	{
6198	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
6199	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
6200	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
6201	if (offAddend)
6202	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
6203	}
6204	return off;
6205	}
6206
6207
6208	/**
6209	* Emits code to load a stack or immediate variable value into an argument GPR,
6210	* optional with a addend.
6211	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
6212	*/
6213	DECL_FORCE_INLINE_THROW(uint32_t)
6214	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6215	int32_t offAddend = 0, uint32_t fHstVolatileRegsAllowed = 0,
6216	bool fSpilledVarsInVolatileRegs = false)
6217	{
6218	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6219	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
6220	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
6221	else
6222	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend,
6223	fHstVolatileRegsAllowed, fSpilledVarsInVolatileRegs);
6224	return off;
6225	}
6226
6227
6228	/**
6229	* Emits code to load the variable address into an argument GRP.
6230	*
6231	* This only works for uninitialized and stack variables.
6232	*/
6233	DECL_FORCE_INLINE_THROW(uint32_t)
6234	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
6235	bool fFlushShadows)
6236	{
6237	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
6238	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
6239	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
6240	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
6241
6242	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
6243	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
6244
6245	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
6246	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
6247	{
6248	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
6249	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
6250	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6251	}
6252	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
6253	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
6254
6255	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
6256	}
6257
6258
6259	/** @} */
6260
6261	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
6262

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102856

Download in other formats: