IEMN8veRecompilerEmit.h@ 102724

Last change on this file since 102724 was 102724, checked in by vboxsync, 11 months ago
VMM/IEM: Moved the TLB lookup code emitting into a separate function and straighten the code path a little by starting with a jump and moving the check_expand_down stuff in after tlbmiss. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 147.9 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102724 2023-12-28 21:15:52Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	pCodeBuf[off++] = 0xcc;
88	RT_NOREF(uInfo); /** @todo use multibyte nop for info? */
89
90	#elif defined(RT_ARCH_ARM64)
91	pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
92
93	#else
94	# error "error"
95	#endif
96	return off;
97	}
98
99
100	/**
101	* Emit a breakpoint instruction.
102	*/
103	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
104	{
105	#ifdef RT_ARCH_AMD64
106	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
107	#elif defined(RT_ARCH_ARM64)
108	off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
109	#else
110	# error "error"
111	#endif
112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
113	return off;
114	}
115
116
117	/*********************************************************************************************************************************
118	* Loads, Stores and Related Stuff. *
119	*********************************************************************************************************************************/
120
121	#ifdef RT_ARCH_AMD64
122	/**
123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
124	*/
125	DECL_FORCE_INLINE(uint32_t)
126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
127	{
128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
129	{
130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
133	}
134	else if (offDisp == (int8_t)offDisp)
135	{
136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
139	pbCodeBuf[off++] = (uint8_t)offDisp;
140	}
141	else
142	{
143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
150	}
151	return off;
152	}
153	#endif /* RT_ARCH_AMD64 */
154
155	/**
156	* Emits setting a GPR to zero.
157	*/
158	DECL_INLINE_THROW(uint32_t)
159	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
160	{
161	#ifdef RT_ARCH_AMD64
162	/* xor gpr32, gpr32 */
163	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
164	if (iGpr >= 8)
165	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
166	pbCodeBuf[off++] = 0x33;
167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
168
169	#elif defined(RT_ARCH_ARM64)
170	/* mov gpr, #0x0 */
171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
172	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
173
174	#else
175	# error "port me"
176	#endif
177	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
178	return off;
179	}
180
181
182	/**
183	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
184	* buffer space.
185	*
186	* Max buffer consumption:
187	* - AMD64: 10 instruction bytes.
188	* - ARM64: 4 instruction words (16 bytes).
189	*/
190	DECL_FORCE_INLINE(uint32_t)
191	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
192	{
193	#ifdef RT_ARCH_AMD64
194	if (uImm64 == 0)
195	{
196	/* xor gpr, gpr */
197	if (iGpr >= 8)
198	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
199	pCodeBuf[off++] = 0x33;
200	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
201	}
202	else if (uImm64 <= UINT32_MAX)
203	{
204	/* mov gpr, imm32 */
205	if (iGpr >= 8)
206	pCodeBuf[off++] = X86_OP_REX_B;
207	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else if (uImm64 == (uint64_t)(int32_t)uImm64)
214	{
215	/* mov gpr, sx(imm32) */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xc7;
221	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
222	pCodeBuf[off++] = RT_BYTE1(uImm64);
223	pCodeBuf[off++] = RT_BYTE2(uImm64);
224	pCodeBuf[off++] = RT_BYTE3(uImm64);
225	pCodeBuf[off++] = RT_BYTE4(uImm64);
226	}
227	else
228	{
229	/* mov gpr, imm64 */
230	if (iGpr < 8)
231	pCodeBuf[off++] = X86_OP_REX_W;
232	else
233	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
234	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
235	pCodeBuf[off++] = RT_BYTE1(uImm64);
236	pCodeBuf[off++] = RT_BYTE2(uImm64);
237	pCodeBuf[off++] = RT_BYTE3(uImm64);
238	pCodeBuf[off++] = RT_BYTE4(uImm64);
239	pCodeBuf[off++] = RT_BYTE5(uImm64);
240	pCodeBuf[off++] = RT_BYTE6(uImm64);
241	pCodeBuf[off++] = RT_BYTE7(uImm64);
242	pCodeBuf[off++] = RT_BYTE8(uImm64);
243	}
244
245	#elif defined(RT_ARCH_ARM64)
246	/*
247	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
248	* supply remaining bits using 'movk grp, imm16, lsl #x'.
249	*
250	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
251	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
252	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
253	* after the first non-zero immediate component so we switch to movk for
254	* the remainder.
255	*/
256	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
257	+ !((uImm64 >> 16) & UINT16_MAX)
258	+ !((uImm64 >> 32) & UINT16_MAX)
259	+ !((uImm64 >> 48) & UINT16_MAX);
260	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
261	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
262	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
263	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
264	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
265	if (cFfffHalfWords <= cZeroHalfWords)
266	{
267	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
268
269	/* movz gpr, imm16 */
270	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
271	if (uImmPart \|\| cZeroHalfWords == 4)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #16 */
277	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
278	if (uImmPart)
279	{
280	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
281	fMovBase \|= RT_BIT_32(29);
282	}
283	/* mov[z/k] gpr, imm16, lsl #32 */
284	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
285	if (uImmPart)
286	{
287	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
288	fMovBase \|= RT_BIT_32(29);
289	}
290	/* mov[z/k] gpr, imm16, lsl #48 */
291	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
292	if (uImmPart)
293	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
294	}
295	else
296	{
297	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
298
299	/* find the first half-word that isn't UINT16_MAX. */
300	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
301	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
302	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
303
304	/* movn gpr, imm16, lsl #iHwNotFfff16 /
305	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
306	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
307	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
308	/* movk gpr, imm16 */
309	if (iHwNotFfff != 0)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #16 */
316	if (iHwNotFfff != 1)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
321	}
322	/* movk gpr, imm16, lsl #32 */
323	if (iHwNotFfff != 2)
324	{
325	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
326	if (uImmPart != UINT32_C(0xffff))
327	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
328	}
329	/* movk gpr, imm16, lsl #48 */
330	if (iHwNotFfff != 3)
331	{
332	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
333	if (uImmPart != UINT32_C(0xffff))
334	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
335	}
336	}
337
338	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
339	* clang 12.x does that, only to use the 'x' version for the
340	* addressing in the following ldr). */
341
342	#else
343	# error "port me"
344	#endif
345	return off;
346	}
347
348
349	/**
350	* Emits loading a constant into a 64-bit GPR
351	*/
352	DECL_INLINE_THROW(uint32_t)
353	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
354	{
355	#ifdef RT_ARCH_AMD64
356	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
357	#elif defined(RT_ARCH_ARM64)
358	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
359	#else
360	# error "port me"
361	#endif
362	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
363	return off;
364	}
365
366
367	/**
368	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
369	* buffer space.
370	*
371	* Max buffer consumption:
372	* - AMD64: 6 instruction bytes.
373	* - ARM64: 2 instruction words (8 bytes).
374	*
375	* @note The top 32 bits will be cleared.
376	*/
377	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
378	{
379	#ifdef RT_ARCH_AMD64
380	if (uImm32 == 0)
381	{
382	/* xor gpr, gpr */
383	if (iGpr >= 8)
384	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
385	pCodeBuf[off++] = 0x33;
386	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
387	}
388	else
389	{
390	/* mov gpr, imm32 */
391	if (iGpr >= 8)
392	pCodeBuf[off++] = X86_OP_REX_B;
393	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
394	pCodeBuf[off++] = RT_BYTE1(uImm32);
395	pCodeBuf[off++] = RT_BYTE2(uImm32);
396	pCodeBuf[off++] = RT_BYTE3(uImm32);
397	pCodeBuf[off++] = RT_BYTE4(uImm32);
398	}
399
400	#elif defined(RT_ARCH_ARM64)
401	if ((uImm32 >> 16) == 0)
402	/* movz gpr, imm16 */
403	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
404	else if ((uImm32 & UINT32_C(0xffff)) == 0)
405	/* movz gpr, imm16, lsl #16 */
406	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
407	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
408	/* movn gpr, imm16, lsl #16 */
409	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
410	else if ((uImm32 >> 16) == UINT32_C(0xffff))
411	/* movn gpr, imm16 */
412	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
413	else
414	{
415	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
416	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
417	}
418
419	#else
420	# error "port me"
421	#endif
422	return off;
423	}
424
425
426	/**
427	* Emits loading a constant into a 32-bit GPR.
428	* @note The top 32 bits will be cleared.
429	*/
430	DECL_INLINE_THROW(uint32_t)
431	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
432	{
433	#ifdef RT_ARCH_AMD64
434	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
435	#elif defined(RT_ARCH_ARM64)
436	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
437	#else
438	# error "port me"
439	#endif
440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
441	return off;
442	}
443
444
445	/**
446	* Emits loading a constant into a 8-bit GPR
447	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
448	* only the ARM64 version does that.
449	*/
450	DECL_INLINE_THROW(uint32_t)
451	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
452	{
453	#ifdef RT_ARCH_AMD64
454	/* mov gpr, imm8 */
455	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
456	if (iGpr >= 8)
457	pbCodeBuf[off++] = X86_OP_REX_B;
458	else if (iGpr >= 4)
459	pbCodeBuf[off++] = X86_OP_REX;
460	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
461	pbCodeBuf[off++] = RT_BYTE1(uImm8);
462
463	#elif defined(RT_ARCH_ARM64)
464	/* movz gpr, imm16, lsl #0 */
465	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
466	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
467
468	#else
469	# error "port me"
470	#endif
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472	return off;
473	}
474
475
476	#ifdef RT_ARCH_AMD64
477	/**
478	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
479	*/
480	DECL_FORCE_INLINE(uint32_t)
481	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
482	{
483	if (offVCpu < 128)
484	{
485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
486	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
487	}
488	else
489	{
490	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
491	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
492	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
493	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
494	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
495	}
496	return off;
497	}
498	#elif defined(RT_ARCH_ARM64)
499	/**
500	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
501	*/
502	DECL_FORCE_INLINE_THROW(uint32_t)
503	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
504	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
505	{
506	/*
507	* There are a couple of ldr variants that takes an immediate offset, so
508	* try use those if we can, otherwise we have to use the temporary register
509	* help with the addressing.
510	*/
511	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
512	{
513	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
514	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
515	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
516	}
517	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
518	{
519	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
520	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
521	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
522	}
523	else
524	{
525	/* The offset is too large, so we must load it into a register and use
526	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
527	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
528	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
529
530	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
531	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
532	IEMNATIVE_REG_FIXED_TMP0);
533	}
534	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
535	return off;
536	}
537	#endif
538
539
540	/**
541	* Emits a 64-bit GPR load of a VCpu value.
542	*/
543	DECL_INLINE_THROW(uint32_t)
544	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
545	{
546	#ifdef RT_ARCH_AMD64
547	/* mov reg64, mem64 */
548	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
549	if (iGpr < 8)
550	pbCodeBuf[off++] = X86_OP_REX_W;
551	else
552	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
553	pbCodeBuf[off++] = 0x8b;
554	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
555	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
556
557	#elif defined(RT_ARCH_ARM64)
558	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
559
560	#else
561	# error "port me"
562	#endif
563	return off;
564	}
565
566
567	/**
568	* Emits a 32-bit GPR load of a VCpu value.
569	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
570	*/
571	DECL_INLINE_THROW(uint32_t)
572	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
573	{
574	#ifdef RT_ARCH_AMD64
575	/* mov reg32, mem32 */
576	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
577	if (iGpr >= 8)
578	pbCodeBuf[off++] = X86_OP_REX_R;
579	pbCodeBuf[off++] = 0x8b;
580	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
581	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
582
583	#elif defined(RT_ARCH_ARM64)
584	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
585
586	#else
587	# error "port me"
588	#endif
589	return off;
590	}
591
592
593	/**
594	* Emits a 16-bit GPR load of a VCpu value.
595	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
596	*/
597	DECL_INLINE_THROW(uint32_t)
598	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
599	{
600	#ifdef RT_ARCH_AMD64
601	/* movzx reg32, mem16 */
602	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
603	if (iGpr >= 8)
604	pbCodeBuf[off++] = X86_OP_REX_R;
605	pbCodeBuf[off++] = 0x0f;
606	pbCodeBuf[off++] = 0xb7;
607	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
608	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
609
610	#elif defined(RT_ARCH_ARM64)
611	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
612
613	#else
614	# error "port me"
615	#endif
616	return off;
617	}
618
619
620	/**
621	* Emits a 8-bit GPR load of a VCpu value.
622	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
623	*/
624	DECL_INLINE_THROW(uint32_t)
625	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
626	{
627	#ifdef RT_ARCH_AMD64
628	/* movzx reg32, mem8 */
629	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
630	if (iGpr >= 8)
631	pbCodeBuf[off++] = X86_OP_REX_R;
632	pbCodeBuf[off++] = 0x0f;
633	pbCodeBuf[off++] = 0xb6;
634	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
635	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
636
637	#elif defined(RT_ARCH_ARM64)
638	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
639
640	#else
641	# error "port me"
642	#endif
643	return off;
644	}
645
646
647	/**
648	* Emits a store of a GPR value to a 64-bit VCpu field.
649	*/
650	DECL_INLINE_THROW(uint32_t)
651	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
652	{
653	#ifdef RT_ARCH_AMD64
654	/* mov mem64, reg64 */
655	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
656	if (iGpr < 8)
657	pbCodeBuf[off++] = X86_OP_REX_W;
658	else
659	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
660	pbCodeBuf[off++] = 0x89;
661	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
662	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
663
664	#elif defined(RT_ARCH_ARM64)
665	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
666
667	#else
668	# error "port me"
669	#endif
670	return off;
671	}
672
673
674	/**
675	* Emits a store of a GPR value to a 32-bit VCpu field.
676	*/
677	DECL_INLINE_THROW(uint32_t)
678	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
679	{
680	#ifdef RT_ARCH_AMD64
681	/* mov mem32, reg32 */
682	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
683	if (iGpr >= 8)
684	pbCodeBuf[off++] = X86_OP_REX_R;
685	pbCodeBuf[off++] = 0x89;
686	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
687	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
688
689	#elif defined(RT_ARCH_ARM64)
690	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
691
692	#else
693	# error "port me"
694	#endif
695	return off;
696	}
697
698
699	/**
700	* Emits a store of a GPR value to a 16-bit VCpu field.
701	*/
702	DECL_INLINE_THROW(uint32_t)
703	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
704	{
705	#ifdef RT_ARCH_AMD64
706	/* mov mem16, reg16 */
707	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
708	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
709	if (iGpr >= 8)
710	pbCodeBuf[off++] = X86_OP_REX_R;
711	pbCodeBuf[off++] = 0x89;
712	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
713	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
714
715	#elif defined(RT_ARCH_ARM64)
716	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
717
718	#else
719	# error "port me"
720	#endif
721	return off;
722	}
723
724
725	/**
726	* Emits a store of a GPR value to a 8-bit VCpu field.
727	*/
728	DECL_INLINE_THROW(uint32_t)
729	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
730	{
731	#ifdef RT_ARCH_AMD64
732	/* mov mem8, reg8 */
733	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
734	if (iGpr >= 8)
735	pbCodeBuf[off++] = X86_OP_REX_R;
736	pbCodeBuf[off++] = 0x88;
737	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
738	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
739
740	#elif defined(RT_ARCH_ARM64)
741	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
742
743	#else
744	# error "port me"
745	#endif
746	return off;
747	}
748
749
750	/**
751	* Emits a store of an immediate value to a 8-bit VCpu field.
752	*/
753	DECL_INLINE_THROW(uint32_t)
754	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
755	{
756	#ifdef RT_ARCH_AMD64
757	/* mov mem8, imm8 */
758	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
759	pbCodeBuf[off++] = 0xc6;
760	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
761	pbCodeBuf[off++] = bImm;
762	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
763
764	#elif defined(RT_ARCH_ARM64)
765	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
766	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
767	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
768	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
769
770	#else
771	# error "port me"
772	#endif
773	return off;
774	}
775
776
777	/**
778	* Emits a load effective address to a GRP of a VCpu field.
779	*/
780	DECL_INLINE_THROW(uint32_t)
781	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
782	{
783	#ifdef RT_ARCH_AMD64
784	/* lea gprdst, [rbx + offDisp] */
785	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
786	if (iGprDst < 8)
787	pbCodeBuf[off++] = X86_OP_REX_W;
788	else
789	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
790	pbCodeBuf[off++] = 0x8d;
791	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
792
793	#elif defined(RT_ARCH_ARM64)
794	if (offVCpu < (unsigned)_4K)
795	{
796	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
797	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
798	}
799	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
800	{
801	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
802	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
803	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
804	}
805	else
806	{
807	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
808	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
809	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
810	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
811	}
812
813	#else
814	# error "port me"
815	#endif
816	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
817	return off;
818	}
819
820
821	/**
822	* Emits a gprdst = gprsrc load.
823	*/
824	DECL_FORCE_INLINE(uint32_t)
825	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
826	{
827	#ifdef RT_ARCH_AMD64
828	/* mov gprdst, gprsrc */
829	if ((iGprDst \| iGprSrc) >= 8)
830	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
831	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
832	: X86_OP_REX_W \| X86_OP_REX_R;
833	else
834	pCodeBuf[off++] = X86_OP_REX_W;
835	pCodeBuf[off++] = 0x8b;
836	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
837
838	#elif defined(RT_ARCH_ARM64)
839	/* mov dst, src; alias for: orr dst, xzr, src */
840	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
841
842	#else
843	# error "port me"
844	#endif
845	return off;
846	}
847
848
849	/**
850	* Emits a gprdst = gprsrc load.
851	*/
852	DECL_INLINE_THROW(uint32_t)
853	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
854	{
855	#ifdef RT_ARCH_AMD64
856	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
857	#elif defined(RT_ARCH_ARM64)
858	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
859	#else
860	# error "port me"
861	#endif
862	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
863	return off;
864	}
865
866
867	/**
868	* Emits a gprdst = gprsrc[31:0] load.
869	* @note Bits 63 thru 32 are cleared.
870	*/
871	DECL_FORCE_INLINE(uint32_t)
872	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
873	{
874	#ifdef RT_ARCH_AMD64
875	/* mov gprdst, gprsrc */
876	if ((iGprDst \| iGprSrc) >= 8)
877	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
878	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
879	: X86_OP_REX_R;
880	pCodeBuf[off++] = 0x8b;
881	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
882
883	#elif defined(RT_ARCH_ARM64)
884	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
885	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
886
887	#else
888	# error "port me"
889	#endif
890	return off;
891	}
892
893
894	/**
895	* Emits a gprdst = gprsrc[31:0] load.
896	* @note Bits 63 thru 32 are cleared.
897	*/
898	DECL_INLINE_THROW(uint32_t)
899	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
900	{
901	#ifdef RT_ARCH_AMD64
902	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
903	#elif defined(RT_ARCH_ARM64)
904	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
905	#else
906	# error "port me"
907	#endif
908	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
909	return off;
910	}
911
912
913	/**
914	* Emits a gprdst = gprsrc[15:0] load.
915	* @note Bits 63 thru 15 are cleared.
916	*/
917	DECL_INLINE_THROW(uint32_t)
918	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
919	{
920	#ifdef RT_ARCH_AMD64
921	/* movzx Gv,Ew */
922	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
923	if ((iGprDst \| iGprSrc) >= 8)
924	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
925	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
926	: X86_OP_REX_R;
927	pbCodeBuf[off++] = 0x0f;
928	pbCodeBuf[off++] = 0xb7;
929	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
930
931	#elif defined(RT_ARCH_ARM64)
932	/* and gprdst, gprsrc, #0xffff */
933	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
934	# if 1
935	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
936	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
937	# else
938	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
939	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
940	# endif
941
942	#else
943	# error "port me"
944	#endif
945	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
946	return off;
947	}
948
949
950	/**
951	* Emits a gprdst = gprsrc[7:0] load.
952	* @note Bits 63 thru 8 are cleared.
953	*/
954	DECL_FORCE_INLINE(uint32_t)
955	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
956	{
957	#ifdef RT_ARCH_AMD64
958	/* movzx Gv,Eb */
959	if (iGprDst >= 8 \|\| iGprSrc >= 8)
960	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
961	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
962	: X86_OP_REX_R;
963	else if (iGprSrc >= 4)
964	pCodeBuf[off++] = X86_OP_REX;
965	pCodeBuf[off++] = 0x0f;
966	pCodeBuf[off++] = 0xb6;
967	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
968
969	#elif defined(RT_ARCH_ARM64)
970	/* and gprdst, gprsrc, #0xff */
971	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
972	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
973
974	#else
975	# error "port me"
976	#endif
977	return off;
978	}
979
980
981	/**
982	* Emits a gprdst = gprsrc[7:0] load.
983	* @note Bits 63 thru 8 are cleared.
984	*/
985	DECL_INLINE_THROW(uint32_t)
986	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
987	{
988	#ifdef RT_ARCH_AMD64
989	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
990	#elif defined(RT_ARCH_ARM64)
991	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
992	#else
993	# error "port me"
994	#endif
995	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
996	return off;
997	}
998
999
1000	/**
1001	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1002	* @note Bits 63 thru 8 are cleared.
1003	*/
1004	DECL_INLINE_THROW(uint32_t)
1005	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1006	{
1007	#ifdef RT_ARCH_AMD64
1008	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1009
1010	/* movzx Gv,Ew */
1011	if ((iGprDst \| iGprSrc) >= 8)
1012	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1013	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1014	: X86_OP_REX_R;
1015	pbCodeBuf[off++] = 0x0f;
1016	pbCodeBuf[off++] = 0xb7;
1017	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1018
1019	/* shr Ev,8 */
1020	if (iGprDst >= 8)
1021	pbCodeBuf[off++] = X86_OP_REX_B;
1022	pbCodeBuf[off++] = 0xc1;
1023	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1024	pbCodeBuf[off++] = 8;
1025
1026	#elif defined(RT_ARCH_ARM64)
1027	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1028	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1029	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1030
1031	#else
1032	# error "port me"
1033	#endif
1034	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1035	return off;
1036	}
1037
1038
1039	/**
1040	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1041	*/
1042	DECL_INLINE_THROW(uint32_t)
1043	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1044	{
1045	#ifdef RT_ARCH_AMD64
1046	/* movsxd r64, r/m32 */
1047	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1048	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1049	pbCodeBuf[off++] = 0x63;
1050	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1051
1052	#elif defined(RT_ARCH_ARM64)
1053	/* sxtw dst, src */
1054	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1055	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1056
1057	#else
1058	# error "port me"
1059	#endif
1060	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1061	return off;
1062	}
1063
1064
1065	/**
1066	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1067	*/
1068	DECL_INLINE_THROW(uint32_t)
1069	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1070	{
1071	#ifdef RT_ARCH_AMD64
1072	/* movsx r64, r/m16 */
1073	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1074	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1075	pbCodeBuf[off++] = 0x0f;
1076	pbCodeBuf[off++] = 0xbf;
1077	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1078
1079	#elif defined(RT_ARCH_ARM64)
1080	/* sxth dst, src */
1081	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1082	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1083
1084	#else
1085	# error "port me"
1086	#endif
1087	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1088	return off;
1089	}
1090
1091
1092	/**
1093	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1094	*/
1095	DECL_INLINE_THROW(uint32_t)
1096	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1097	{
1098	#ifdef RT_ARCH_AMD64
1099	/* movsx r64, r/m16 */
1100	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1101	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1102	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1103	pbCodeBuf[off++] = 0x0f;
1104	pbCodeBuf[off++] = 0xbf;
1105	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1106
1107	#elif defined(RT_ARCH_ARM64)
1108	/* sxth dst32, src */
1109	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1110	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1111
1112	#else
1113	# error "port me"
1114	#endif
1115	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1116	return off;
1117	}
1118
1119
1120	/**
1121	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1122	*/
1123	DECL_INLINE_THROW(uint32_t)
1124	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1125	{
1126	#ifdef RT_ARCH_AMD64
1127	/* movsx r64, r/m8 */
1128	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1129	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1130	pbCodeBuf[off++] = 0x0f;
1131	pbCodeBuf[off++] = 0xbe;
1132	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1133
1134	#elif defined(RT_ARCH_ARM64)
1135	/* sxtb dst, src */
1136	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1137	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1138
1139	#else
1140	# error "port me"
1141	#endif
1142	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1143	return off;
1144	}
1145
1146
1147	/**
1148	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1149	* @note Bits 63 thru 32 are cleared.
1150	*/
1151	DECL_INLINE_THROW(uint32_t)
1152	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1153	{
1154	#ifdef RT_ARCH_AMD64
1155	/* movsx r32, r/m8 */
1156	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1157	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1158	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1159	pbCodeBuf[off++] = 0x0f;
1160	pbCodeBuf[off++] = 0xbe;
1161	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1162
1163	#elif defined(RT_ARCH_ARM64)
1164	/* sxtb dst32, src32 */
1165	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1166	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1167
1168	#else
1169	# error "port me"
1170	#endif
1171	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1172	return off;
1173	}
1174
1175
1176	/**
1177	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1178	* @note Bits 63 thru 16 are cleared.
1179	*/
1180	DECL_INLINE_THROW(uint32_t)
1181	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1182	{
1183	#ifdef RT_ARCH_AMD64
1184	/* movsx r16, r/m8 */
1185	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1186	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1187	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1188	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1189	pbCodeBuf[off++] = 0x0f;
1190	pbCodeBuf[off++] = 0xbe;
1191	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1192
1193	/* movzx r32, r/m16 */
1194	if (iGprDst >= 8)
1195	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1196	pbCodeBuf[off++] = 0x0f;
1197	pbCodeBuf[off++] = 0xb7;
1198	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1199
1200	#elif defined(RT_ARCH_ARM64)
1201	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1202	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1203	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1204	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1205	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1206
1207	#else
1208	# error "port me"
1209	#endif
1210	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1211	return off;
1212	}
1213
1214
1215	/**
1216	* Emits a gprdst = gprsrc + addend load.
1217	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1218	*/
1219	#ifdef RT_ARCH_AMD64
1220	DECL_INLINE_THROW(uint32_t)
1221	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1222	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1223	{
1224	Assert(iAddend != 0);
1225
1226	/* lea gprdst, [gprsrc + iAddend] */
1227	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1228	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1229	pbCodeBuf[off++] = 0x8d;
1230	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1231	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1232	return off;
1233	}
1234
1235	#elif defined(RT_ARCH_ARM64)
1236	DECL_INLINE_THROW(uint32_t)
1237	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1238	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1239	{
1240	if ((uint32_t)iAddend < 4096)
1241	{
1242	/* add dst, src, uimm12 */
1243	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1244	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1245	}
1246	else if ((uint32_t)-iAddend < 4096)
1247	{
1248	/* sub dst, src, uimm12 */
1249	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1250	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1251	}
1252	else
1253	{
1254	Assert(iGprSrc != iGprDst);
1255	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1256	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1257	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1258	}
1259	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1260	return off;
1261	}
1262	#else
1263	# error "port me"
1264	#endif
1265
1266	/**
1267	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1268	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1269	*/
1270	#ifdef RT_ARCH_AMD64
1271	DECL_INLINE_THROW(uint32_t)
1272	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1273	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1274	#else
1275	DECL_INLINE_THROW(uint32_t)
1276	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1277	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1278	#endif
1279	{
1280	if (iAddend != 0)
1281	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1282	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1283	}
1284
1285
1286	/**
1287	* Emits a gprdst = gprsrc32 + addend load.
1288	* @note Bits 63 thru 32 are cleared.
1289	*/
1290	DECL_INLINE_THROW(uint32_t)
1291	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1292	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1293	{
1294	Assert(iAddend != 0);
1295
1296	#ifdef RT_ARCH_AMD64
1297	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1298	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1299	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1300	if ((iGprDst \| iGprSrc) >= 8)
1301	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1302	pbCodeBuf[off++] = 0x8d;
1303	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1304
1305	#elif defined(RT_ARCH_ARM64)
1306	if ((uint32_t)iAddend < 4096)
1307	{
1308	/* add dst, src, uimm12 */
1309	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1310	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1311	}
1312	else if ((uint32_t)-iAddend < 4096)
1313	{
1314	/* sub dst, src, uimm12 */
1315	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1316	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1317	}
1318	else
1319	{
1320	Assert(iGprSrc != iGprDst);
1321	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1322	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1323	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1324	}
1325
1326	#else
1327	# error "port me"
1328	#endif
1329	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1330	return off;
1331	}
1332
1333
1334	/**
1335	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1336	*/
1337	DECL_INLINE_THROW(uint32_t)
1338	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1339	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1340	{
1341	if (iAddend != 0)
1342	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1343	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1344	}
1345
1346
1347
1348	#ifdef RT_ARCH_AMD64
1349	/**
1350	* Common bit of iemNativeEmitLoadGprByBp and friends.
1351	*/
1352	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1353	PIEMRECOMPILERSTATE pReNativeAssert)
1354	{
1355	if (offDisp < 128 && offDisp >= -128)
1356	{
1357	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1358	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1359	}
1360	else
1361	{
1362	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1363	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1364	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1365	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1366	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1367	}
1368	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1369	return off;
1370	}
1371	#elif defined(RT_ARCH_ARM64)
1372	/**
1373	* Common bit of iemNativeEmitLoadGprByBp and friends.
1374	*/
1375	DECL_FORCE_INLINE_THROW(uint32_t)
1376	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1377	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1378	{
1379	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1380	{
1381	/* str w/ unsigned imm12 (scaled) */
1382	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1383	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1384	}
1385	else if (offDisp >= -256 && offDisp <= 256)
1386	{
1387	/* stur w/ signed imm9 (unscaled) */
1388	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1389	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1390	}
1391	else
1392	{
1393	/* Use temporary indexing register. */
1394	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1395	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1396	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1397	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1398	}
1399	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1400	return off;
1401	}
1402	#endif
1403
1404
1405	/**
1406	* Emits a 64-bit GRP load instruction with an BP relative source address.
1407	*/
1408	DECL_INLINE_THROW(uint32_t)
1409	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1410	{
1411	#ifdef RT_ARCH_AMD64
1412	/* mov gprdst, qword [rbp + offDisp] */
1413	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1414	if (iGprDst < 8)
1415	pbCodeBuf[off++] = X86_OP_REX_W;
1416	else
1417	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1418	pbCodeBuf[off++] = 0x8b;
1419	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1420
1421	#elif defined(RT_ARCH_ARM64)
1422	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1423
1424	#else
1425	# error "port me"
1426	#endif
1427	}
1428
1429
1430	/**
1431	* Emits a 32-bit GRP load instruction with an BP relative source address.
1432	* @note Bits 63 thru 32 of the GPR will be cleared.
1433	*/
1434	DECL_INLINE_THROW(uint32_t)
1435	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1436	{
1437	#ifdef RT_ARCH_AMD64
1438	/* mov gprdst, dword [rbp + offDisp] */
1439	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1440	if (iGprDst >= 8)
1441	pbCodeBuf[off++] = X86_OP_REX_R;
1442	pbCodeBuf[off++] = 0x8b;
1443	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1444
1445	#elif defined(RT_ARCH_ARM64)
1446	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1447
1448	#else
1449	# error "port me"
1450	#endif
1451	}
1452
1453
1454	/**
1455	* Emits a 16-bit GRP load instruction with an BP relative source address.
1456	* @note Bits 63 thru 16 of the GPR will be cleared.
1457	*/
1458	DECL_INLINE_THROW(uint32_t)
1459	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1460	{
1461	#ifdef RT_ARCH_AMD64
1462	/* movzx gprdst, word [rbp + offDisp] */
1463	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1464	if (iGprDst >= 8)
1465	pbCodeBuf[off++] = X86_OP_REX_R;
1466	pbCodeBuf[off++] = 0x0f;
1467	pbCodeBuf[off++] = 0xb7;
1468	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1469
1470	#elif defined(RT_ARCH_ARM64)
1471	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1472
1473	#else
1474	# error "port me"
1475	#endif
1476	}
1477
1478
1479	/**
1480	* Emits a 8-bit GRP load instruction with an BP relative source address.
1481	* @note Bits 63 thru 8 of the GPR will be cleared.
1482	*/
1483	DECL_INLINE_THROW(uint32_t)
1484	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1485	{
1486	#ifdef RT_ARCH_AMD64
1487	/* movzx gprdst, byte [rbp + offDisp] */
1488	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1489	if (iGprDst >= 8)
1490	pbCodeBuf[off++] = X86_OP_REX_R;
1491	pbCodeBuf[off++] = 0x0f;
1492	pbCodeBuf[off++] = 0xb6;
1493	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1494
1495	#elif defined(RT_ARCH_ARM64)
1496	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1497
1498	#else
1499	# error "port me"
1500	#endif
1501	}
1502
1503
1504	/**
1505	* Emits a load effective address to a GRP with an BP relative source address.
1506	*/
1507	DECL_INLINE_THROW(uint32_t)
1508	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1509	{
1510	#ifdef RT_ARCH_AMD64
1511	/* lea gprdst, [rbp + offDisp] */
1512	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1513	if (iGprDst < 8)
1514	pbCodeBuf[off++] = X86_OP_REX_W;
1515	else
1516	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1517	pbCodeBuf[off++] = 0x8d;
1518	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1519
1520	#elif defined(RT_ARCH_ARM64)
1521	if ((uint32_t)offDisp < (unsigned)_4K)
1522	{
1523	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1524	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1525	}
1526	else if ((uint32_t)-offDisp < (unsigned)_4K)
1527	{
1528	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1529	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1530	}
1531	else
1532	{
1533	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1534	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1535	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1536	if (offDisp >= 0)
1537	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1538	else
1539	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1540	}
1541
1542	#else
1543	# error "port me"
1544	#endif
1545
1546	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1547	return off;
1548	}
1549
1550
1551	/**
1552	* Emits a 64-bit GPR store with an BP relative destination address.
1553	*
1554	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1555	*/
1556	DECL_INLINE_THROW(uint32_t)
1557	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1558	{
1559	#ifdef RT_ARCH_AMD64
1560	/* mov qword [rbp + offDisp], gprdst */
1561	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1562	if (iGprSrc < 8)
1563	pbCodeBuf[off++] = X86_OP_REX_W;
1564	else
1565	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1566	pbCodeBuf[off++] = 0x89;
1567	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1568
1569	#elif defined(RT_ARCH_ARM64)
1570	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1571	{
1572	/* str w/ unsigned imm12 (scaled) */
1573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1574	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1575	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1576	}
1577	else if (offDisp >= -256 && offDisp <= 256)
1578	{
1579	/* stur w/ signed imm9 (unscaled) */
1580	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1581	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1582	}
1583	else if ((uint32_t)-offDisp < (unsigned)_4K)
1584	{
1585	/* Use temporary indexing register w/ sub uimm12. */
1586	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1587	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1588	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1589	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1590	}
1591	else
1592	{
1593	/* Use temporary indexing register. */
1594	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1595	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1596	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1597	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1598	}
1599	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1600	return off;
1601
1602	#else
1603	# error "Port me!"
1604	#endif
1605	}
1606
1607
1608	/**
1609	* Emits a 64-bit immediate store with an BP relative destination address.
1610	*
1611	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1612	*/
1613	DECL_INLINE_THROW(uint32_t)
1614	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1615	{
1616	#ifdef RT_ARCH_AMD64
1617	if ((int64_t)uImm64 == (int32_t)uImm64)
1618	{
1619	/* mov qword [rbp + offDisp], imm32 - sign extended */
1620	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1621	pbCodeBuf[off++] = X86_OP_REX_W;
1622	pbCodeBuf[off++] = 0xc7;
1623	if (offDisp < 128 && offDisp >= -128)
1624	{
1625	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1626	pbCodeBuf[off++] = (uint8_t)offDisp;
1627	}
1628	else
1629	{
1630	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1631	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1632	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1633	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1634	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1635	}
1636	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1637	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1638	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1639	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1640	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1641	return off;
1642	}
1643	#endif
1644
1645	/* Load tmp0, imm64; Store tmp to bp+disp. */
1646	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1647	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1648	}
1649
1650
1651	#if defined(RT_ARCH_ARM64)
1652	/**
1653	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1654	*/
1655	DECL_FORCE_INLINE_THROW(uint32_t)
1656	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1657	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1658	{
1659	/*
1660	* There are a couple of ldr variants that takes an immediate offset, so
1661	* try use those if we can, otherwise we have to use the temporary register
1662	* help with the addressing.
1663	*/
1664	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1665	{
1666	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1667	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1668	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1669	}
1670	else
1671	{
1672	/* The offset is too large, so we must load it into a register and use
1673	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1674	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1675	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1676
1677	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1678	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1679
1680	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1681	}
1682	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1683	return off;
1684	}
1685	#endif
1686
1687
1688	/**
1689	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1690	*/
1691	DECL_INLINE_THROW(uint32_t)
1692	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1693	{
1694	#ifdef RT_ARCH_AMD64
1695	/* mov reg64, mem64 */
1696	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1697	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1698	pbCodeBuf[off++] = 0x8b;
1699	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1700	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1701
1702	#elif defined(RT_ARCH_ARM64)
1703	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1704
1705	#else
1706	# error "port me"
1707	#endif
1708	return off;
1709	}
1710
1711
1712	/**
1713	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1714	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1715	*/
1716	DECL_INLINE_THROW(uint32_t)
1717	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1718	{
1719	#ifdef RT_ARCH_AMD64
1720	/* mov reg32, mem32 */
1721	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1722	if (iGprDst >= 8 \|\| iGprBase >= 8)
1723	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1724	pbCodeBuf[off++] = 0x8b;
1725	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1726	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1727
1728	#elif defined(RT_ARCH_ARM64)
1729	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1730
1731	#else
1732	# error "port me"
1733	#endif
1734	return off;
1735	}
1736
1737
1738	/*********************************************************************************************************************************
1739	* Subtraction and Additions *
1740	*********************************************************************************************************************************/
1741
1742	/**
1743	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
1744	* @note The AMD64 version sets flags.
1745	*/
1746	DECL_INLINE_THROW(uint32_t)
1747	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1748	{
1749	#if defined(RT_ARCH_AMD64)
1750	/* sub Gv,Ev */
1751	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1752	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1753	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1754	pbCodeBuf[off++] = 0x2b;
1755	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1756
1757	#elif defined(RT_ARCH_ARM64)
1758	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1759	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
1760
1761	#else
1762	# error "Port me"
1763	#endif
1764	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1765	return off;
1766	}
1767
1768
1769	/**
1770	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
1771	* @note The AMD64 version sets flags.
1772	*/
1773	DECL_FORCE_INLINE(uint32_t)
1774	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1775	{
1776	#if defined(RT_ARCH_AMD64)
1777	/* sub Gv,Ev */
1778	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
1779	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
1780	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1781	pCodeBuf[off++] = 0x2b;
1782	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1783
1784	#elif defined(RT_ARCH_ARM64)
1785	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
1786
1787	#else
1788	# error "Port me"
1789	#endif
1790	return off;
1791	}
1792
1793
1794	/**
1795	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
1796	* @note The AMD64 version sets flags.
1797	*/
1798	DECL_INLINE_THROW(uint32_t)
1799	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1800	{
1801	#if defined(RT_ARCH_AMD64)
1802	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
1803	#elif defined(RT_ARCH_ARM64)
1804	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
1805	#else
1806	# error "Port me"
1807	#endif
1808	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1809	return off;
1810	}
1811
1812
1813	#ifdef RT_ARCH_AMD64
1814	/**
1815	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1816	*/
1817	DECL_INLINE_THROW(uint32_t)
1818	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1819	{
1820	/* sub gprdst, imm8/imm32 */
1821	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1822	if (iGprDst < 8)
1823	pbCodeBuf[off++] = X86_OP_REX_W;
1824	else
1825	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1826	if (iSubtrahend < 128 && iSubtrahend >= -128)
1827	{
1828	pbCodeBuf[off++] = 0x83;
1829	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1830	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1831	}
1832	else
1833	{
1834	pbCodeBuf[off++] = 0x81;
1835	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1836	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1837	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1838	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1839	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1840	}
1841	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1842	return off;
1843	}
1844	#endif
1845
1846
1847	#ifdef RT_ARCH_AMD64
1848	/**
1849	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
1850	*
1851	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
1852	*/
1853	DECL_FORCE_INLINE(uint32_t)
1854	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1855	{
1856	if (iGprDst >= 8)
1857	pCodeBuf[off++] = X86_OP_REX_B;
1858	if (iSubtrahend == 1)
1859	{
1860	/* dec r/m32 */
1861	pCodeBuf[off++] = 0xff;
1862	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
1863	}
1864	else if (iSubtrahend == -1)
1865	{
1866	/* inc r/m32 */
1867	pCodeBuf[off++] = 0xff;
1868	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1869	}
1870	else if (iSubtrahend < 128 && iSubtrahend >= -128)
1871	{
1872	/* sub r/m32, imm8 */
1873	pCodeBuf[off++] = 0x83;
1874	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1875	pCodeBuf[off++] = (uint8_t)iSubtrahend;
1876	}
1877	else
1878	{
1879	/* sub r/m32, imm32 */
1880	pCodeBuf[off++] = 0x81;
1881	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1882	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1883	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1884	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1885	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1886	}
1887	return off;
1888	}
1889	#endif
1890
1891
1892	/**
1893	* Emits adding a 64-bit GPR to another, storing the result in the first.
1894	* @note The AMD64 version sets flags.
1895	*/
1896	DECL_FORCE_INLINE(uint32_t)
1897	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1898	{
1899	#if defined(RT_ARCH_AMD64)
1900	/* add Gv,Ev */
1901	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1902	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1903	pCodeBuf[off++] = 0x03;
1904	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1905
1906	#elif defined(RT_ARCH_ARM64)
1907	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1908
1909	#else
1910	# error "Port me"
1911	#endif
1912	return off;
1913	}
1914
1915
1916	/**
1917	* Emits adding a 64-bit GPR to another, storing the result in the first.
1918	* @note The AMD64 version sets flags.
1919	*/
1920	DECL_INLINE_THROW(uint32_t)
1921	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1922	{
1923	#if defined(RT_ARCH_AMD64)
1924	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
1925	#elif defined(RT_ARCH_ARM64)
1926	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
1927	#else
1928	# error "Port me"
1929	#endif
1930	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1931	return off;
1932	}
1933
1934
1935	/**
1936	* Emits adding a 64-bit GPR to another, storing the result in the first.
1937	* @note The AMD64 version sets flags.
1938	*/
1939	DECL_FORCE_INLINE(uint32_t)
1940	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1941	{
1942	#if defined(RT_ARCH_AMD64)
1943	/* add Gv,Ev */
1944	if (iGprDst >= 8 \|\| iGprAddend >= 8)
1945	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
1946	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
1947	pCodeBuf[off++] = 0x03;
1948	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1949
1950	#elif defined(RT_ARCH_ARM64)
1951	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
1952
1953	#else
1954	# error "Port me"
1955	#endif
1956	return off;
1957	}
1958
1959
1960	/**
1961	* Emits adding a 64-bit GPR to another, storing the result in the first.
1962	* @note The AMD64 version sets flags.
1963	*/
1964	DECL_INLINE_THROW(uint32_t)
1965	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1966	{
1967	#if defined(RT_ARCH_AMD64)
1968	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
1969	#elif defined(RT_ARCH_ARM64)
1970	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
1971	#else
1972	# error "Port me"
1973	#endif
1974	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1975	return off;
1976	}
1977
1978
1979	/**
1980	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1981	*/
1982	DECL_INLINE_THROW(uint32_t)
1983	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1984	{
1985	#if defined(RT_ARCH_AMD64)
1986	/* add or inc */
1987	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1988	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1989	if (iImm8 != 1)
1990	{
1991	pbCodeBuf[off++] = 0x83;
1992	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1993	pbCodeBuf[off++] = (uint8_t)iImm8;
1994	}
1995	else
1996	{
1997	pbCodeBuf[off++] = 0xff;
1998	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1999	}
2000
2001	#elif defined(RT_ARCH_ARM64)
2002	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2003	if (iImm8 >= 0)
2004	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
2005	else
2006	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
2007
2008	#else
2009	# error "Port me"
2010	#endif
2011	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2012	return off;
2013	}
2014
2015
2016	/**
2017	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
2018	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2019	*/
2020	DECL_FORCE_INLINE(uint32_t)
2021	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
2022	{
2023	#if defined(RT_ARCH_AMD64)
2024	/* add or inc */
2025	if (iGprDst >= 8)
2026	pCodeBuf[off++] = X86_OP_REX_B;
2027	if (iImm8 != 1)
2028	{
2029	pCodeBuf[off++] = 0x83;
2030	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2031	pCodeBuf[off++] = (uint8_t)iImm8;
2032	}
2033	else
2034	{
2035	pCodeBuf[off++] = 0xff;
2036	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2037	}
2038
2039	#elif defined(RT_ARCH_ARM64)
2040	if (iImm8 >= 0)
2041	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
2042	else
2043	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
2044
2045	#else
2046	# error "Port me"
2047	#endif
2048	return off;
2049	}
2050
2051
2052	/**
2053	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
2054	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2055	*/
2056	DECL_INLINE_THROW(uint32_t)
2057	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
2058	{
2059	#if defined(RT_ARCH_AMD64)
2060	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
2061	#elif defined(RT_ARCH_ARM64)
2062	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
2063	#else
2064	# error "Port me"
2065	#endif
2066	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2067	return off;
2068	}
2069
2070
2071	/**
2072	* Emits a 64-bit GPR additions with a 64-bit signed addend.
2073	*/
2074	DECL_INLINE_THROW(uint32_t)
2075	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
2076	{
2077	#if defined(RT_ARCH_AMD64)
2078	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2079	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
2080
2081	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
2082	{
2083	/* add grp, imm32 */
2084	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2085	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2086	pbCodeBuf[off++] = 0x81;
2087	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2088	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2089	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2090	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2091	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2092	}
2093	else
2094	{
2095	/* Best to use a temporary register to deal with this in the simplest way: */
2096	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2097
2098	/* add dst, tmpreg */
2099	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2100	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2101	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2102	pbCodeBuf[off++] = 0x03;
2103	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
2104
2105	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2106	}
2107
2108	#elif defined(RT_ARCH_ARM64)
2109	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2110	{
2111	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2112	if (iAddend >= 0)
2113	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
2114	else
2115	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
2116	}
2117	else
2118	{
2119	/* Use temporary register for the immediate. */
2120	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2121
2122	/* add gprdst, gprdst, tmpreg */
2123	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2124	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
2125
2126	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2127	}
2128
2129	#else
2130	# error "Port me"
2131	#endif
2132	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2133	return off;
2134	}
2135
2136
2137	/**
2138	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
2139	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2140	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
2141	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
2142	* the lower 12 bits always zero). The negative ranges are also allowed,
2143	* making it behave like a subtraction. If the constant does not conform,
2144	* bad stuff will happen.
2145	*/
2146	DECL_FORCE_INLINE_THROW(uint32_t)
2147	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2148	{
2149	#if defined(RT_ARCH_AMD64)
2150	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2151	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
2152
2153	/* add grp, imm32 */
2154	if (iGprDst >= 8)
2155	pCodeBuf[off++] = X86_OP_REX_B;
2156	pCodeBuf[off++] = 0x81;
2157	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2158	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2159	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2160	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2161	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2162
2163	#elif defined(RT_ARCH_ARM64)
2164	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
2165	if (uAbsAddend <= 0xfff)
2166	{
2167	if (iAddend >= 0)
2168	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
2169	else
2170	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
2171	}
2172	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
2173	{
2174	if (iAddend >= 0)
2175	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
2176	false /f64Bit/, true /fShift12/);
2177	else
2178	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
2179	false /f64Bit/, true /fShift12/);
2180	}
2181	else
2182	# ifdef IEM_WITH_THROW_CATCH
2183	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2184	# else
2185	AssertReleaseFailedStmt(off = UINT32_MAX);
2186	# endif
2187
2188	#else
2189	# error "Port me"
2190	#endif
2191	return off;
2192	}
2193
2194
2195	/**
2196	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
2197	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2198	*/
2199	DECL_INLINE_THROW(uint32_t)
2200	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2201	{
2202	#if defined(RT_ARCH_AMD64)
2203	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
2204
2205	#elif defined(RT_ARCH_ARM64)
2206	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2207	{
2208	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2209	if (iAddend >= 0)
2210	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
2211	else
2212	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
2213	}
2214	else
2215	{
2216	/* Use temporary register for the immediate. */
2217	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
2218
2219	/* add gprdst, gprdst, tmpreg */
2220	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2221	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
2222
2223	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2224	}
2225
2226	#else
2227	# error "Port me"
2228	#endif
2229	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2230	return off;
2231	}
2232
2233
2234	/*********************************************************************************************************************************
2235	* Unary Operations *
2236	*********************************************************************************************************************************/
2237
2238	/**
2239	* Emits code for two complement negation of a 64-bit GPR.
2240	*/
2241	DECL_FORCE_INLINE_THROW(uint32_t)
2242	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
2243	{
2244	#if defined(RT_ARCH_AMD64)
2245	/* neg Ev */
2246	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2247	pCodeBuf[off++] = 0xf7;
2248	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
2249
2250	#elif defined(RT_ARCH_ARM64)
2251	/* sub dst, xzr, dst */
2252	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
2253
2254	#else
2255	# error "Port me"
2256	#endif
2257	return off;
2258	}
2259
2260
2261	/**
2262	* Emits code for two complement negation of a 64-bit GPR.
2263	*/
2264	DECL_INLINE_THROW(uint32_t)
2265	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2266	{
2267	#if defined(RT_ARCH_AMD64)
2268	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
2269	#elif defined(RT_ARCH_ARM64)
2270	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
2271	#else
2272	# error "Port me"
2273	#endif
2274	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2275	return off;
2276	}
2277
2278
2279	/**
2280	* Emits code for two complement negation of a 32-bit GPR.
2281	* @note bit 32 thru 63 are set to zero.
2282	*/
2283	DECL_FORCE_INLINE_THROW(uint32_t)
2284	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
2285	{
2286	#if defined(RT_ARCH_AMD64)
2287	/* neg Ev */
2288	if (iGprDst >= 8)
2289	pCodeBuf[off++] = X86_OP_REX_B;
2290	pCodeBuf[off++] = 0xf7;
2291	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
2292
2293	#elif defined(RT_ARCH_ARM64)
2294	/* sub dst, xzr, dst */
2295	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
2296
2297	#else
2298	# error "Port me"
2299	#endif
2300	return off;
2301	}
2302
2303
2304	/**
2305	* Emits code for two complement negation of a 32-bit GPR.
2306	* @note bit 32 thru 63 are set to zero.
2307	*/
2308	DECL_INLINE_THROW(uint32_t)
2309	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2310	{
2311	#if defined(RT_ARCH_AMD64)
2312	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
2313	#elif defined(RT_ARCH_ARM64)
2314	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
2315	#else
2316	# error "Port me"
2317	#endif
2318	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2319	return off;
2320	}
2321
2322
2323
2324	/*********************************************************************************************************************************
2325	* Bit Operations *
2326	*********************************************************************************************************************************/
2327
2328	/**
2329	* Emits code for clearing bits 16 thru 63 in the GPR.
2330	*/
2331	DECL_INLINE_THROW(uint32_t)
2332	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2333	{
2334	#if defined(RT_ARCH_AMD64)
2335	/* movzx Gv,Ew */
2336	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2337	if (iGprDst >= 8)
2338	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2339	pbCodeBuf[off++] = 0x0f;
2340	pbCodeBuf[off++] = 0xb7;
2341	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2342
2343	#elif defined(RT_ARCH_ARM64)
2344	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2345	# if 1
2346	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2347	# else
2348	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2349	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2350	# endif
2351	#else
2352	# error "Port me"
2353	#endif
2354	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2355	return off;
2356	}
2357
2358
2359	/**
2360	* Emits code for AND'ing two 64-bit GPRs.
2361	*
2362	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2363	* and ARM64 hosts.
2364	*/
2365	DECL_INLINE_THROW(uint32_t)
2366	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2367	{
2368	#if defined(RT_ARCH_AMD64)
2369	/* and Gv, Ev */
2370	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2371	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2372	pbCodeBuf[off++] = 0x23;
2373	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2374	RT_NOREF(fSetFlags);
2375
2376	#elif defined(RT_ARCH_ARM64)
2377	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2378	if (!fSetFlags)
2379	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2380	else
2381	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2382
2383	#else
2384	# error "Port me"
2385	#endif
2386	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2387	return off;
2388	}
2389
2390
2391	/**
2392	* Emits code for AND'ing two 32-bit GPRs.
2393	*/
2394	DECL_FORCE_INLINE(uint32_t)
2395	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2396	{
2397	#if defined(RT_ARCH_AMD64)
2398	/* and Gv, Ev */
2399	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2400	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2401	pCodeBuf[off++] = 0x23;
2402	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2403	RT_NOREF(fSetFlags);
2404
2405	#elif defined(RT_ARCH_ARM64)
2406	if (!fSetFlags)
2407	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2408	else
2409	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2410
2411	#else
2412	# error "Port me"
2413	#endif
2414	return off;
2415	}
2416
2417
2418	/**
2419	* Emits code for AND'ing two 32-bit GPRs.
2420	*/
2421	DECL_INLINE_THROW(uint32_t)
2422	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2423	{
2424	#if defined(RT_ARCH_AMD64)
2425	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
2426	#elif defined(RT_ARCH_ARM64)
2427	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
2428	#else
2429	# error "Port me"
2430	#endif
2431	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2432	return off;
2433	}
2434
2435
2436	/**
2437	* Emits code for AND'ing a 64-bit GPRs with a constant.
2438	*
2439	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2440	* and ARM64 hosts.
2441	*/
2442	DECL_INLINE_THROW(uint32_t)
2443	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2444	{
2445	#if defined(RT_ARCH_AMD64)
2446	if ((int64_t)uImm == (int8_t)uImm)
2447	{
2448	/* and Ev, imm8 */
2449	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2450	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2451	pbCodeBuf[off++] = 0x83;
2452	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2453	pbCodeBuf[off++] = (uint8_t)uImm;
2454	}
2455	else if ((int64_t)uImm == (int32_t)uImm)
2456	{
2457	/* and Ev, imm32 */
2458	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2459	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2460	pbCodeBuf[off++] = 0x81;
2461	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2462	pbCodeBuf[off++] = RT_BYTE1(uImm);
2463	pbCodeBuf[off++] = RT_BYTE2(uImm);
2464	pbCodeBuf[off++] = RT_BYTE3(uImm);
2465	pbCodeBuf[off++] = RT_BYTE4(uImm);
2466	}
2467	else
2468	{
2469	/* Use temporary register for the 64-bit immediate. */
2470	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2471	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2472	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2473	}
2474	RT_NOREF(fSetFlags);
2475
2476	#elif defined(RT_ARCH_ARM64)
2477	uint32_t uImmR = 0;
2478	uint32_t uImmNandS = 0;
2479	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
2480	{
2481	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2482	if (!fSetFlags)
2483	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2484	else
2485	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2486	}
2487	else
2488	{
2489	/* Use temporary register for the 64-bit immediate. */
2490	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2491	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2492	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2493	}
2494
2495	#else
2496	# error "Port me"
2497	#endif
2498	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2499	return off;
2500	}
2501
2502
2503	/**
2504	* Emits code for AND'ing an 32-bit GPRs with a constant.
2505	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2506	* @note For ARM64 this only supports @a uImm values that can be expressed using
2507	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
2508	* make sure this is possible!
2509	*/
2510	DECL_FORCE_INLINE_THROW(uint32_t)
2511	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2512	{
2513	#if defined(RT_ARCH_AMD64)
2514	/* and Ev, imm */
2515	if (iGprDst >= 8)
2516	pCodeBuf[off++] = X86_OP_REX_B;
2517	if ((int32_t)uImm == (int8_t)uImm)
2518	{
2519	pCodeBuf[off++] = 0x83;
2520	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2521	pCodeBuf[off++] = (uint8_t)uImm;
2522	}
2523	else
2524	{
2525	pCodeBuf[off++] = 0x81;
2526	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2527	pCodeBuf[off++] = RT_BYTE1(uImm);
2528	pCodeBuf[off++] = RT_BYTE2(uImm);
2529	pCodeBuf[off++] = RT_BYTE3(uImm);
2530	pCodeBuf[off++] = RT_BYTE4(uImm);
2531	}
2532	RT_NOREF(fSetFlags);
2533
2534	#elif defined(RT_ARCH_ARM64)
2535	uint32_t uImmR = 0;
2536	uint32_t uImmNandS = 0;
2537	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2538	{
2539	if (!fSetFlags)
2540	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2541	else
2542	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2543	}
2544	else
2545	# ifdef IEM_WITH_THROW_CATCH
2546	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2547	# else
2548	AssertReleaseFailedStmt(off = UINT32_MAX);
2549	# endif
2550
2551	#else
2552	# error "Port me"
2553	#endif
2554	return off;
2555	}
2556
2557
2558	/**
2559	* Emits code for AND'ing an 32-bit GPRs with a constant.
2560	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2561	*/
2562	DECL_INLINE_THROW(uint32_t)
2563	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2564	{
2565	#if defined(RT_ARCH_AMD64)
2566	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
2567
2568	#elif defined(RT_ARCH_ARM64)
2569	uint32_t uImmR = 0;
2570	uint32_t uImmNandS = 0;
2571	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2572	{
2573	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2574	if (!fSetFlags)
2575	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2576	else
2577	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2578	}
2579	else
2580	{
2581	/* Use temporary register for the 64-bit immediate. */
2582	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2583	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2584	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2585	}
2586
2587	#else
2588	# error "Port me"
2589	#endif
2590	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2591	return off;
2592	}
2593
2594
2595	/**
2596	* Emits code for XOR'ing two 64-bit GPRs.
2597	*/
2598	DECL_INLINE_THROW(uint32_t)
2599	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2600	{
2601	#if defined(RT_ARCH_AMD64)
2602	/* and Gv, Ev */
2603	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2604	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2605	pbCodeBuf[off++] = 0x33;
2606	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2607
2608	#elif defined(RT_ARCH_ARM64)
2609	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2610	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2611
2612	#else
2613	# error "Port me"
2614	#endif
2615	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2616	return off;
2617	}
2618
2619
2620	/**
2621	* Emits code for XOR'ing two 32-bit GPRs.
2622	*/
2623	DECL_INLINE_THROW(uint32_t)
2624	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2625	{
2626	#if defined(RT_ARCH_AMD64)
2627	/* and Gv, Ev */
2628	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2629	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2630	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2631	pbCodeBuf[off++] = 0x33;
2632	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2633
2634	#elif defined(RT_ARCH_ARM64)
2635	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2636	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2637
2638	#else
2639	# error "Port me"
2640	#endif
2641	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2642	return off;
2643	}
2644
2645
2646	/**
2647	* Emits code for XOR'ing an 32-bit GPRs with a constant.
2648	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2649	* @note For ARM64 this only supports @a uImm values that can be expressed using
2650	* the two 6-bit immediates of the EOR instructions. The caller must make
2651	* sure this is possible!
2652	*/
2653	DECL_FORCE_INLINE_THROW(uint32_t)
2654	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
2655	{
2656	#if defined(RT_ARCH_AMD64)
2657	/* and Ev, imm */
2658	if (iGprDst >= 8)
2659	pCodeBuf[off++] = X86_OP_REX_B;
2660	if ((int32_t)uImm == (int8_t)uImm)
2661	{
2662	pCodeBuf[off++] = 0x83;
2663	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
2664	pCodeBuf[off++] = (uint8_t)uImm;
2665	}
2666	else
2667	{
2668	pCodeBuf[off++] = 0x81;
2669	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
2670	pCodeBuf[off++] = RT_BYTE1(uImm);
2671	pCodeBuf[off++] = RT_BYTE2(uImm);
2672	pCodeBuf[off++] = RT_BYTE3(uImm);
2673	pCodeBuf[off++] = RT_BYTE4(uImm);
2674	}
2675
2676	#elif defined(RT_ARCH_ARM64)
2677	uint32_t uImmR = 0;
2678	uint32_t uImmNandS = 0;
2679	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2680	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2681	else
2682	# ifdef IEM_WITH_THROW_CATCH
2683	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2684	# else
2685	AssertReleaseFailedStmt(off = UINT32_MAX);
2686	# endif
2687
2688	#else
2689	# error "Port me"
2690	#endif
2691	return off;
2692	}
2693
2694
2695	/*********************************************************************************************************************************
2696	* Shifting *
2697	*********************************************************************************************************************************/
2698
2699	/**
2700	* Emits code for shifting a GPR a fixed number of bits to the left.
2701	*/
2702	DECL_FORCE_INLINE(uint32_t)
2703	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2704	{
2705	Assert(cShift > 0 && cShift < 64);
2706
2707	#if defined(RT_ARCH_AMD64)
2708	/* shl dst, cShift */
2709	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2710	if (cShift != 1)
2711	{
2712	pCodeBuf[off++] = 0xc1;
2713	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2714	pCodeBuf[off++] = cShift;
2715	}
2716	else
2717	{
2718	pCodeBuf[off++] = 0xd1;
2719	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2720	}
2721
2722	#elif defined(RT_ARCH_ARM64)
2723	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2724
2725	#else
2726	# error "Port me"
2727	#endif
2728	return off;
2729	}
2730
2731
2732	/**
2733	* Emits code for shifting a GPR a fixed number of bits to the left.
2734	*/
2735	DECL_INLINE_THROW(uint32_t)
2736	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2737	{
2738	#if defined(RT_ARCH_AMD64)
2739	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2740	#elif defined(RT_ARCH_ARM64)
2741	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2742	#else
2743	# error "Port me"
2744	#endif
2745	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2746	return off;
2747	}
2748
2749
2750	/**
2751	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2752	*/
2753	DECL_FORCE_INLINE(uint32_t)
2754	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2755	{
2756	Assert(cShift > 0 && cShift < 32);
2757
2758	#if defined(RT_ARCH_AMD64)
2759	/* shl dst, cShift */
2760	if (iGprDst >= 8)
2761	pCodeBuf[off++] = X86_OP_REX_B;
2762	if (cShift != 1)
2763	{
2764	pCodeBuf[off++] = 0xc1;
2765	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2766	pCodeBuf[off++] = cShift;
2767	}
2768	else
2769	{
2770	pCodeBuf[off++] = 0xd1;
2771	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2772	}
2773
2774	#elif defined(RT_ARCH_ARM64)
2775	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2776
2777	#else
2778	# error "Port me"
2779	#endif
2780	return off;
2781	}
2782
2783
2784	/**
2785	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2786	*/
2787	DECL_INLINE_THROW(uint32_t)
2788	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2789	{
2790	#if defined(RT_ARCH_AMD64)
2791	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2792	#elif defined(RT_ARCH_ARM64)
2793	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2794	#else
2795	# error "Port me"
2796	#endif
2797	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2798	return off;
2799	}
2800
2801
2802	/**
2803	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2804	*/
2805	DECL_FORCE_INLINE(uint32_t)
2806	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2807	{
2808	Assert(cShift > 0 && cShift < 64);
2809
2810	#if defined(RT_ARCH_AMD64)
2811	/* shr dst, cShift */
2812	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2813	if (cShift != 1)
2814	{
2815	pCodeBuf[off++] = 0xc1;
2816	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2817	pCodeBuf[off++] = cShift;
2818	}
2819	else
2820	{
2821	pCodeBuf[off++] = 0xd1;
2822	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2823	}
2824
2825	#elif defined(RT_ARCH_ARM64)
2826	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2827
2828	#else
2829	# error "Port me"
2830	#endif
2831	return off;
2832	}
2833
2834
2835	/**
2836	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2837	*/
2838	DECL_INLINE_THROW(uint32_t)
2839	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2840	{
2841	#if defined(RT_ARCH_AMD64)
2842	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2843	#elif defined(RT_ARCH_ARM64)
2844	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2845	#else
2846	# error "Port me"
2847	#endif
2848	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2849	return off;
2850	}
2851
2852
2853	/**
2854	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2855	* right.
2856	*/
2857	DECL_FORCE_INLINE(uint32_t)
2858	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2859	{
2860	Assert(cShift > 0 && cShift < 32);
2861
2862	#if defined(RT_ARCH_AMD64)
2863	/* shr dst, cShift */
2864	if (iGprDst >= 8)
2865	pCodeBuf[off++] = X86_OP_REX_B;
2866	if (cShift != 1)
2867	{
2868	pCodeBuf[off++] = 0xc1;
2869	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2870	pCodeBuf[off++] = cShift;
2871	}
2872	else
2873	{
2874	pCodeBuf[off++] = 0xd1;
2875	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2876	}
2877
2878	#elif defined(RT_ARCH_ARM64)
2879	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2880
2881	#else
2882	# error "Port me"
2883	#endif
2884	return off;
2885	}
2886
2887
2888	/**
2889	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2890	* right.
2891	*/
2892	DECL_INLINE_THROW(uint32_t)
2893	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2894	{
2895	#if defined(RT_ARCH_AMD64)
2896	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2897	#elif defined(RT_ARCH_ARM64)
2898	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2899	#else
2900	# error "Port me"
2901	#endif
2902	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2903	return off;
2904	}
2905
2906
2907	/**
2908	* Emits code for rotating a GPR a fixed number of bits to the left.
2909	*/
2910	DECL_FORCE_INLINE(uint32_t)
2911	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2912	{
2913	Assert(cShift > 0 && cShift < 64);
2914
2915	#if defined(RT_ARCH_AMD64)
2916	/* rol dst, cShift */
2917	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2918	if (cShift != 1)
2919	{
2920	pCodeBuf[off++] = 0xc1;
2921	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2922	pCodeBuf[off++] = cShift;
2923	}
2924	else
2925	{
2926	pCodeBuf[off++] = 0xd1;
2927	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2928	}
2929
2930	#elif defined(RT_ARCH_ARM64)
2931	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
2932
2933	#else
2934	# error "Port me"
2935	#endif
2936	return off;
2937	}
2938
2939
2940
2941	/*********************************************************************************************************************************
2942	* Compare and Testing *
2943	*********************************************************************************************************************************/
2944
2945
2946	#ifdef RT_ARCH_ARM64
2947	/**
2948	* Emits an ARM64 compare instruction.
2949	*/
2950	DECL_INLINE_THROW(uint32_t)
2951	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2952	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2953	{
2954	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2955	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2956	f64Bit, true /fSetFlags/, cShift, enmShift);
2957	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2958	return off;
2959	}
2960	#endif
2961
2962
2963	/**
2964	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2965	* with conditional instruction.
2966	*/
2967	DECL_FORCE_INLINE(uint32_t)
2968	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2969	{
2970	#ifdef RT_ARCH_AMD64
2971	/* cmp Gv, Ev */
2972	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2973	pCodeBuf[off++] = 0x3b;
2974	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2975
2976	#elif defined(RT_ARCH_ARM64)
2977	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
2978
2979	#else
2980	# error "Port me!"
2981	#endif
2982	return off;
2983	}
2984
2985
2986	/**
2987	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2988	* with conditional instruction.
2989	*/
2990	DECL_INLINE_THROW(uint32_t)
2991	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2992	{
2993	#ifdef RT_ARCH_AMD64
2994	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
2995	#elif defined(RT_ARCH_ARM64)
2996	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
2997	#else
2998	# error "Port me!"
2999	#endif
3000	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3001	return off;
3002	}
3003
3004
3005	/**
3006	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
3007	* with conditional instruction.
3008	*/
3009	DECL_FORCE_INLINE(uint32_t)
3010	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
3011	{
3012	#ifdef RT_ARCH_AMD64
3013	/* cmp Gv, Ev */
3014	if (iGprLeft >= 8 \|\| iGprRight >= 8)
3015	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
3016	pCodeBuf[off++] = 0x3b;
3017	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
3018
3019	#elif defined(RT_ARCH_ARM64)
3020	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
3021
3022	#else
3023	# error "Port me!"
3024	#endif
3025	return off;
3026	}
3027
3028
3029	/**
3030	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
3031	* with conditional instruction.
3032	*/
3033	DECL_INLINE_THROW(uint32_t)
3034	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
3035	{
3036	#ifdef RT_ARCH_AMD64
3037	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
3038	#elif defined(RT_ARCH_ARM64)
3039	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
3040	#else
3041	# error "Port me!"
3042	#endif
3043	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3044	return off;
3045	}
3046
3047
3048	/**
3049	* Emits a compare of a 64-bit GPR with a constant value, settings status
3050	* flags/whatever for use with conditional instruction.
3051	*/
3052	DECL_INLINE_THROW(uint32_t)
3053	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
3054	{
3055	#ifdef RT_ARCH_AMD64
3056	if (uImm <= UINT32_C(0xff))
3057	{
3058	/* cmp Ev, Ib */
3059	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3060	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
3061	pbCodeBuf[off++] = 0x83;
3062	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3063	pbCodeBuf[off++] = (uint8_t)uImm;
3064	}
3065	else if ((int64_t)uImm == (int32_t)uImm)
3066	{
3067	/* cmp Ev, imm */
3068	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3069	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
3070	pbCodeBuf[off++] = 0x81;
3071	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3072	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3073	pbCodeBuf[off++] = RT_BYTE1(uImm);
3074	pbCodeBuf[off++] = RT_BYTE2(uImm);
3075	pbCodeBuf[off++] = RT_BYTE3(uImm);
3076	pbCodeBuf[off++] = RT_BYTE4(uImm);
3077	}
3078	else
3079	{
3080	/* Use temporary register for the immediate. */
3081	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3082	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
3083	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3084	}
3085
3086	#elif defined(RT_ARCH_ARM64)
3087	/** @todo guess there are clevere things we can do here... */
3088	if (uImm < _4K)
3089	{
3090	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3091	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3092	true /64Bit/, true /fSetFlags/);
3093	}
3094	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3095	{
3096	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3097	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
3098	true /64Bit/, true /fSetFlags/, true /fShift12/);
3099	}
3100	else
3101	{
3102	/* Use temporary register for the immediate. */
3103	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3104	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
3105	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3106	}
3107
3108	#else
3109	# error "Port me!"
3110	#endif
3111
3112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3113	return off;
3114	}
3115
3116
3117	/**
3118	* Emits a compare of a 32-bit GPR with a constant value, settings status
3119	* flags/whatever for use with conditional instruction.
3120	*
3121	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
3122	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
3123	* bits all zero). Will release assert or throw exception if the caller
3124	* violates this restriction.
3125	*/
3126	DECL_FORCE_INLINE_THROW(uint32_t)
3127	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
3128	{
3129	#ifdef RT_ARCH_AMD64
3130	if (iGprLeft >= 8)
3131	pCodeBuf[off++] = X86_OP_REX_B;
3132	if (uImm <= UINT32_C(0x7f))
3133	{
3134	/* cmp Ev, Ib */
3135	pCodeBuf[off++] = 0x83;
3136	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3137	pCodeBuf[off++] = (uint8_t)uImm;
3138	}
3139	else
3140	{
3141	/* cmp Ev, imm */
3142	pCodeBuf[off++] = 0x81;
3143	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3144	pCodeBuf[off++] = RT_BYTE1(uImm);
3145	pCodeBuf[off++] = RT_BYTE2(uImm);
3146	pCodeBuf[off++] = RT_BYTE3(uImm);
3147	pCodeBuf[off++] = RT_BYTE4(uImm);
3148	}
3149
3150	#elif defined(RT_ARCH_ARM64)
3151	/** @todo guess there are clevere things we can do here... */
3152	if (uImm < _4K)
3153	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3154	false /64Bit/, true /fSetFlags/);
3155	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3156	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3157	false /64Bit/, true /fSetFlags/, true /fShift12/);
3158	else
3159	# ifdef IEM_WITH_THROW_CATCH
3160	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3161	# else
3162	AssertReleaseFailedStmt(off = UINT32_MAX);
3163	# endif
3164
3165	#else
3166	# error "Port me!"
3167	#endif
3168	return off;
3169	}
3170
3171
3172	/**
3173	* Emits a compare of a 32-bit GPR with a constant value, settings status
3174	* flags/whatever for use with conditional instruction.
3175	*/
3176	DECL_INLINE_THROW(uint32_t)
3177	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
3178	{
3179	#ifdef RT_ARCH_AMD64
3180	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
3181
3182	#elif defined(RT_ARCH_ARM64)
3183	/** @todo guess there are clevere things we can do here... */
3184	if (uImm < _4K)
3185	{
3186	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3187	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3188	false /64Bit/, true /fSetFlags/);
3189	}
3190	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3191	{
3192	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3193	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3194	false /64Bit/, true /fSetFlags/, true /fShift12/);
3195	}
3196	else
3197	{
3198	/* Use temporary register for the immediate. */
3199	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3200	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
3201	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3202	}
3203
3204	#else
3205	# error "Port me!"
3206	#endif
3207
3208	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3209	return off;
3210	}
3211
3212
3213
3214	/*********************************************************************************************************************************
3215	* Branching *
3216	*********************************************************************************************************************************/
3217
3218	/**
3219	* Emits a JMP rel32 / B imm19 to the given label.
3220	*/
3221	DECL_FORCE_INLINE_THROW(uint32_t)
3222	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
3223	{
3224	Assert(idxLabel < pReNative->cLabels);
3225
3226	#ifdef RT_ARCH_AMD64
3227	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
3228	{
3229	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
3230	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
3231	{
3232	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
3233	pCodeBuf[off++] = (uint8_t)offRel;
3234	}
3235	else
3236	{
3237	offRel -= 3;
3238	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
3239	pCodeBuf[off++] = RT_BYTE1(offRel);
3240	pCodeBuf[off++] = RT_BYTE2(offRel);
3241	pCodeBuf[off++] = RT_BYTE3(offRel);
3242	pCodeBuf[off++] = RT_BYTE4(offRel);
3243	}
3244	}
3245	else
3246	{
3247	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
3248	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
3249	pCodeBuf[off++] = 0xfe;
3250	pCodeBuf[off++] = 0xff;
3251	pCodeBuf[off++] = 0xff;
3252	pCodeBuf[off++] = 0xff;
3253	}
3254	pCodeBuf[off++] = 0xcc; /* int3 poison */
3255
3256	#elif defined(RT_ARCH_ARM64)
3257	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
3258	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
3259	else
3260	{
3261	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
3262	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
3263	}
3264
3265	#else
3266	# error "Port me!"
3267	#endif
3268	return off;
3269	}
3270
3271
3272	/**
3273	* Emits a JMP rel32 / B imm19 to the given label.
3274	*/
3275	DECL_INLINE_THROW(uint32_t)
3276	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3277	{
3278	#ifdef RT_ARCH_AMD64
3279	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
3280	#elif defined(RT_ARCH_ARM64)
3281	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
3282	#else
3283	# error "Port me!"
3284	#endif
3285	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3286	return off;
3287	}
3288
3289
3290	/**
3291	* Emits a JMP rel32 / B imm19 to a new undefined label.
3292	*/
3293	DECL_INLINE_THROW(uint32_t)
3294	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3295	{
3296	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3297	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
3298	}
3299
3300	/** Condition type. */
3301	#ifdef RT_ARCH_AMD64
3302	typedef enum IEMNATIVEINSTRCOND : uint8_t
3303	{
3304	kIemNativeInstrCond_o = 0,
3305	kIemNativeInstrCond_no,
3306	kIemNativeInstrCond_c,
3307	kIemNativeInstrCond_nc,
3308	kIemNativeInstrCond_e,
3309	kIemNativeInstrCond_ne,
3310	kIemNativeInstrCond_be,
3311	kIemNativeInstrCond_nbe,
3312	kIemNativeInstrCond_s,
3313	kIemNativeInstrCond_ns,
3314	kIemNativeInstrCond_p,
3315	kIemNativeInstrCond_np,
3316	kIemNativeInstrCond_l,
3317	kIemNativeInstrCond_nl,
3318	kIemNativeInstrCond_le,
3319	kIemNativeInstrCond_nle
3320	} IEMNATIVEINSTRCOND;
3321	#elif defined(RT_ARCH_ARM64)
3322	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
3323	#else
3324	# error "Port me!"
3325	#endif
3326
3327
3328	/**
3329	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
3330	*/
3331	DECL_FORCE_INLINE_THROW(uint32_t)
3332	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
3333	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
3334	{
3335	Assert(idxLabel < pReNative->cLabels);
3336
3337	uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
3338	#ifdef RT_ARCH_AMD64
3339	if (offLabel >= off)
3340	{
3341	/* jcc rel32 */
3342	pCodeBuf[off++] = 0x0f;
3343	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
3344	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
3345	pCodeBuf[off++] = 0x00;
3346	pCodeBuf[off++] = 0x00;
3347	pCodeBuf[off++] = 0x00;
3348	pCodeBuf[off++] = 0x00;
3349	}
3350	else
3351	{
3352	int32_t offDisp = offLabel - (off + 2);
3353	if ((int8_t)offDisp == offDisp)
3354	{
3355	/* jcc rel8 */
3356	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
3357	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3358	}
3359	else
3360	{
3361	/* jcc rel32 */
3362	offDisp -= 4;
3363	pCodeBuf[off++] = 0x0f;
3364	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
3365	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3366	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3367	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3368	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3369	}
3370	}
3371
3372	#elif defined(RT_ARCH_ARM64)
3373	if (offLabel >= off)
3374	{
3375	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3376	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
3377	}
3378	else
3379	{
3380	Assert(offLabel - off <= -0x3ffff);
3381	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
3382	}
3383
3384	#else
3385	# error "Port me!"
3386	#endif
3387	return off;
3388	}
3389
3390
3391	/**
3392	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
3393	*/
3394	DECL_INLINE_THROW(uint32_t)
3395	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
3396	{
3397	#ifdef RT_ARCH_AMD64
3398	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
3399	#elif defined(RT_ARCH_ARM64)
3400	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
3401	#else
3402	# error "Port me!"
3403	#endif
3404	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3405	return off;
3406	}
3407
3408
3409	/**
3410	* Emits a Jcc rel32 / B.cc imm19 to a new label.
3411	*/
3412	DECL_INLINE_THROW(uint32_t)
3413	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3414	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
3415	{
3416	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3417	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
3418	}
3419
3420
3421	/**
3422	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
3423	*/
3424	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3425	{
3426	#ifdef RT_ARCH_AMD64
3427	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
3428	#elif defined(RT_ARCH_ARM64)
3429	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
3430	#else
3431	# error "Port me!"
3432	#endif
3433	}
3434
3435	/**
3436	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
3437	*/
3438	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3439	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3440	{
3441	#ifdef RT_ARCH_AMD64
3442	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
3443	#elif defined(RT_ARCH_ARM64)
3444	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
3445	#else
3446	# error "Port me!"
3447	#endif
3448	}
3449
3450
3451	/**
3452	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
3453	*/
3454	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3455	{
3456	#ifdef RT_ARCH_AMD64
3457	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
3458	#elif defined(RT_ARCH_ARM64)
3459	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
3460	#else
3461	# error "Port me!"
3462	#endif
3463	}
3464
3465	/**
3466	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
3467	*/
3468	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3469	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3470	{
3471	#ifdef RT_ARCH_AMD64
3472	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
3473	#elif defined(RT_ARCH_ARM64)
3474	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
3475	#else
3476	# error "Port me!"
3477	#endif
3478	}
3479
3480
3481	/**
3482	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
3483	*/
3484	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3485	{
3486	#ifdef RT_ARCH_AMD64
3487	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
3488	#elif defined(RT_ARCH_ARM64)
3489	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
3490	#else
3491	# error "Port me!"
3492	#endif
3493	}
3494
3495	/**
3496	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
3497	*/
3498	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3499	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3500	{
3501	#ifdef RT_ARCH_AMD64
3502	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
3503	#elif defined(RT_ARCH_ARM64)
3504	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
3505	#else
3506	# error "Port me!"
3507	#endif
3508	}
3509
3510
3511	/**
3512	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
3513	*/
3514	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3515	{
3516	#ifdef RT_ARCH_AMD64
3517	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
3518	#elif defined(RT_ARCH_ARM64)
3519	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
3520	#else
3521	# error "Port me!"
3522	#endif
3523	}
3524
3525	/**
3526	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
3527	*/
3528	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3529	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3530	{
3531	#ifdef RT_ARCH_AMD64
3532	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
3533	#elif defined(RT_ARCH_ARM64)
3534	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
3535	#else
3536	# error "Port me!"
3537	#endif
3538	}
3539
3540
3541	/**
3542	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
3543	*/
3544	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3545	{
3546	#ifdef RT_ARCH_AMD64
3547	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
3548	#elif defined(RT_ARCH_ARM64)
3549	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
3550	#else
3551	# error "Port me!"
3552	#endif
3553	}
3554
3555	/**
3556	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
3557	*/
3558	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3559	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3560	{
3561	#ifdef RT_ARCH_AMD64
3562	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
3563	#elif defined(RT_ARCH_ARM64)
3564	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
3565	#else
3566	# error "Port me!"
3567	#endif
3568	}
3569
3570
3571	/**
3572	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
3573	*
3574	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
3575	*
3576	* Only use hardcoded jumps forward when emitting for exactly one
3577	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
3578	* the right target address on all platforms!
3579	*
3580	* Please also note that on x86 it is necessary pass off + 256 or higher
3581	* for @a offTarget one believe the intervening code is more than 127
3582	* bytes long.
3583	*/
3584	DECL_FORCE_INLINE(uint32_t)
3585	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
3586	{
3587	#ifdef RT_ARCH_AMD64
3588	/* jcc rel8 / rel32 */
3589	int32_t offDisp = (int32_t)(offTarget - (off + 2));
3590	if (offDisp < 128 && offDisp >= -128)
3591	{
3592	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
3593	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3594	}
3595	else
3596	{
3597	offDisp -= 4;
3598	pCodeBuf[off++] = 0x0f;
3599	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
3600	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3601	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3602	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3603	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3604	}
3605
3606	#elif defined(RT_ARCH_ARM64)
3607	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
3608
3609	#else
3610	# error "Port me!"
3611	#endif
3612	return off;
3613	}
3614
3615
3616	/**
3617	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
3618	*
3619	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
3620	*
3621	* Only use hardcoded jumps forward when emitting for exactly one
3622	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
3623	* the right target address on all platforms!
3624	*
3625	* Please also note that on x86 it is necessary pass off + 256 or higher
3626	* for @a offTarget one believe the intervening code is more than 127
3627	* bytes long.
3628	*/
3629	DECL_INLINE_THROW(uint32_t)
3630	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
3631	{
3632	#ifdef RT_ARCH_AMD64
3633	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
3634	#elif defined(RT_ARCH_ARM64)
3635	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
3636	#else
3637	# error "Port me!"
3638	#endif
3639	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3640	return off;
3641	}
3642
3643
3644	/**
3645	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
3646	*
3647	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3648	*/
3649	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3650	{
3651	#ifdef RT_ARCH_AMD64
3652	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
3653	#elif defined(RT_ARCH_ARM64)
3654	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
3655	#else
3656	# error "Port me!"
3657	#endif
3658	}
3659
3660
3661	/**
3662	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
3663	*
3664	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3665	*/
3666	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3667	{
3668	#ifdef RT_ARCH_AMD64
3669	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
3670	#elif defined(RT_ARCH_ARM64)
3671	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
3672	#else
3673	# error "Port me!"
3674	#endif
3675	}
3676
3677
3678	/**
3679	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
3680	*
3681	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3682	*/
3683	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3684	{
3685	#ifdef RT_ARCH_AMD64
3686	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
3687	#elif defined(RT_ARCH_ARM64)
3688	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
3689	#else
3690	# error "Port me!"
3691	#endif
3692	}
3693
3694
3695	/**
3696	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
3697	*
3698	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3699	*/
3700	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3701	{
3702	#ifdef RT_ARCH_AMD64
3703	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3704	#elif defined(RT_ARCH_ARM64)
3705	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3706	#else
3707	# error "Port me!"
3708	#endif
3709	}
3710
3711
3712	/**
3713	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
3714	*
3715	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3716	*/
3717	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
3718	{
3719	#ifdef RT_ARCH_AMD64
3720	/* jmp rel8 or rel32 */
3721	int32_t offDisp = offTarget - (off + 2);
3722	if (offDisp < 128 && offDisp >= -128)
3723	{
3724	pCodeBuf[off++] = 0xeb;
3725	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3726	}
3727	else
3728	{
3729	offDisp -= 3;
3730	pCodeBuf[off++] = 0xe9;
3731	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3732	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3733	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3734	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3735	}
3736
3737	#elif defined(RT_ARCH_ARM64)
3738	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
3739
3740	#else
3741	# error "Port me!"
3742	#endif
3743	return off;
3744	}
3745
3746
3747	/**
3748	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
3749	*
3750	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3751	*/
3752	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3753	{
3754	#ifdef RT_ARCH_AMD64
3755	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
3756	#elif defined(RT_ARCH_ARM64)
3757	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
3758	#else
3759	# error "Port me!"
3760	#endif
3761	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3762	return off;
3763	}
3764
3765
3766	/**
3767	* Fixes up a conditional jump to a fixed label.
3768	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
3769	* iemNativeEmitJzToFixed, ...
3770	*/
3771	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3772	{
3773	#ifdef RT_ARCH_AMD64
3774	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3775	uint8_t const bOpcode = pbCodeBuf[offFixup];
3776	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
3777	{
3778	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3779	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
3780	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
3781	}
3782	else
3783	{
3784	if (bOpcode != 0x0f)
3785	Assert(bOpcode == 0xe9);
3786	else
3787	{
3788	offFixup += 1;
3789	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
3790	}
3791	uint32_t const offRel32 = offTarget - (offFixup + 5);
3792	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
3793	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
3794	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
3795	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
3796	}
3797
3798	#elif defined(RT_ARCH_ARM64)
3799	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3800	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
3801	{
3802	/* B.COND + BC.COND */
3803	int32_t const offDisp = offTarget - offFixup;
3804	Assert(offDisp >= -262144 && offDisp < 262144);
3805	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
3806	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
3807	}
3808	else
3809	{
3810	/* B imm26 */
3811	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
3812	int32_t const offDisp = offTarget - offFixup;
3813	Assert(offDisp >= -33554432 && offDisp < 33554432);
3814	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
3815	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
3816	}
3817
3818	#else
3819	# error "Port me!"
3820	#endif
3821	}
3822
3823
3824	/**
3825	* Internal helper, don't call directly.
3826	*/
3827	DECL_INLINE_THROW(uint32_t)
3828	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3829	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3830	{
3831	Assert(iBitNo < 64);
3832	#ifdef RT_ARCH_AMD64
3833	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3834	if (iBitNo < 8)
3835	{
3836	/* test Eb, imm8 */
3837	if (iGprSrc >= 4)
3838	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3839	pbCodeBuf[off++] = 0xf6;
3840	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3841	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3842	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3843	}
3844	else
3845	{
3846	/* bt Ev, imm8 */
3847	if (iBitNo >= 32)
3848	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3849	else if (iGprSrc >= 8)
3850	pbCodeBuf[off++] = X86_OP_REX_B;
3851	pbCodeBuf[off++] = 0x0f;
3852	pbCodeBuf[off++] = 0xba;
3853	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3854	pbCodeBuf[off++] = iBitNo;
3855	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3856	}
3857
3858	#elif defined(RT_ARCH_ARM64)
3859	/* Use the TBNZ instruction here. */
3860	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3861	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3862	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3863
3864	#else
3865	# error "Port me!"
3866	#endif
3867	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3868	return off;
3869	}
3870
3871
3872	/**
3873	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3874	* @a iGprSrc.
3875	*
3876	* @note On ARM64 the range is only +/-8191 instructions.
3877	*/
3878	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3879	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3880	{
3881	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
3882	}
3883
3884
3885	/**
3886	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3887	* _set_ in @a iGprSrc.
3888	*
3889	* @note On ARM64 the range is only +/-8191 instructions.
3890	*/
3891	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3892	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3893	{
3894	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
3895	}
3896
3897
3898	/**
3899	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3900	* flags accordingly.
3901	*/
3902	DECL_INLINE_THROW(uint32_t)
3903	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3904	{
3905	Assert(fBits != 0);
3906	#ifdef RT_ARCH_AMD64
3907
3908	if (fBits >= UINT32_MAX)
3909	{
3910	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3911
3912	/* test Ev,Gv */
3913	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3914	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3915	pbCodeBuf[off++] = 0x85;
3916	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3917
3918	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3919	}
3920	else if (fBits <= UINT32_MAX)
3921	{
3922	/* test Eb, imm8 or test Ev, imm32 */
3923	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3924	if (fBits <= UINT8_MAX)
3925	{
3926	if (iGprSrc >= 4)
3927	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3928	pbCodeBuf[off++] = 0xf6;
3929	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3930	pbCodeBuf[off++] = (uint8_t)fBits;
3931	}
3932	else
3933	{
3934	if (iGprSrc >= 8)
3935	pbCodeBuf[off++] = X86_OP_REX_B;
3936	pbCodeBuf[off++] = 0xf7;
3937	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3938	pbCodeBuf[off++] = RT_BYTE1(fBits);
3939	pbCodeBuf[off++] = RT_BYTE2(fBits);
3940	pbCodeBuf[off++] = RT_BYTE3(fBits);
3941	pbCodeBuf[off++] = RT_BYTE4(fBits);
3942	}
3943	}
3944	/** @todo implement me. */
3945	else
3946	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3947
3948	#elif defined(RT_ARCH_ARM64)
3949
3950	if (false)
3951	{
3952	/** @todo figure out how to work the immr / N:imms constants. */
3953	}
3954	else
3955	{
3956	/* ands Zr, iGprSrc, iTmpReg */
3957	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3958	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3959	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3960	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3961	}
3962
3963	#else
3964	# error "Port me!"
3965	#endif
3966	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3967	return off;
3968	}
3969
3970
3971	/**
3972	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
3973	* @a iGprSrc, setting CPU flags accordingly.
3974	*
3975	* @note For ARM64 this only supports @a fBits values that can be expressed
3976	* using the two 6-bit immediates of the ANDS instruction. The caller
3977	* must make sure this is possible!
3978	*/
3979	DECL_FORCE_INLINE_THROW(uint32_t)
3980	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
3981	{
3982	Assert(fBits != 0);
3983
3984	/* test Eb, imm8 */
3985	#ifdef RT_ARCH_AMD64
3986	if (iGprSrc >= 4)
3987	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3988	pCodeBuf[off++] = 0xf6;
3989	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3990	pCodeBuf[off++] = fBits;
3991
3992	#elif defined(RT_ARCH_ARM64)
3993	/* ands xzr, src, [tmp\|#imm] */
3994	uint32_t uImmR = 0;
3995	uint32_t uImmNandS = 0;
3996	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
3997	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3998	else
3999	# ifdef IEM_WITH_THROW_CATCH
4000	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
4001	# else
4002	AssertReleaseFailedStmt(off = UINT32_MAX);
4003	# endif
4004
4005	#else
4006	# error "Port me!"
4007	#endif
4008	return off;
4009	}
4010
4011
4012	/**
4013	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
4014	* @a iGprSrc, setting CPU flags accordingly.
4015	*/
4016	DECL_INLINE_THROW(uint32_t)
4017	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
4018	{
4019	Assert(fBits != 0);
4020
4021	#ifdef RT_ARCH_AMD64
4022	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
4023
4024	#elif defined(RT_ARCH_ARM64)
4025	/* ands xzr, src, [tmp\|#imm] */
4026	uint32_t uImmR = 0;
4027	uint32_t uImmNandS = 0;
4028	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
4029	{
4030	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4031	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
4032	}
4033	else
4034	{
4035	/* Use temporary register for the 64-bit immediate. */
4036	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
4037	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4038	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
4039	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
4040	}
4041
4042	#else
4043	# error "Port me!"
4044	#endif
4045	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4046	return off;
4047	}
4048
4049
4050	/**
4051	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
4052	* are set in @a iGprSrc.
4053	*/
4054	DECL_INLINE_THROW(uint32_t)
4055	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4056	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
4057	{
4058	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
4059
4060	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
4061	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4062
4063	return off;
4064	}
4065
4066
4067	/**
4068	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
4069	* are set in @a iGprSrc.
4070	*/
4071	DECL_INLINE_THROW(uint32_t)
4072	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4073	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
4074	{
4075	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
4076
4077	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
4078	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
4079
4080	return off;
4081	}
4082
4083
4084	/**
4085	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
4086	*
4087	* The operand size is given by @a f64Bit.
4088	*/
4089	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4090	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
4091	{
4092	Assert(idxLabel < pReNative->cLabels);
4093
4094	#ifdef RT_ARCH_AMD64
4095	/* test reg32,reg32 / test reg64,reg64 */
4096	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4097	if (f64Bit)
4098	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
4099	else if (iGprSrc >= 8)
4100	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
4101	pbCodeBuf[off++] = 0x85;
4102	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
4103	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4104
4105	/* jz idxLabel */
4106	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
4107
4108	#elif defined(RT_ARCH_ARM64)
4109	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4110	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4111	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
4112	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4113
4114	#else
4115	# error "Port me!"
4116	#endif
4117	return off;
4118	}
4119
4120
4121	/**
4122	* Emits code that jumps to a new label if @a iGprSrc is zero.
4123	*
4124	* The operand size is given by @a f64Bit.
4125	*/
4126	DECL_INLINE_THROW(uint32_t)
4127	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
4128	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4129	{
4130	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4131	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
4132	}
4133
4134
4135	/**
4136	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
4137	*
4138	* The operand size is given by @a f64Bit.
4139	*/
4140	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4141	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
4142	{
4143	Assert(idxLabel < pReNative->cLabels);
4144
4145	#ifdef RT_ARCH_AMD64
4146	/* test reg32,reg32 / test reg64,reg64 */
4147	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4148	if (f64Bit)
4149	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
4150	else if (iGprSrc >= 8)
4151	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
4152	pbCodeBuf[off++] = 0x85;
4153	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
4154	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4155
4156	/* jnz idxLabel */
4157	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4158
4159	#elif defined(RT_ARCH_ARM64)
4160	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4161	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4162	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(true /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
4163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4164
4165	#else
4166	# error "Port me!"
4167	#endif
4168	return off;
4169	}
4170
4171
4172	/**
4173	* Emits code that jumps to a new label if @a iGprSrc is not zero.
4174	*
4175	* The operand size is given by @a f64Bit.
4176	*/
4177	DECL_INLINE_THROW(uint32_t)
4178	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
4179	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4180	{
4181	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4182	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
4183	}
4184
4185
4186	/**
4187	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
4188	* differs.
4189	*/
4190	DECL_INLINE_THROW(uint32_t)
4191	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4192	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
4193	{
4194	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
4195	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4196	return off;
4197	}
4198
4199
4200	/**
4201	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
4202	*/
4203	DECL_INLINE_THROW(uint32_t)
4204	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4205	uint8_t iGprLeft, uint8_t iGprRight,
4206	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4207	{
4208	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4209	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
4210	}
4211
4212
4213	/**
4214	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
4215	*/
4216	DECL_INLINE_THROW(uint32_t)
4217	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4218	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
4219	{
4220	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
4221	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4222	return off;
4223	}
4224
4225
4226	/**
4227	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
4228	*/
4229	DECL_INLINE_THROW(uint32_t)
4230	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4231	uint8_t iGprSrc, uint64_t uImm,
4232	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4233	{
4234	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4235	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
4236	}
4237
4238
4239	/**
4240	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
4241	* @a uImm.
4242	*/
4243	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4244	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
4245	{
4246	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
4247	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4248	return off;
4249	}
4250
4251
4252	/**
4253	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
4254	* @a uImm.
4255	*/
4256	DECL_INLINE_THROW(uint32_t)
4257	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4258	uint8_t iGprSrc, uint32_t uImm,
4259	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4260	{
4261	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4262	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
4263	}
4264
4265
4266	/*********************************************************************************************************************************
4267	* Calls. *
4268	*********************************************************************************************************************************/
4269
4270	/**
4271	* Emits a call to a 64-bit address.
4272	*/
4273	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
4274	{
4275	#ifdef RT_ARCH_AMD64
4276	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
4277
4278	/* call rax */
4279	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
4280	pbCodeBuf[off++] = 0xff;
4281	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
4282
4283	#elif defined(RT_ARCH_ARM64)
4284	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
4285
4286	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4287	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
4288
4289	#else
4290	# error "port me"
4291	#endif
4292	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4293	return off;
4294	}
4295
4296
4297	/**
4298	* Emits code to load a stack variable into an argument GPR.
4299	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
4300	*/
4301	DECL_FORCE_INLINE_THROW(uint32_t)
4302	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4303	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
4304	{
4305	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4306	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
4307	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
4308
4309	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
4310	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
4311	{
4312	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK) \|\| fVarAllowInVolatileReg);
4313	RT_NOREF(fVarAllowInVolatileReg);
4314	if (!offAddend)
4315	{
4316	if (idxRegArg != idxRegVar)
4317	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
4318	}
4319	else
4320	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
4321	}
4322	else
4323	{
4324	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
4325	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
4326	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
4327	if (offAddend)
4328	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
4329	}
4330	return off;
4331	}
4332
4333
4334	/**
4335	* Emits code to load a stack or immediate variable value into an argument GPR,
4336	* optional with a addend.
4337	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
4338	*/
4339	DECL_FORCE_INLINE_THROW(uint32_t)
4340	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4341	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
4342	{
4343	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4344	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
4345	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
4346	else
4347	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend, fVarAllowInVolatileReg);
4348	return off;
4349	}
4350
4351
4352	/**
4353	* Emits code to load the variable address into an argument GRP.
4354	*
4355	* This only works for uninitialized and stack variables.
4356	*/
4357	DECL_FORCE_INLINE_THROW(uint32_t)
4358	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4359	bool fFlushShadows)
4360	{
4361	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4362	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
4363	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
4364	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
4365
4366	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
4367	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
4368
4369	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
4370	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
4371	{
4372	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
4373	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
4374	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
4375	}
4376	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
4377	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
4378
4379	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
4380	}
4381
4382
4383	/** @} */
4384
4385	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
4386

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102724

Download in other formats: