IEMN8veRecompilerEmit.h@ 102720

Last change on this file since 102720 was 102720, checked in by vboxsync, 11 months ago
VMM/IEM: Enabled the memmap TLB code on x86. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 146.6 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102720 2023-12-28 00:28:00Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
88	pbCodeBuf[off++] = 0xcc;
89	RT_NOREF(uInfo);
90
91	#elif defined(RT_ARCH_ARM64)
92	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
93	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
94
95	#else
96	# error "error"
97	#endif
98	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
99	return off;
100	}
101
102
103	/*********************************************************************************************************************************
104	* Loads, Stores and Related Stuff. *
105	*********************************************************************************************************************************/
106
107	#ifdef RT_ARCH_AMD64
108	/**
109	* Common bit of iemNativeEmitLoadGprByGpr and friends.
110	*/
111	DECL_FORCE_INLINE(uint32_t)
112	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
113	{
114	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
115	{
116	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
117	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
118	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
119	}
120	else if (offDisp == (int8_t)offDisp)
121	{
122	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
123	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
124	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
125	pbCodeBuf[off++] = (uint8_t)offDisp;
126	}
127	else
128	{
129	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
130	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
131	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
132	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
133	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
134	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
135	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
136	}
137	return off;
138	}
139	#endif /* RT_ARCH_AMD64 */
140
141	/**
142	* Emits setting a GPR to zero.
143	*/
144	DECL_INLINE_THROW(uint32_t)
145	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
146	{
147	#ifdef RT_ARCH_AMD64
148	/* xor gpr32, gpr32 */
149	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
150	if (iGpr >= 8)
151	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
152	pbCodeBuf[off++] = 0x33;
153	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
154
155	#elif defined(RT_ARCH_ARM64)
156	/* mov gpr, #0x0 */
157	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
158	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
159
160	#else
161	# error "port me"
162	#endif
163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
164	return off;
165	}
166
167
168	/**
169	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
170	* buffer space.
171	*
172	* Max buffer consumption:
173	* - AMD64: 10 instruction bytes.
174	* - ARM64: 4 instruction words (16 bytes).
175	*/
176	DECL_FORCE_INLINE(uint32_t)
177	iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
178	{
179	#ifdef RT_ARCH_AMD64
180	if (uImm64 == 0)
181	{
182	/* xor gpr, gpr */
183	if (iGpr >= 8)
184	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
185	pCodeBuf[off++] = 0x33;
186	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
187	}
188	else if (uImm64 <= UINT32_MAX)
189	{
190	/* mov gpr, imm32 */
191	if (iGpr >= 8)
192	pCodeBuf[off++] = X86_OP_REX_B;
193	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
194	pCodeBuf[off++] = RT_BYTE1(uImm64);
195	pCodeBuf[off++] = RT_BYTE2(uImm64);
196	pCodeBuf[off++] = RT_BYTE3(uImm64);
197	pCodeBuf[off++] = RT_BYTE4(uImm64);
198	}
199	else if (uImm64 == (uint64_t)(int32_t)uImm64)
200	{
201	/* mov gpr, sx(imm32) */
202	if (iGpr < 8)
203	pCodeBuf[off++] = X86_OP_REX_W;
204	else
205	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
206	pCodeBuf[off++] = 0xc7;
207	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
208	pCodeBuf[off++] = RT_BYTE1(uImm64);
209	pCodeBuf[off++] = RT_BYTE2(uImm64);
210	pCodeBuf[off++] = RT_BYTE3(uImm64);
211	pCodeBuf[off++] = RT_BYTE4(uImm64);
212	}
213	else
214	{
215	/* mov gpr, imm64 */
216	if (iGpr < 8)
217	pCodeBuf[off++] = X86_OP_REX_W;
218	else
219	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
220	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
221	pCodeBuf[off++] = RT_BYTE1(uImm64);
222	pCodeBuf[off++] = RT_BYTE2(uImm64);
223	pCodeBuf[off++] = RT_BYTE3(uImm64);
224	pCodeBuf[off++] = RT_BYTE4(uImm64);
225	pCodeBuf[off++] = RT_BYTE5(uImm64);
226	pCodeBuf[off++] = RT_BYTE6(uImm64);
227	pCodeBuf[off++] = RT_BYTE7(uImm64);
228	pCodeBuf[off++] = RT_BYTE8(uImm64);
229	}
230
231	#elif defined(RT_ARCH_ARM64)
232	/*
233	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
234	* supply remaining bits using 'movk grp, imm16, lsl #x'.
235	*
236	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
237	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
238	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
239	* after the first non-zero immediate component so we switch to movk for
240	* the remainder.
241	*/
242	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
243	+ !((uImm64 >> 16) & UINT16_MAX)
244	+ !((uImm64 >> 32) & UINT16_MAX)
245	+ !((uImm64 >> 48) & UINT16_MAX);
246	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
247	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
248	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
249	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
250	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
251	if (cFfffHalfWords <= cZeroHalfWords)
252	{
253	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
254
255	/* movz gpr, imm16 */
256	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
257	if (uImmPart \|\| cZeroHalfWords == 4)
258	{
259	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
260	fMovBase \|= RT_BIT_32(29);
261	}
262	/* mov[z/k] gpr, imm16, lsl #16 */
263	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
264	if (uImmPart)
265	{
266	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
267	fMovBase \|= RT_BIT_32(29);
268	}
269	/* mov[z/k] gpr, imm16, lsl #32 */
270	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
271	if (uImmPart)
272	{
273	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
274	fMovBase \|= RT_BIT_32(29);
275	}
276	/* mov[z/k] gpr, imm16, lsl #48 */
277	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
278	if (uImmPart)
279	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
280	}
281	else
282	{
283	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
284
285	/* find the first half-word that isn't UINT16_MAX. */
286	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
287	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
288	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
289
290	/* movn gpr, imm16, lsl #iHwNotFfff16 /
291	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
292	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
293	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
294	/* movk gpr, imm16 */
295	if (iHwNotFfff != 0)
296	{
297	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
298	if (uImmPart != UINT32_C(0xffff))
299	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
300	}
301	/* movk gpr, imm16, lsl #16 */
302	if (iHwNotFfff != 1)
303	{
304	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
305	if (uImmPart != UINT32_C(0xffff))
306	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
307	}
308	/* movk gpr, imm16, lsl #32 */
309	if (iHwNotFfff != 2)
310	{
311	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
312	if (uImmPart != UINT32_C(0xffff))
313	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
314	}
315	/* movk gpr, imm16, lsl #48 */
316	if (iHwNotFfff != 3)
317	{
318	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
319	if (uImmPart != UINT32_C(0xffff))
320	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
321	}
322	}
323
324	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
325	* clang 12.x does that, only to use the 'x' version for the
326	* addressing in the following ldr). */
327
328	#else
329	# error "port me"
330	#endif
331	return off;
332	}
333
334
335	/**
336	* Emits loading a constant into a 64-bit GPR
337	*/
338	DECL_INLINE_THROW(uint32_t)
339	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
340	{
341	#ifdef RT_ARCH_AMD64
342	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
343	#elif defined(RT_ARCH_ARM64)
344	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
345	#else
346	# error "port me"
347	#endif
348	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
349	return off;
350	}
351
352
353	/**
354	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
355	* buffer space.
356	*
357	* Max buffer consumption:
358	* - AMD64: 6 instruction bytes.
359	* - ARM64: 2 instruction words (8 bytes).
360	*
361	* @note The top 32 bits will be cleared.
362	*/
363	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
364	{
365	#ifdef RT_ARCH_AMD64
366	if (uImm32 == 0)
367	{
368	/* xor gpr, gpr */
369	if (iGpr >= 8)
370	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
371	pCodeBuf[off++] = 0x33;
372	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
373	}
374	else
375	{
376	/* mov gpr, imm32 */
377	if (iGpr >= 8)
378	pCodeBuf[off++] = X86_OP_REX_B;
379	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
380	pCodeBuf[off++] = RT_BYTE1(uImm32);
381	pCodeBuf[off++] = RT_BYTE2(uImm32);
382	pCodeBuf[off++] = RT_BYTE3(uImm32);
383	pCodeBuf[off++] = RT_BYTE4(uImm32);
384	}
385
386	#elif defined(RT_ARCH_ARM64)
387	if ((uImm32 >> 16) == 0)
388	/* movz gpr, imm16 */
389	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
390	else if ((uImm32 & UINT32_C(0xffff)) == 0)
391	/* movz gpr, imm16, lsl #16 */
392	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
393	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
394	/* movn gpr, imm16, lsl #16 */
395	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
396	else if ((uImm32 >> 16) == UINT32_C(0xffff))
397	/* movn gpr, imm16 */
398	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
399	else
400	{
401	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
402	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
403	}
404
405	#else
406	# error "port me"
407	#endif
408	return off;
409	}
410
411
412	/**
413	* Emits loading a constant into a 32-bit GPR.
414	* @note The top 32 bits will be cleared.
415	*/
416	DECL_INLINE_THROW(uint32_t)
417	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
418	{
419	#ifdef RT_ARCH_AMD64
420	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
421	#elif defined(RT_ARCH_ARM64)
422	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
423	#else
424	# error "port me"
425	#endif
426	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
427	return off;
428	}
429
430
431	/**
432	* Emits loading a constant into a 8-bit GPR
433	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
434	* only the ARM64 version does that.
435	*/
436	DECL_INLINE_THROW(uint32_t)
437	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
438	{
439	#ifdef RT_ARCH_AMD64
440	/* mov gpr, imm8 */
441	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
442	if (iGpr >= 8)
443	pbCodeBuf[off++] = X86_OP_REX_B;
444	else if (iGpr >= 4)
445	pbCodeBuf[off++] = X86_OP_REX;
446	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
447	pbCodeBuf[off++] = RT_BYTE1(uImm8);
448
449	#elif defined(RT_ARCH_ARM64)
450	/* movz gpr, imm16, lsl #0 */
451	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
452	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
453
454	#else
455	# error "port me"
456	#endif
457	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
458	return off;
459	}
460
461
462	#ifdef RT_ARCH_AMD64
463	/**
464	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
465	*/
466	DECL_FORCE_INLINE(uint32_t)
467	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
468	{
469	if (offVCpu < 128)
470	{
471	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
472	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
473	}
474	else
475	{
476	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
477	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
478	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
479	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
480	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
481	}
482	return off;
483	}
484	#elif defined(RT_ARCH_ARM64)
485	/**
486	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
487	*/
488	DECL_FORCE_INLINE_THROW(uint32_t)
489	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
490	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
491	{
492	/*
493	* There are a couple of ldr variants that takes an immediate offset, so
494	* try use those if we can, otherwise we have to use the temporary register
495	* help with the addressing.
496	*/
497	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
498	{
499	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
500	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
501	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
502	}
503	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
504	{
505	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
506	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
507	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
508	}
509	else
510	{
511	/* The offset is too large, so we must load it into a register and use
512	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
513	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
514	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
515
516	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
517	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
518	IEMNATIVE_REG_FIXED_TMP0);
519	}
520	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
521	return off;
522	}
523	#endif
524
525
526	/**
527	* Emits a 64-bit GPR load of a VCpu value.
528	*/
529	DECL_INLINE_THROW(uint32_t)
530	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
531	{
532	#ifdef RT_ARCH_AMD64
533	/* mov reg64, mem64 */
534	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
535	if (iGpr < 8)
536	pbCodeBuf[off++] = X86_OP_REX_W;
537	else
538	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
539	pbCodeBuf[off++] = 0x8b;
540	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
541	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
542
543	#elif defined(RT_ARCH_ARM64)
544	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
545
546	#else
547	# error "port me"
548	#endif
549	return off;
550	}
551
552
553	/**
554	* Emits a 32-bit GPR load of a VCpu value.
555	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
556	*/
557	DECL_INLINE_THROW(uint32_t)
558	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
559	{
560	#ifdef RT_ARCH_AMD64
561	/* mov reg32, mem32 */
562	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
563	if (iGpr >= 8)
564	pbCodeBuf[off++] = X86_OP_REX_R;
565	pbCodeBuf[off++] = 0x8b;
566	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
567	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
568
569	#elif defined(RT_ARCH_ARM64)
570	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
571
572	#else
573	# error "port me"
574	#endif
575	return off;
576	}
577
578
579	/**
580	* Emits a 16-bit GPR load of a VCpu value.
581	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
582	*/
583	DECL_INLINE_THROW(uint32_t)
584	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
585	{
586	#ifdef RT_ARCH_AMD64
587	/* movzx reg32, mem16 */
588	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
589	if (iGpr >= 8)
590	pbCodeBuf[off++] = X86_OP_REX_R;
591	pbCodeBuf[off++] = 0x0f;
592	pbCodeBuf[off++] = 0xb7;
593	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
594	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
595
596	#elif defined(RT_ARCH_ARM64)
597	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
598
599	#else
600	# error "port me"
601	#endif
602	return off;
603	}
604
605
606	/**
607	* Emits a 8-bit GPR load of a VCpu value.
608	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
609	*/
610	DECL_INLINE_THROW(uint32_t)
611	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
612	{
613	#ifdef RT_ARCH_AMD64
614	/* movzx reg32, mem8 */
615	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
616	if (iGpr >= 8)
617	pbCodeBuf[off++] = X86_OP_REX_R;
618	pbCodeBuf[off++] = 0x0f;
619	pbCodeBuf[off++] = 0xb6;
620	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
621	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
622
623	#elif defined(RT_ARCH_ARM64)
624	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
625
626	#else
627	# error "port me"
628	#endif
629	return off;
630	}
631
632
633	/**
634	* Emits a store of a GPR value to a 64-bit VCpu field.
635	*/
636	DECL_INLINE_THROW(uint32_t)
637	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
638	{
639	#ifdef RT_ARCH_AMD64
640	/* mov mem64, reg64 */
641	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
642	if (iGpr < 8)
643	pbCodeBuf[off++] = X86_OP_REX_W;
644	else
645	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
646	pbCodeBuf[off++] = 0x89;
647	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
648	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
649
650	#elif defined(RT_ARCH_ARM64)
651	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
652
653	#else
654	# error "port me"
655	#endif
656	return off;
657	}
658
659
660	/**
661	* Emits a store of a GPR value to a 32-bit VCpu field.
662	*/
663	DECL_INLINE_THROW(uint32_t)
664	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
665	{
666	#ifdef RT_ARCH_AMD64
667	/* mov mem32, reg32 */
668	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
669	if (iGpr >= 8)
670	pbCodeBuf[off++] = X86_OP_REX_R;
671	pbCodeBuf[off++] = 0x89;
672	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
673	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
674
675	#elif defined(RT_ARCH_ARM64)
676	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
677
678	#else
679	# error "port me"
680	#endif
681	return off;
682	}
683
684
685	/**
686	* Emits a store of a GPR value to a 16-bit VCpu field.
687	*/
688	DECL_INLINE_THROW(uint32_t)
689	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
690	{
691	#ifdef RT_ARCH_AMD64
692	/* mov mem16, reg16 */
693	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
694	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
695	if (iGpr >= 8)
696	pbCodeBuf[off++] = X86_OP_REX_R;
697	pbCodeBuf[off++] = 0x89;
698	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
699	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
700
701	#elif defined(RT_ARCH_ARM64)
702	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
703
704	#else
705	# error "port me"
706	#endif
707	return off;
708	}
709
710
711	/**
712	* Emits a store of a GPR value to a 8-bit VCpu field.
713	*/
714	DECL_INLINE_THROW(uint32_t)
715	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
716	{
717	#ifdef RT_ARCH_AMD64
718	/* mov mem8, reg8 */
719	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
720	if (iGpr >= 8)
721	pbCodeBuf[off++] = X86_OP_REX_R;
722	pbCodeBuf[off++] = 0x88;
723	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
724	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
725
726	#elif defined(RT_ARCH_ARM64)
727	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
728
729	#else
730	# error "port me"
731	#endif
732	return off;
733	}
734
735
736	/**
737	* Emits a store of an immediate value to a 8-bit VCpu field.
738	*/
739	DECL_INLINE_THROW(uint32_t)
740	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
741	{
742	#ifdef RT_ARCH_AMD64
743	/* mov mem8, imm8 */
744	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
745	pbCodeBuf[off++] = 0xc6;
746	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
747	pbCodeBuf[off++] = bImm;
748	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
749
750	#elif defined(RT_ARCH_ARM64)
751	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
752	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
753	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
754	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
755
756	#else
757	# error "port me"
758	#endif
759	return off;
760	}
761
762
763	/**
764	* Emits a load effective address to a GRP of a VCpu field.
765	*/
766	DECL_INLINE_THROW(uint32_t)
767	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
768	{
769	#ifdef RT_ARCH_AMD64
770	/* lea gprdst, [rbx + offDisp] */
771	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
772	if (iGprDst < 8)
773	pbCodeBuf[off++] = X86_OP_REX_W;
774	else
775	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
776	pbCodeBuf[off++] = 0x8d;
777	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
778
779	#elif defined(RT_ARCH_ARM64)
780	if (offVCpu < (unsigned)_4K)
781	{
782	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
783	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
784	}
785	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
786	{
787	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
788	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
789	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
790	}
791	else
792	{
793	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
794	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
795	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
796	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
797	}
798
799	#else
800	# error "port me"
801	#endif
802	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
803	return off;
804	}
805
806
807	/**
808	* Emits a gprdst = gprsrc load.
809	*/
810	DECL_FORCE_INLINE(uint32_t)
811	iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
812	{
813	#ifdef RT_ARCH_AMD64
814	/* mov gprdst, gprsrc */
815	if ((iGprDst \| iGprSrc) >= 8)
816	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
817	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
818	: X86_OP_REX_W \| X86_OP_REX_R;
819	else
820	pCodeBuf[off++] = X86_OP_REX_W;
821	pCodeBuf[off++] = 0x8b;
822	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
823
824	#elif defined(RT_ARCH_ARM64)
825	/* mov dst, src; alias for: orr dst, xzr, src */
826	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
827
828	#else
829	# error "port me"
830	#endif
831	return off;
832	}
833
834
835	/**
836	* Emits a gprdst = gprsrc load.
837	*/
838	DECL_INLINE_THROW(uint32_t)
839	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
840	{
841	#ifdef RT_ARCH_AMD64
842	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
843	#elif defined(RT_ARCH_ARM64)
844	off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
845	#else
846	# error "port me"
847	#endif
848	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
849	return off;
850	}
851
852
853	/**
854	* Emits a gprdst = gprsrc[31:0] load.
855	* @note Bits 63 thru 32 are cleared.
856	*/
857	DECL_FORCE_INLINE(uint32_t)
858	iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
859	{
860	#ifdef RT_ARCH_AMD64
861	/* mov gprdst, gprsrc */
862	if ((iGprDst \| iGprSrc) >= 8)
863	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
864	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
865	: X86_OP_REX_R;
866	pCodeBuf[off++] = 0x8b;
867	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
868
869	#elif defined(RT_ARCH_ARM64)
870	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
871	pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
872
873	#else
874	# error "port me"
875	#endif
876	return off;
877	}
878
879
880	/**
881	* Emits a gprdst = gprsrc[31:0] load.
882	* @note Bits 63 thru 32 are cleared.
883	*/
884	DECL_INLINE_THROW(uint32_t)
885	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
886	{
887	#ifdef RT_ARCH_AMD64
888	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
889	#elif defined(RT_ARCH_ARM64)
890	off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
891	#else
892	# error "port me"
893	#endif
894	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
895	return off;
896	}
897
898
899	/**
900	* Emits a gprdst = gprsrc[15:0] load.
901	* @note Bits 63 thru 15 are cleared.
902	*/
903	DECL_INLINE_THROW(uint32_t)
904	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
905	{
906	#ifdef RT_ARCH_AMD64
907	/* movzx Gv,Ew */
908	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
909	if ((iGprDst \| iGprSrc) >= 8)
910	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
911	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
912	: X86_OP_REX_R;
913	pbCodeBuf[off++] = 0x0f;
914	pbCodeBuf[off++] = 0xb7;
915	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
916
917	#elif defined(RT_ARCH_ARM64)
918	/* and gprdst, gprsrc, #0xffff */
919	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
920	# if 1
921	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
922	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
923	# else
924	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
925	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
926	# endif
927
928	#else
929	# error "port me"
930	#endif
931	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
932	return off;
933	}
934
935
936	/**
937	* Emits a gprdst = gprsrc[7:0] load.
938	* @note Bits 63 thru 8 are cleared.
939	*/
940	DECL_FORCE_INLINE(uint32_t)
941	iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
942	{
943	#ifdef RT_ARCH_AMD64
944	/* movzx Gv,Eb */
945	if (iGprDst >= 8 \|\| iGprSrc >= 8)
946	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
947	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
948	: X86_OP_REX_R;
949	else if (iGprSrc >= 4)
950	pCodeBuf[off++] = X86_OP_REX;
951	pCodeBuf[off++] = 0x0f;
952	pCodeBuf[off++] = 0xb6;
953	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
954
955	#elif defined(RT_ARCH_ARM64)
956	/* and gprdst, gprsrc, #0xff */
957	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
958	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
959
960	#else
961	# error "port me"
962	#endif
963	return off;
964	}
965
966
967	/**
968	* Emits a gprdst = gprsrc[7:0] load.
969	* @note Bits 63 thru 8 are cleared.
970	*/
971	DECL_INLINE_THROW(uint32_t)
972	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
973	{
974	#ifdef RT_ARCH_AMD64
975	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
976	#elif defined(RT_ARCH_ARM64)
977	off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
978	#else
979	# error "port me"
980	#endif
981	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
982	return off;
983	}
984
985
986	/**
987	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
988	* @note Bits 63 thru 8 are cleared.
989	*/
990	DECL_INLINE_THROW(uint32_t)
991	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
992	{
993	#ifdef RT_ARCH_AMD64
994	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
995
996	/* movzx Gv,Ew */
997	if ((iGprDst \| iGprSrc) >= 8)
998	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
999	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
1000	: X86_OP_REX_R;
1001	pbCodeBuf[off++] = 0x0f;
1002	pbCodeBuf[off++] = 0xb7;
1003	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1004
1005	/* shr Ev,8 */
1006	if (iGprDst >= 8)
1007	pbCodeBuf[off++] = X86_OP_REX_B;
1008	pbCodeBuf[off++] = 0xc1;
1009	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1010	pbCodeBuf[off++] = 8;
1011
1012	#elif defined(RT_ARCH_ARM64)
1013	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
1014	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1015	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
1016
1017	#else
1018	# error "port me"
1019	#endif
1020	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1021	return off;
1022	}
1023
1024
1025	/**
1026	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1027	*/
1028	DECL_INLINE_THROW(uint32_t)
1029	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1030	{
1031	#ifdef RT_ARCH_AMD64
1032	/* movsxd r64, r/m32 */
1033	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1034	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1035	pbCodeBuf[off++] = 0x63;
1036	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1037
1038	#elif defined(RT_ARCH_ARM64)
1039	/* sxtw dst, src */
1040	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1041	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
1042
1043	#else
1044	# error "port me"
1045	#endif
1046	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1047	return off;
1048	}
1049
1050
1051	/**
1052	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1053	*/
1054	DECL_INLINE_THROW(uint32_t)
1055	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1056	{
1057	#ifdef RT_ARCH_AMD64
1058	/* movsx r64, r/m16 */
1059	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1060	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1061	pbCodeBuf[off++] = 0x0f;
1062	pbCodeBuf[off++] = 0xbf;
1063	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1064
1065	#elif defined(RT_ARCH_ARM64)
1066	/* sxth dst, src */
1067	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1068	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1069
1070	#else
1071	# error "port me"
1072	#endif
1073	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1074	return off;
1075	}
1076
1077
1078	/**
1079	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1080	*/
1081	DECL_INLINE_THROW(uint32_t)
1082	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1083	{
1084	#ifdef RT_ARCH_AMD64
1085	/* movsx r64, r/m16 */
1086	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1087	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1088	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1089	pbCodeBuf[off++] = 0x0f;
1090	pbCodeBuf[off++] = 0xbf;
1091	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1092
1093	#elif defined(RT_ARCH_ARM64)
1094	/* sxth dst32, src */
1095	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1096	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1097
1098	#else
1099	# error "port me"
1100	#endif
1101	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1102	return off;
1103	}
1104
1105
1106	/**
1107	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1108	*/
1109	DECL_INLINE_THROW(uint32_t)
1110	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1111	{
1112	#ifdef RT_ARCH_AMD64
1113	/* movsx r64, r/m8 */
1114	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1115	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1116	pbCodeBuf[off++] = 0x0f;
1117	pbCodeBuf[off++] = 0xbe;
1118	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1119
1120	#elif defined(RT_ARCH_ARM64)
1121	/* sxtb dst, src */
1122	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1123	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1124
1125	#else
1126	# error "port me"
1127	#endif
1128	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1129	return off;
1130	}
1131
1132
1133	/**
1134	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1135	* @note Bits 63 thru 32 are cleared.
1136	*/
1137	DECL_INLINE_THROW(uint32_t)
1138	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1139	{
1140	#ifdef RT_ARCH_AMD64
1141	/* movsx r32, r/m8 */
1142	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1143	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1144	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1145	pbCodeBuf[off++] = 0x0f;
1146	pbCodeBuf[off++] = 0xbe;
1147	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1148
1149	#elif defined(RT_ARCH_ARM64)
1150	/* sxtb dst32, src32 */
1151	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1152	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1153
1154	#else
1155	# error "port me"
1156	#endif
1157	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1158	return off;
1159	}
1160
1161
1162	/**
1163	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1164	* @note Bits 63 thru 16 are cleared.
1165	*/
1166	DECL_INLINE_THROW(uint32_t)
1167	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1168	{
1169	#ifdef RT_ARCH_AMD64
1170	/* movsx r16, r/m8 */
1171	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1172	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1173	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1174	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1175	pbCodeBuf[off++] = 0x0f;
1176	pbCodeBuf[off++] = 0xbe;
1177	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1178
1179	/* movzx r32, r/m16 */
1180	if (iGprDst >= 8)
1181	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1182	pbCodeBuf[off++] = 0x0f;
1183	pbCodeBuf[off++] = 0xb7;
1184	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1185
1186	#elif defined(RT_ARCH_ARM64)
1187	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1188	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1189	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1190	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1191	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1192
1193	#else
1194	# error "port me"
1195	#endif
1196	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1197	return off;
1198	}
1199
1200
1201	/**
1202	* Emits a gprdst = gprsrc + addend load.
1203	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1204	*/
1205	#ifdef RT_ARCH_AMD64
1206	DECL_INLINE_THROW(uint32_t)
1207	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1208	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1209	{
1210	Assert(iAddend != 0);
1211
1212	/* lea gprdst, [gprsrc + iAddend] */
1213	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1214	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1215	pbCodeBuf[off++] = 0x8d;
1216	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1217	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1218	return off;
1219	}
1220
1221	#elif defined(RT_ARCH_ARM64)
1222	DECL_INLINE_THROW(uint32_t)
1223	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1224	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1225	{
1226	if ((uint32_t)iAddend < 4096)
1227	{
1228	/* add dst, src, uimm12 */
1229	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1230	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1231	}
1232	else if ((uint32_t)-iAddend < 4096)
1233	{
1234	/* sub dst, src, uimm12 */
1235	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1236	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1237	}
1238	else
1239	{
1240	Assert(iGprSrc != iGprDst);
1241	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1242	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1243	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1244	}
1245	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1246	return off;
1247	}
1248	#else
1249	# error "port me"
1250	#endif
1251
1252	/**
1253	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1254	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1255	*/
1256	#ifdef RT_ARCH_AMD64
1257	DECL_INLINE_THROW(uint32_t)
1258	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1259	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1260	#else
1261	DECL_INLINE_THROW(uint32_t)
1262	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1263	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1264	#endif
1265	{
1266	if (iAddend != 0)
1267	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1268	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1269	}
1270
1271
1272	/**
1273	* Emits a gprdst = gprsrc32 + addend load.
1274	* @note Bits 63 thru 32 are cleared.
1275	*/
1276	DECL_INLINE_THROW(uint32_t)
1277	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1278	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1279	{
1280	Assert(iAddend != 0);
1281
1282	#ifdef RT_ARCH_AMD64
1283	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1284	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1285	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1286	if ((iGprDst \| iGprSrc) >= 8)
1287	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1288	pbCodeBuf[off++] = 0x8d;
1289	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1290
1291	#elif defined(RT_ARCH_ARM64)
1292	if ((uint32_t)iAddend < 4096)
1293	{
1294	/* add dst, src, uimm12 */
1295	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1296	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1297	}
1298	else if ((uint32_t)-iAddend < 4096)
1299	{
1300	/* sub dst, src, uimm12 */
1301	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1302	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1303	}
1304	else
1305	{
1306	Assert(iGprSrc != iGprDst);
1307	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1308	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1309	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1310	}
1311
1312	#else
1313	# error "port me"
1314	#endif
1315	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1316	return off;
1317	}
1318
1319
1320	/**
1321	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1322	*/
1323	DECL_INLINE_THROW(uint32_t)
1324	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1325	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1326	{
1327	if (iAddend != 0)
1328	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1329	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1330	}
1331
1332
1333
1334	#ifdef RT_ARCH_AMD64
1335	/**
1336	* Common bit of iemNativeEmitLoadGprByBp and friends.
1337	*/
1338	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1339	PIEMRECOMPILERSTATE pReNativeAssert)
1340	{
1341	if (offDisp < 128 && offDisp >= -128)
1342	{
1343	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1344	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1345	}
1346	else
1347	{
1348	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1349	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1350	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1351	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1352	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1353	}
1354	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1355	return off;
1356	}
1357	#elif defined(RT_ARCH_ARM64)
1358	/**
1359	* Common bit of iemNativeEmitLoadGprByBp and friends.
1360	*/
1361	DECL_FORCE_INLINE_THROW(uint32_t)
1362	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1363	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1364	{
1365	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1366	{
1367	/* str w/ unsigned imm12 (scaled) */
1368	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1369	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1370	}
1371	else if (offDisp >= -256 && offDisp <= 256)
1372	{
1373	/* stur w/ signed imm9 (unscaled) */
1374	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1375	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1376	}
1377	else
1378	{
1379	/* Use temporary indexing register. */
1380	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1381	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1382	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1383	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1384	}
1385	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1386	return off;
1387	}
1388	#endif
1389
1390
1391	/**
1392	* Emits a 64-bit GRP load instruction with an BP relative source address.
1393	*/
1394	DECL_INLINE_THROW(uint32_t)
1395	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1396	{
1397	#ifdef RT_ARCH_AMD64
1398	/* mov gprdst, qword [rbp + offDisp] */
1399	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1400	if (iGprDst < 8)
1401	pbCodeBuf[off++] = X86_OP_REX_W;
1402	else
1403	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1404	pbCodeBuf[off++] = 0x8b;
1405	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1406
1407	#elif defined(RT_ARCH_ARM64)
1408	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1409
1410	#else
1411	# error "port me"
1412	#endif
1413	}
1414
1415
1416	/**
1417	* Emits a 32-bit GRP load instruction with an BP relative source address.
1418	* @note Bits 63 thru 32 of the GPR will be cleared.
1419	*/
1420	DECL_INLINE_THROW(uint32_t)
1421	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1422	{
1423	#ifdef RT_ARCH_AMD64
1424	/* mov gprdst, dword [rbp + offDisp] */
1425	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1426	if (iGprDst >= 8)
1427	pbCodeBuf[off++] = X86_OP_REX_R;
1428	pbCodeBuf[off++] = 0x8b;
1429	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1430
1431	#elif defined(RT_ARCH_ARM64)
1432	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1433
1434	#else
1435	# error "port me"
1436	#endif
1437	}
1438
1439
1440	/**
1441	* Emits a 16-bit GRP load instruction with an BP relative source address.
1442	* @note Bits 63 thru 16 of the GPR will be cleared.
1443	*/
1444	DECL_INLINE_THROW(uint32_t)
1445	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1446	{
1447	#ifdef RT_ARCH_AMD64
1448	/* movzx gprdst, word [rbp + offDisp] */
1449	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1450	if (iGprDst >= 8)
1451	pbCodeBuf[off++] = X86_OP_REX_R;
1452	pbCodeBuf[off++] = 0x0f;
1453	pbCodeBuf[off++] = 0xb7;
1454	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1455
1456	#elif defined(RT_ARCH_ARM64)
1457	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1458
1459	#else
1460	# error "port me"
1461	#endif
1462	}
1463
1464
1465	/**
1466	* Emits a 8-bit GRP load instruction with an BP relative source address.
1467	* @note Bits 63 thru 8 of the GPR will be cleared.
1468	*/
1469	DECL_INLINE_THROW(uint32_t)
1470	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1471	{
1472	#ifdef RT_ARCH_AMD64
1473	/* movzx gprdst, byte [rbp + offDisp] */
1474	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1475	if (iGprDst >= 8)
1476	pbCodeBuf[off++] = X86_OP_REX_R;
1477	pbCodeBuf[off++] = 0x0f;
1478	pbCodeBuf[off++] = 0xb6;
1479	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1480
1481	#elif defined(RT_ARCH_ARM64)
1482	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1483
1484	#else
1485	# error "port me"
1486	#endif
1487	}
1488
1489
1490	/**
1491	* Emits a load effective address to a GRP with an BP relative source address.
1492	*/
1493	DECL_INLINE_THROW(uint32_t)
1494	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1495	{
1496	#ifdef RT_ARCH_AMD64
1497	/* lea gprdst, [rbp + offDisp] */
1498	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1499	if (iGprDst < 8)
1500	pbCodeBuf[off++] = X86_OP_REX_W;
1501	else
1502	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1503	pbCodeBuf[off++] = 0x8d;
1504	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1505
1506	#elif defined(RT_ARCH_ARM64)
1507	if ((uint32_t)offDisp < (unsigned)_4K)
1508	{
1509	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1510	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1511	}
1512	else if ((uint32_t)-offDisp < (unsigned)_4K)
1513	{
1514	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1515	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1516	}
1517	else
1518	{
1519	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1520	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1521	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1522	if (offDisp >= 0)
1523	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1524	else
1525	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1526	}
1527
1528	#else
1529	# error "port me"
1530	#endif
1531
1532	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1533	return off;
1534	}
1535
1536
1537	/**
1538	* Emits a 64-bit GPR store with an BP relative destination address.
1539	*
1540	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1541	*/
1542	DECL_INLINE_THROW(uint32_t)
1543	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1544	{
1545	#ifdef RT_ARCH_AMD64
1546	/* mov qword [rbp + offDisp], gprdst */
1547	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1548	if (iGprSrc < 8)
1549	pbCodeBuf[off++] = X86_OP_REX_W;
1550	else
1551	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1552	pbCodeBuf[off++] = 0x89;
1553	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1554
1555	#elif defined(RT_ARCH_ARM64)
1556	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1557	{
1558	/* str w/ unsigned imm12 (scaled) */
1559	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1560	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1561	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1562	}
1563	else if (offDisp >= -256 && offDisp <= 256)
1564	{
1565	/* stur w/ signed imm9 (unscaled) */
1566	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1567	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1568	}
1569	else if ((uint32_t)-offDisp < (unsigned)_4K)
1570	{
1571	/* Use temporary indexing register w/ sub uimm12. */
1572	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1573	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1574	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1575	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1576	}
1577	else
1578	{
1579	/* Use temporary indexing register. */
1580	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1581	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1582	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1583	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1584	}
1585	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1586	return off;
1587
1588	#else
1589	# error "Port me!"
1590	#endif
1591	}
1592
1593
1594	/**
1595	* Emits a 64-bit immediate store with an BP relative destination address.
1596	*
1597	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1598	*/
1599	DECL_INLINE_THROW(uint32_t)
1600	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1601	{
1602	#ifdef RT_ARCH_AMD64
1603	if ((int64_t)uImm64 == (int32_t)uImm64)
1604	{
1605	/* mov qword [rbp + offDisp], imm32 - sign extended */
1606	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1607	pbCodeBuf[off++] = X86_OP_REX_W;
1608	pbCodeBuf[off++] = 0xc7;
1609	if (offDisp < 128 && offDisp >= -128)
1610	{
1611	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1612	pbCodeBuf[off++] = (uint8_t)offDisp;
1613	}
1614	else
1615	{
1616	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1617	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1618	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1619	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1620	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1621	}
1622	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1623	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1624	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1625	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1626	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1627	return off;
1628	}
1629	#endif
1630
1631	/* Load tmp0, imm64; Store tmp to bp+disp. */
1632	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1633	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1634	}
1635
1636
1637	#if defined(RT_ARCH_ARM64)
1638	/**
1639	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1640	*/
1641	DECL_FORCE_INLINE_THROW(uint32_t)
1642	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1643	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1644	{
1645	/*
1646	* There are a couple of ldr variants that takes an immediate offset, so
1647	* try use those if we can, otherwise we have to use the temporary register
1648	* help with the addressing.
1649	*/
1650	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1651	{
1652	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1653	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1654	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1655	}
1656	else
1657	{
1658	/* The offset is too large, so we must load it into a register and use
1659	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1660	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1661	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1662
1663	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1664	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1665
1666	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1667	}
1668	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1669	return off;
1670	}
1671	#endif
1672
1673
1674	/**
1675	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1676	*/
1677	DECL_INLINE_THROW(uint32_t)
1678	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1679	{
1680	#ifdef RT_ARCH_AMD64
1681	/* mov reg64, mem64 */
1682	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1683	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1684	pbCodeBuf[off++] = 0x8b;
1685	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1686	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1687
1688	#elif defined(RT_ARCH_ARM64)
1689	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1690
1691	#else
1692	# error "port me"
1693	#endif
1694	return off;
1695	}
1696
1697
1698	/**
1699	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1700	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1701	*/
1702	DECL_INLINE_THROW(uint32_t)
1703	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1704	{
1705	#ifdef RT_ARCH_AMD64
1706	/* mov reg32, mem32 */
1707	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1708	if (iGprDst >= 8 \|\| iGprBase >= 8)
1709	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1710	pbCodeBuf[off++] = 0x8b;
1711	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1712	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1713
1714	#elif defined(RT_ARCH_ARM64)
1715	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1716
1717	#else
1718	# error "port me"
1719	#endif
1720	return off;
1721	}
1722
1723
1724	/*********************************************************************************************************************************
1725	* Subtraction and Additions *
1726	*********************************************************************************************************************************/
1727
1728	/**
1729	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
1730	* @note The AMD64 version sets flags.
1731	*/
1732	DECL_INLINE_THROW(uint32_t)
1733	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1734	{
1735	#if defined(RT_ARCH_AMD64)
1736	/* sub Gv,Ev */
1737	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1738	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1739	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1740	pbCodeBuf[off++] = 0x2b;
1741	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1742
1743	#elif defined(RT_ARCH_ARM64)
1744	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1745	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
1746
1747	#else
1748	# error "Port me"
1749	#endif
1750	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1751	return off;
1752	}
1753
1754
1755	/**
1756	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
1757	* @note The AMD64 version sets flags.
1758	*/
1759	DECL_FORCE_INLINE(uint32_t)
1760	iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1761	{
1762	#if defined(RT_ARCH_AMD64)
1763	/* sub Gv,Ev */
1764	if (iGprDst >= 8 \|\| iGprSubtrahend >= 8)
1765	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R)
1766	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1767	pCodeBuf[off++] = 0x2b;
1768	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1769
1770	#elif defined(RT_ARCH_ARM64)
1771	pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /f64Bit/);
1772
1773	#else
1774	# error "Port me"
1775	#endif
1776	return off;
1777	}
1778
1779
1780	/**
1781	* Emits subtracting a 32-bit GPR from another, storing the result in the first.
1782	* @note The AMD64 version sets flags.
1783	*/
1784	DECL_INLINE_THROW(uint32_t)
1785	iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1786	{
1787	#if defined(RT_ARCH_AMD64)
1788	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
1789	#elif defined(RT_ARCH_ARM64)
1790	off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
1791	#else
1792	# error "Port me"
1793	#endif
1794	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1795	return off;
1796	}
1797
1798
1799	#ifdef RT_ARCH_AMD64
1800	/**
1801	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1802	*/
1803	DECL_INLINE_THROW(uint32_t)
1804	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1805	{
1806	/* sub gprdst, imm8/imm32 */
1807	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1808	if (iGprDst < 8)
1809	pbCodeBuf[off++] = X86_OP_REX_W;
1810	else
1811	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1812	if (iSubtrahend < 128 && iSubtrahend >= -128)
1813	{
1814	pbCodeBuf[off++] = 0x83;
1815	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1816	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1817	}
1818	else
1819	{
1820	pbCodeBuf[off++] = 0x81;
1821	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1822	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1823	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1824	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1825	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1826	}
1827	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1828	return off;
1829	}
1830	#endif
1831
1832
1833	#ifdef RT_ARCH_AMD64
1834	/**
1835	* Emits a 32-bit GPR subtract with a signed immediate subtrahend.
1836	*
1837	* This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
1838	*/
1839	DECL_FORCE_INLINE(uint32_t)
1840	iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1841	{
1842	if (iGprDst >= 8)
1843	pCodeBuf[off++] = X86_OP_REX_B;
1844	if (iSubtrahend == 1)
1845	{
1846	/* dec r/m32 */
1847	pCodeBuf[off++] = 0xff;
1848	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
1849	}
1850	else if (iSubtrahend == -1)
1851	{
1852	/* inc r/m32 */
1853	pCodeBuf[off++] = 0xff;
1854	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1855	}
1856	else if (iSubtrahend < 128 && iSubtrahend >= -128)
1857	{
1858	/* sub r/m32, imm8 */
1859	pCodeBuf[off++] = 0x83;
1860	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1861	pCodeBuf[off++] = (uint8_t)iSubtrahend;
1862	}
1863	else
1864	{
1865	/* sub r/m32, imm32 */
1866	pCodeBuf[off++] = 0x81;
1867	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1868	pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1869	pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1870	pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1871	pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1872	}
1873	return off;
1874	}
1875	#endif
1876
1877
1878	/**
1879	* Emits adding a 64-bit GPR to another, storing the result in the first.
1880	* @note The AMD64 version sets flags.
1881	*/
1882	DECL_FORCE_INLINE(uint32_t)
1883	iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1884	{
1885	#if defined(RT_ARCH_AMD64)
1886	/* add Gv,Ev */
1887	pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1888	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1889	pCodeBuf[off++] = 0x03;
1890	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1891
1892	#elif defined(RT_ARCH_ARM64)
1893	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1894
1895	#else
1896	# error "Port me"
1897	#endif
1898	return off;
1899	}
1900
1901
1902	/**
1903	* Emits adding a 64-bit GPR to another, storing the result in the first.
1904	* @note The AMD64 version sets flags.
1905	*/
1906	DECL_INLINE_THROW(uint32_t)
1907	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1908	{
1909	#if defined(RT_ARCH_AMD64)
1910	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
1911	#elif defined(RT_ARCH_ARM64)
1912	off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
1913	#else
1914	# error "Port me"
1915	#endif
1916	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1917	return off;
1918	}
1919
1920
1921	/**
1922	* Emits adding a 64-bit GPR to another, storing the result in the first.
1923	* @note The AMD64 version sets flags.
1924	*/
1925	DECL_FORCE_INLINE(uint32_t)
1926	iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1927	{
1928	#if defined(RT_ARCH_AMD64)
1929	/* add Gv,Ev */
1930	if (iGprDst >= 8 \|\| iGprAddend >= 8)
1931	pCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0)
1932	\| (iGprAddend >= 8 ? X86_OP_REX_B : 0);
1933	pCodeBuf[off++] = 0x03;
1934	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1935
1936	#elif defined(RT_ARCH_ARM64)
1937	pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend, false /f64Bit/);
1938
1939	#else
1940	# error "Port me"
1941	#endif
1942	return off;
1943	}
1944
1945
1946	/**
1947	* Emits adding a 64-bit GPR to another, storing the result in the first.
1948	* @note The AMD64 version sets flags.
1949	*/
1950	DECL_INLINE_THROW(uint32_t)
1951	iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1952	{
1953	#if defined(RT_ARCH_AMD64)
1954	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
1955	#elif defined(RT_ARCH_ARM64)
1956	off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
1957	#else
1958	# error "Port me"
1959	#endif
1960	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1961	return off;
1962	}
1963
1964
1965	/**
1966	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1967	*/
1968	DECL_INLINE_THROW(uint32_t)
1969	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1970	{
1971	#if defined(RT_ARCH_AMD64)
1972	/* add or inc */
1973	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1974	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1975	if (iImm8 != 1)
1976	{
1977	pbCodeBuf[off++] = 0x83;
1978	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1979	pbCodeBuf[off++] = (uint8_t)iImm8;
1980	}
1981	else
1982	{
1983	pbCodeBuf[off++] = 0xff;
1984	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1985	}
1986
1987	#elif defined(RT_ARCH_ARM64)
1988	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1989	if (iImm8 >= 0)
1990	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1991	else
1992	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1993
1994	#else
1995	# error "Port me"
1996	#endif
1997	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1998	return off;
1999	}
2000
2001
2002	/**
2003	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
2004	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2005	*/
2006	DECL_FORCE_INLINE(uint32_t)
2007	iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
2008	{
2009	#if defined(RT_ARCH_AMD64)
2010	/* add or inc */
2011	if (iGprDst >= 8)
2012	pCodeBuf[off++] = X86_OP_REX_B;
2013	if (iImm8 != 1)
2014	{
2015	pCodeBuf[off++] = 0x83;
2016	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2017	pCodeBuf[off++] = (uint8_t)iImm8;
2018	}
2019	else
2020	{
2021	pCodeBuf[off++] = 0xff;
2022	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2023	}
2024
2025	#elif defined(RT_ARCH_ARM64)
2026	if (iImm8 >= 0)
2027	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
2028	else
2029	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
2030
2031	#else
2032	# error "Port me"
2033	#endif
2034	return off;
2035	}
2036
2037
2038	/**
2039	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
2040	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2041	*/
2042	DECL_INLINE_THROW(uint32_t)
2043	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
2044	{
2045	#if defined(RT_ARCH_AMD64)
2046	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
2047	#elif defined(RT_ARCH_ARM64)
2048	off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
2049	#else
2050	# error "Port me"
2051	#endif
2052	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2053	return off;
2054	}
2055
2056
2057	/**
2058	* Emits a 64-bit GPR additions with a 64-bit signed addend.
2059	*/
2060	DECL_INLINE_THROW(uint32_t)
2061	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
2062	{
2063	#if defined(RT_ARCH_AMD64)
2064	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2065	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
2066
2067	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
2068	{
2069	/* add grp, imm32 */
2070	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2071	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2072	pbCodeBuf[off++] = 0x81;
2073	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2074	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2075	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2076	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2077	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2078	}
2079	else
2080	{
2081	/* Best to use a temporary register to deal with this in the simplest way: */
2082	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2083
2084	/* add dst, tmpreg */
2085	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2086	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
2087	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2088	pbCodeBuf[off++] = 0x03;
2089	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
2090
2091	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2092	}
2093
2094	#elif defined(RT_ARCH_ARM64)
2095	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2096	{
2097	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2098	if (iAddend >= 0)
2099	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
2100	else
2101	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
2102	}
2103	else
2104	{
2105	/* Use temporary register for the immediate. */
2106	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2107
2108	/* add gprdst, gprdst, tmpreg */
2109	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2110	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
2111
2112	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2113	}
2114
2115	#else
2116	# error "Port me"
2117	#endif
2118	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2119	return off;
2120	}
2121
2122
2123	/**
2124	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
2125	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2126	* @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
2127	* or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
2128	* the lower 12 bits always zero). The negative ranges are also allowed,
2129	* making it behave like a subtraction. If the constant does not conform,
2130	* bad stuff will happen.
2131	*/
2132	DECL_FORCE_INLINE_THROW(uint32_t)
2133	iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2134	{
2135	#if defined(RT_ARCH_AMD64)
2136	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2137	return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
2138
2139	/* add grp, imm32 */
2140	if (iGprDst >= 8)
2141	pCodeBuf[off++] = X86_OP_REX_B;
2142	pCodeBuf[off++] = 0x81;
2143	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2144	pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2145	pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2146	pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2147	pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2148
2149	#elif defined(RT_ARCH_ARM64)
2150	uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddend);
2151	if (uAbsAddend <= 0xfff)
2152	{
2153	if (iAddend >= 0)
2154	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
2155	else
2156	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend, false /f64Bit/);
2157	}
2158	else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
2159	{
2160	if (iAddend >= 0)
2161	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
2162	false /f64Bit/, true /fShift12/);
2163	else
2164	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, uAbsAddend >> 12,
2165	false /f64Bit/, true /fShift12/);
2166	}
2167	else
2168	# ifdef IEM_WITH_THROW_CATCH
2169	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2170	# else
2171	AssertReleaseFailedStmt(off = UINT32_MAX);
2172	# endif
2173
2174	#else
2175	# error "Port me"
2176	#endif
2177	return off;
2178	}
2179
2180
2181	/**
2182	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
2183	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
2184	*/
2185	DECL_INLINE_THROW(uint32_t)
2186	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2187	{
2188	#if defined(RT_ARCH_AMD64)
2189	off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
2190
2191	#elif defined(RT_ARCH_ARM64)
2192	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2193	{
2194	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2195	if (iAddend >= 0)
2196	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
2197	else
2198	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
2199	}
2200	else
2201	{
2202	/* Use temporary register for the immediate. */
2203	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
2204
2205	/* add gprdst, gprdst, tmpreg */
2206	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2207	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
2208
2209	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2210	}
2211
2212	#else
2213	# error "Port me"
2214	#endif
2215	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2216	return off;
2217	}
2218
2219
2220	/*********************************************************************************************************************************
2221	* Unary Operations *
2222	*********************************************************************************************************************************/
2223
2224	/**
2225	* Emits code for two complement negation of a 64-bit GPR.
2226	*/
2227	DECL_FORCE_INLINE_THROW(uint32_t)
2228	iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
2229	{
2230	#if defined(RT_ARCH_AMD64)
2231	/* neg Ev */
2232	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2233	pCodeBuf[off++] = 0xf7;
2234	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
2235
2236	#elif defined(RT_ARCH_ARM64)
2237	/* sub dst, xzr, dst */
2238	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
2239
2240	#else
2241	# error "Port me"
2242	#endif
2243	return off;
2244	}
2245
2246
2247	/**
2248	* Emits code for two complement negation of a 64-bit GPR.
2249	*/
2250	DECL_INLINE_THROW(uint32_t)
2251	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2252	{
2253	#if defined(RT_ARCH_AMD64)
2254	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
2255	#elif defined(RT_ARCH_ARM64)
2256	off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
2257	#else
2258	# error "Port me"
2259	#endif
2260	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2261	return off;
2262	}
2263
2264
2265	/**
2266	* Emits code for two complement negation of a 32-bit GPR.
2267	* @note bit 32 thru 63 are set to zero.
2268	*/
2269	DECL_FORCE_INLINE_THROW(uint32_t)
2270	iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
2271	{
2272	#if defined(RT_ARCH_AMD64)
2273	/* neg Ev */
2274	if (iGprDst >= 8)
2275	pCodeBuf[off++] = X86_OP_REX_B;
2276	pCodeBuf[off++] = 0xf7;
2277	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
2278
2279	#elif defined(RT_ARCH_ARM64)
2280	/* sub dst, xzr, dst */
2281	pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /f64Bit/);
2282
2283	#else
2284	# error "Port me"
2285	#endif
2286	return off;
2287	}
2288
2289
2290	/**
2291	* Emits code for two complement negation of a 32-bit GPR.
2292	* @note bit 32 thru 63 are set to zero.
2293	*/
2294	DECL_INLINE_THROW(uint32_t)
2295	iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2296	{
2297	#if defined(RT_ARCH_AMD64)
2298	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
2299	#elif defined(RT_ARCH_ARM64)
2300	off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
2301	#else
2302	# error "Port me"
2303	#endif
2304	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2305	return off;
2306	}
2307
2308
2309
2310	/*********************************************************************************************************************************
2311	* Bit Operations *
2312	*********************************************************************************************************************************/
2313
2314	/**
2315	* Emits code for clearing bits 16 thru 63 in the GPR.
2316	*/
2317	DECL_INLINE_THROW(uint32_t)
2318	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2319	{
2320	#if defined(RT_ARCH_AMD64)
2321	/* movzx Gv,Ew */
2322	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2323	if (iGprDst >= 8)
2324	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
2325	pbCodeBuf[off++] = 0x0f;
2326	pbCodeBuf[off++] = 0xb7;
2327	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2328
2329	#elif defined(RT_ARCH_ARM64)
2330	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2331	# if 1
2332	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2333	# else
2334	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2335	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2336	# endif
2337	#else
2338	# error "Port me"
2339	#endif
2340	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2341	return off;
2342	}
2343
2344
2345	/**
2346	* Emits code for AND'ing two 64-bit GPRs.
2347	*
2348	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2349	* and ARM64 hosts.
2350	*/
2351	DECL_INLINE_THROW(uint32_t)
2352	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2353	{
2354	#if defined(RT_ARCH_AMD64)
2355	/* and Gv, Ev */
2356	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2357	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2358	pbCodeBuf[off++] = 0x23;
2359	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2360	RT_NOREF(fSetFlags);
2361
2362	#elif defined(RT_ARCH_ARM64)
2363	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2364	if (!fSetFlags)
2365	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2366	else
2367	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2368
2369	#else
2370	# error "Port me"
2371	#endif
2372	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2373	return off;
2374	}
2375
2376
2377	/**
2378	* Emits code for AND'ing two 32-bit GPRs.
2379	*/
2380	DECL_FORCE_INLINE(uint32_t)
2381	iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2382	{
2383	#if defined(RT_ARCH_AMD64)
2384	/* and Gv, Ev */
2385	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2386	pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2387	pCodeBuf[off++] = 0x23;
2388	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2389	RT_NOREF(fSetFlags);
2390
2391	#elif defined(RT_ARCH_ARM64)
2392	if (!fSetFlags)
2393	pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2394	else
2395	pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2396
2397	#else
2398	# error "Port me"
2399	#endif
2400	return off;
2401	}
2402
2403
2404	/**
2405	* Emits code for AND'ing two 32-bit GPRs.
2406	*/
2407	DECL_INLINE_THROW(uint32_t)
2408	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2409	{
2410	#if defined(RT_ARCH_AMD64)
2411	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
2412	#elif defined(RT_ARCH_ARM64)
2413	off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
2414	#else
2415	# error "Port me"
2416	#endif
2417	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2418	return off;
2419	}
2420
2421
2422	/**
2423	* Emits code for AND'ing a 64-bit GPRs with a constant.
2424	*
2425	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2426	* and ARM64 hosts.
2427	*/
2428	DECL_INLINE_THROW(uint32_t)
2429	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2430	{
2431	#if defined(RT_ARCH_AMD64)
2432	if ((int64_t)uImm == (int8_t)uImm)
2433	{
2434	/* and Ev, imm8 */
2435	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2436	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2437	pbCodeBuf[off++] = 0x83;
2438	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2439	pbCodeBuf[off++] = (uint8_t)uImm;
2440	}
2441	else if ((int64_t)uImm == (int32_t)uImm)
2442	{
2443	/* and Ev, imm32 */
2444	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2445	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2446	pbCodeBuf[off++] = 0x81;
2447	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2448	pbCodeBuf[off++] = RT_BYTE1(uImm);
2449	pbCodeBuf[off++] = RT_BYTE2(uImm);
2450	pbCodeBuf[off++] = RT_BYTE3(uImm);
2451	pbCodeBuf[off++] = RT_BYTE4(uImm);
2452	}
2453	else
2454	{
2455	/* Use temporary register for the 64-bit immediate. */
2456	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2457	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2458	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2459	}
2460	RT_NOREF(fSetFlags);
2461
2462	#elif defined(RT_ARCH_ARM64)
2463	uint32_t uImmR = 0;
2464	uint32_t uImmNandS = 0;
2465	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
2466	{
2467	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2468	if (!fSetFlags)
2469	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2470	else
2471	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2472	}
2473	else
2474	{
2475	/* Use temporary register for the 64-bit immediate. */
2476	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2477	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2478	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2479	}
2480
2481	#else
2482	# error "Port me"
2483	#endif
2484	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2485	return off;
2486	}
2487
2488
2489	/**
2490	* Emits code for AND'ing an 32-bit GPRs with a constant.
2491	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2492	* @note For ARM64 this only supports @a uImm values that can be expressed using
2493	* the two 6-bit immediates of the AND/ANDS instructions. The caller must
2494	* make sure this is possible!
2495	*/
2496	DECL_FORCE_INLINE_THROW(uint32_t)
2497	iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2498	{
2499	#if defined(RT_ARCH_AMD64)
2500	/* and Ev, imm */
2501	if (iGprDst >= 8)
2502	pCodeBuf[off++] = X86_OP_REX_B;
2503	if ((int32_t)uImm == (int8_t)uImm)
2504	{
2505	pCodeBuf[off++] = 0x83;
2506	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2507	pCodeBuf[off++] = (uint8_t)uImm;
2508	}
2509	else
2510	{
2511	pCodeBuf[off++] = 0x81;
2512	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2513	pCodeBuf[off++] = RT_BYTE1(uImm);
2514	pCodeBuf[off++] = RT_BYTE2(uImm);
2515	pCodeBuf[off++] = RT_BYTE3(uImm);
2516	pCodeBuf[off++] = RT_BYTE4(uImm);
2517	}
2518	RT_NOREF(fSetFlags);
2519
2520	#elif defined(RT_ARCH_ARM64)
2521	uint32_t uImmR = 0;
2522	uint32_t uImmNandS = 0;
2523	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2524	{
2525	if (!fSetFlags)
2526	pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2527	else
2528	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2529	}
2530	else
2531	# ifdef IEM_WITH_THROW_CATCH
2532	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2533	# else
2534	AssertReleaseFailedStmt(off = UINT32_MAX);
2535	# endif
2536
2537	#else
2538	# error "Port me"
2539	#endif
2540	return off;
2541	}
2542
2543
2544	/**
2545	* Emits code for AND'ing an 32-bit GPRs with a constant.
2546	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2547	*/
2548	DECL_INLINE_THROW(uint32_t)
2549	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2550	{
2551	#if defined(RT_ARCH_AMD64)
2552	off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
2553
2554	#elif defined(RT_ARCH_ARM64)
2555	uint32_t uImmR = 0;
2556	uint32_t uImmNandS = 0;
2557	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2558	{
2559	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2560	if (!fSetFlags)
2561	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2562	else
2563	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2564	}
2565	else
2566	{
2567	/* Use temporary register for the 64-bit immediate. */
2568	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2569	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2570	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2571	}
2572
2573	#else
2574	# error "Port me"
2575	#endif
2576	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2577	return off;
2578	}
2579
2580
2581	/**
2582	* Emits code for XOR'ing two 64-bit GPRs.
2583	*/
2584	DECL_INLINE_THROW(uint32_t)
2585	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2586	{
2587	#if defined(RT_ARCH_AMD64)
2588	/* and Gv, Ev */
2589	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2590	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2591	pbCodeBuf[off++] = 0x33;
2592	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2593
2594	#elif defined(RT_ARCH_ARM64)
2595	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2596	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2597
2598	#else
2599	# error "Port me"
2600	#endif
2601	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2602	return off;
2603	}
2604
2605
2606	/**
2607	* Emits code for XOR'ing two 32-bit GPRs.
2608	*/
2609	DECL_INLINE_THROW(uint32_t)
2610	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2611	{
2612	#if defined(RT_ARCH_AMD64)
2613	/* and Gv, Ev */
2614	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2615	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2616	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2617	pbCodeBuf[off++] = 0x33;
2618	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2619
2620	#elif defined(RT_ARCH_ARM64)
2621	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2622	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2623
2624	#else
2625	# error "Port me"
2626	#endif
2627	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2628	return off;
2629	}
2630
2631
2632	/**
2633	* Emits code for XOR'ing an 32-bit GPRs with a constant.
2634	* @note Bits 32 thru 63 in the destination will be zero after the operation.
2635	* @note For ARM64 this only supports @a uImm values that can be expressed using
2636	* the two 6-bit immediates of the EOR instructions. The caller must make
2637	* sure this is possible!
2638	*/
2639	DECL_FORCE_INLINE_THROW(uint32_t)
2640	iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
2641	{
2642	#if defined(RT_ARCH_AMD64)
2643	/* and Ev, imm */
2644	if (iGprDst >= 8)
2645	pCodeBuf[off++] = X86_OP_REX_B;
2646	if ((int32_t)uImm == (int8_t)uImm)
2647	{
2648	pCodeBuf[off++] = 0x83;
2649	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
2650	pCodeBuf[off++] = (uint8_t)uImm;
2651	}
2652	else
2653	{
2654	pCodeBuf[off++] = 0x81;
2655	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
2656	pCodeBuf[off++] = RT_BYTE1(uImm);
2657	pCodeBuf[off++] = RT_BYTE2(uImm);
2658	pCodeBuf[off++] = RT_BYTE3(uImm);
2659	pCodeBuf[off++] = RT_BYTE4(uImm);
2660	}
2661
2662	#elif defined(RT_ARCH_ARM64)
2663	uint32_t uImmR = 0;
2664	uint32_t uImmNandS = 0;
2665	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2666	pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2667	else
2668	# ifdef IEM_WITH_THROW_CATCH
2669	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
2670	# else
2671	AssertReleaseFailedStmt(off = UINT32_MAX);
2672	# endif
2673
2674	#else
2675	# error "Port me"
2676	#endif
2677	return off;
2678	}
2679
2680
2681	/*********************************************************************************************************************************
2682	* Shifting *
2683	*********************************************************************************************************************************/
2684
2685	/**
2686	* Emits code for shifting a GPR a fixed number of bits to the left.
2687	*/
2688	DECL_FORCE_INLINE(uint32_t)
2689	iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2690	{
2691	Assert(cShift > 0 && cShift < 64);
2692
2693	#if defined(RT_ARCH_AMD64)
2694	/* shl dst, cShift */
2695	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2696	if (cShift != 1)
2697	{
2698	pCodeBuf[off++] = 0xc1;
2699	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2700	pCodeBuf[off++] = cShift;
2701	}
2702	else
2703	{
2704	pCodeBuf[off++] = 0xd1;
2705	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2706	}
2707
2708	#elif defined(RT_ARCH_ARM64)
2709	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2710
2711	#else
2712	# error "Port me"
2713	#endif
2714	return off;
2715	}
2716
2717
2718	/**
2719	* Emits code for shifting a GPR a fixed number of bits to the left.
2720	*/
2721	DECL_INLINE_THROW(uint32_t)
2722	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2723	{
2724	#if defined(RT_ARCH_AMD64)
2725	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2726	#elif defined(RT_ARCH_ARM64)
2727	off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2728	#else
2729	# error "Port me"
2730	#endif
2731	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2732	return off;
2733	}
2734
2735
2736	/**
2737	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2738	*/
2739	DECL_FORCE_INLINE(uint32_t)
2740	iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2741	{
2742	Assert(cShift > 0 && cShift < 32);
2743
2744	#if defined(RT_ARCH_AMD64)
2745	/* shl dst, cShift */
2746	if (iGprDst >= 8)
2747	pCodeBuf[off++] = X86_OP_REX_B;
2748	if (cShift != 1)
2749	{
2750	pCodeBuf[off++] = 0xc1;
2751	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2752	pCodeBuf[off++] = cShift;
2753	}
2754	else
2755	{
2756	pCodeBuf[off++] = 0xd1;
2757	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2758	}
2759
2760	#elif defined(RT_ARCH_ARM64)
2761	pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2762
2763	#else
2764	# error "Port me"
2765	#endif
2766	return off;
2767	}
2768
2769
2770	/**
2771	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2772	*/
2773	DECL_INLINE_THROW(uint32_t)
2774	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2775	{
2776	#if defined(RT_ARCH_AMD64)
2777	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2778	#elif defined(RT_ARCH_ARM64)
2779	off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2780	#else
2781	# error "Port me"
2782	#endif
2783	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2784	return off;
2785	}
2786
2787
2788	/**
2789	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2790	*/
2791	DECL_FORCE_INLINE(uint32_t)
2792	iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2793	{
2794	Assert(cShift > 0 && cShift < 64);
2795
2796	#if defined(RT_ARCH_AMD64)
2797	/* shr dst, cShift */
2798	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2799	if (cShift != 1)
2800	{
2801	pCodeBuf[off++] = 0xc1;
2802	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2803	pCodeBuf[off++] = cShift;
2804	}
2805	else
2806	{
2807	pCodeBuf[off++] = 0xd1;
2808	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2809	}
2810
2811	#elif defined(RT_ARCH_ARM64)
2812	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2813
2814	#else
2815	# error "Port me"
2816	#endif
2817	return off;
2818	}
2819
2820
2821	/**
2822	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2823	*/
2824	DECL_INLINE_THROW(uint32_t)
2825	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2826	{
2827	#if defined(RT_ARCH_AMD64)
2828	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2829	#elif defined(RT_ARCH_ARM64)
2830	off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2831	#else
2832	# error "Port me"
2833	#endif
2834	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2835	return off;
2836	}
2837
2838
2839	/**
2840	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2841	* right.
2842	*/
2843	DECL_FORCE_INLINE(uint32_t)
2844	iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2845	{
2846	Assert(cShift > 0 && cShift < 32);
2847
2848	#if defined(RT_ARCH_AMD64)
2849	/* shr dst, cShift */
2850	if (iGprDst >= 8)
2851	pCodeBuf[off++] = X86_OP_REX_B;
2852	if (cShift != 1)
2853	{
2854	pCodeBuf[off++] = 0xc1;
2855	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2856	pCodeBuf[off++] = cShift;
2857	}
2858	else
2859	{
2860	pCodeBuf[off++] = 0xd1;
2861	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2862	}
2863
2864	#elif defined(RT_ARCH_ARM64)
2865	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2866
2867	#else
2868	# error "Port me"
2869	#endif
2870	return off;
2871	}
2872
2873
2874	/**
2875	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2876	* right.
2877	*/
2878	DECL_INLINE_THROW(uint32_t)
2879	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2880	{
2881	#if defined(RT_ARCH_AMD64)
2882	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
2883	#elif defined(RT_ARCH_ARM64)
2884	off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
2885	#else
2886	# error "Port me"
2887	#endif
2888	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2889	return off;
2890	}
2891
2892
2893	/**
2894	* Emits code for rotating a GPR a fixed number of bits to the left.
2895	*/
2896	DECL_FORCE_INLINE(uint32_t)
2897	iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2898	{
2899	Assert(cShift > 0 && cShift < 64);
2900
2901	#if defined(RT_ARCH_AMD64)
2902	/* rol dst, cShift */
2903	pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2904	if (cShift != 1)
2905	{
2906	pCodeBuf[off++] = 0xc1;
2907	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2908	pCodeBuf[off++] = cShift;
2909	}
2910	else
2911	{
2912	pCodeBuf[off++] = 0xd1;
2913	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2914	}
2915
2916	#elif defined(RT_ARCH_ARM64)
2917	pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
2918
2919	#else
2920	# error "Port me"
2921	#endif
2922	return off;
2923	}
2924
2925
2926
2927	/*********************************************************************************************************************************
2928	* Compare and Testing *
2929	*********************************************************************************************************************************/
2930
2931
2932	#ifdef RT_ARCH_ARM64
2933	/**
2934	* Emits an ARM64 compare instruction.
2935	*/
2936	DECL_INLINE_THROW(uint32_t)
2937	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2938	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2939	{
2940	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2941	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2942	f64Bit, true /fSetFlags/, cShift, enmShift);
2943	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2944	return off;
2945	}
2946	#endif
2947
2948
2949	/**
2950	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2951	* with conditional instruction.
2952	*/
2953	DECL_FORCE_INLINE(uint32_t)
2954	iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2955	{
2956	#ifdef RT_ARCH_AMD64
2957	/* cmp Gv, Ev */
2958	pCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2959	pCodeBuf[off++] = 0x3b;
2960	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2961
2962	#elif defined(RT_ARCH_ARM64)
2963	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
2964
2965	#else
2966	# error "Port me!"
2967	#endif
2968	return off;
2969	}
2970
2971
2972	/**
2973	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2974	* with conditional instruction.
2975	*/
2976	DECL_INLINE_THROW(uint32_t)
2977	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2978	{
2979	#ifdef RT_ARCH_AMD64
2980	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
2981	#elif defined(RT_ARCH_ARM64)
2982	off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
2983	#else
2984	# error "Port me!"
2985	#endif
2986	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2987	return off;
2988	}
2989
2990
2991	/**
2992	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2993	* with conditional instruction.
2994	*/
2995	DECL_FORCE_INLINE(uint32_t)
2996	iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2997	{
2998	#ifdef RT_ARCH_AMD64
2999	/* cmp Gv, Ev */
3000	if (iGprLeft >= 8 \|\| iGprRight >= 8)
3001	pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
3002	pCodeBuf[off++] = 0x3b;
3003	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
3004
3005	#elif defined(RT_ARCH_ARM64)
3006	pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /f64Bit/);
3007
3008	#else
3009	# error "Port me!"
3010	#endif
3011	return off;
3012	}
3013
3014
3015	/**
3016	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
3017	* with conditional instruction.
3018	*/
3019	DECL_INLINE_THROW(uint32_t)
3020	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
3021	{
3022	#ifdef RT_ARCH_AMD64
3023	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
3024	#elif defined(RT_ARCH_ARM64)
3025	off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
3026	#else
3027	# error "Port me!"
3028	#endif
3029	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3030	return off;
3031	}
3032
3033
3034	/**
3035	* Emits a compare of a 64-bit GPR with a constant value, settings status
3036	* flags/whatever for use with conditional instruction.
3037	*/
3038	DECL_INLINE_THROW(uint32_t)
3039	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
3040	{
3041	#ifdef RT_ARCH_AMD64
3042	if (uImm <= UINT32_C(0xff))
3043	{
3044	/* cmp Ev, Ib */
3045	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3046	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
3047	pbCodeBuf[off++] = 0x83;
3048	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3049	pbCodeBuf[off++] = (uint8_t)uImm;
3050	}
3051	else if ((int64_t)uImm == (int32_t)uImm)
3052	{
3053	/* cmp Ev, imm */
3054	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3055	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
3056	pbCodeBuf[off++] = 0x81;
3057	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3058	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3059	pbCodeBuf[off++] = RT_BYTE1(uImm);
3060	pbCodeBuf[off++] = RT_BYTE2(uImm);
3061	pbCodeBuf[off++] = RT_BYTE3(uImm);
3062	pbCodeBuf[off++] = RT_BYTE4(uImm);
3063	}
3064	else
3065	{
3066	/* Use temporary register for the immediate. */
3067	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3068	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
3069	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3070	}
3071
3072	#elif defined(RT_ARCH_ARM64)
3073	/** @todo guess there are clevere things we can do here... */
3074	if (uImm < _4K)
3075	{
3076	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3077	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3078	true /64Bit/, true /fSetFlags/);
3079	}
3080	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3081	{
3082	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3083	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
3084	true /64Bit/, true /fSetFlags/, true /fShift12/);
3085	}
3086	else
3087	{
3088	/* Use temporary register for the immediate. */
3089	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3090	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
3091	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3092	}
3093
3094	#else
3095	# error "Port me!"
3096	#endif
3097
3098	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3099	return off;
3100	}
3101
3102
3103	/**
3104	* Emits a compare of a 32-bit GPR with a constant value, settings status
3105	* flags/whatever for use with conditional instruction.
3106	*
3107	* @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
3108	* shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
3109	* bits all zero). Will release assert or throw exception if the caller
3110	* violates this restriction.
3111	*/
3112	DECL_FORCE_INLINE_THROW(uint32_t)
3113	iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
3114	{
3115	#ifdef RT_ARCH_AMD64
3116	if (iGprLeft >= 8)
3117	pCodeBuf[off++] = X86_OP_REX_B;
3118	if (uImm <= UINT32_C(0x7f))
3119	{
3120	/* cmp Ev, Ib */
3121	pCodeBuf[off++] = 0x83;
3122	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3123	pCodeBuf[off++] = (uint8_t)uImm;
3124	}
3125	else
3126	{
3127	/* cmp Ev, imm */
3128	pCodeBuf[off++] = 0x81;
3129	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
3130	pCodeBuf[off++] = RT_BYTE1(uImm);
3131	pCodeBuf[off++] = RT_BYTE2(uImm);
3132	pCodeBuf[off++] = RT_BYTE3(uImm);
3133	pCodeBuf[off++] = RT_BYTE4(uImm);
3134	}
3135
3136	#elif defined(RT_ARCH_ARM64)
3137	/** @todo guess there are clevere things we can do here... */
3138	if (uImm < _4K)
3139	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3140	false /64Bit/, true /fSetFlags/);
3141	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3142	pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3143	false /64Bit/, true /fSetFlags/, true /fShift12/);
3144	else
3145	# ifdef IEM_WITH_THROW_CATCH
3146	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3147	# else
3148	AssertReleaseFailedStmt(off = UINT32_MAX);
3149	# endif
3150
3151	#else
3152	# error "Port me!"
3153	#endif
3154	return off;
3155	}
3156
3157
3158	/**
3159	* Emits a compare of a 32-bit GPR with a constant value, settings status
3160	* flags/whatever for use with conditional instruction.
3161	*/
3162	DECL_INLINE_THROW(uint32_t)
3163	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
3164	{
3165	#ifdef RT_ARCH_AMD64
3166	off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
3167
3168	#elif defined(RT_ARCH_ARM64)
3169	/** @todo guess there are clevere things we can do here... */
3170	if (uImm < _4K)
3171	{
3172	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3173	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3174	false /64Bit/, true /fSetFlags/);
3175	}
3176	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
3177	{
3178	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3179	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
3180	false /64Bit/, true /fSetFlags/, true /fShift12/);
3181	}
3182	else
3183	{
3184	/* Use temporary register for the immediate. */
3185	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
3186	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
3187	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3188	}
3189
3190	#else
3191	# error "Port me!"
3192	#endif
3193
3194	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3195	return off;
3196	}
3197
3198
3199
3200	/*********************************************************************************************************************************
3201	* Branching *
3202	*********************************************************************************************************************************/
3203
3204	/**
3205	* Emits a JMP rel32 / B imm19 to the given label.
3206	*/
3207	DECL_FORCE_INLINE_THROW(uint32_t)
3208	iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
3209	{
3210	Assert(idxLabel < pReNative->cLabels);
3211
3212	#ifdef RT_ARCH_AMD64
3213	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
3214	{
3215	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
3216	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
3217	{
3218	pCodeBuf[off++] = 0xeb; /* jmp rel8 */
3219	pCodeBuf[off++] = (uint8_t)offRel;
3220	}
3221	else
3222	{
3223	offRel -= 3;
3224	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
3225	pCodeBuf[off++] = RT_BYTE1(offRel);
3226	pCodeBuf[off++] = RT_BYTE2(offRel);
3227	pCodeBuf[off++] = RT_BYTE3(offRel);
3228	pCodeBuf[off++] = RT_BYTE4(offRel);
3229	}
3230	}
3231	else
3232	{
3233	pCodeBuf[off++] = 0xe9; /* jmp rel32 */
3234	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
3235	pCodeBuf[off++] = 0xfe;
3236	pCodeBuf[off++] = 0xff;
3237	pCodeBuf[off++] = 0xff;
3238	pCodeBuf[off++] = 0xff;
3239	}
3240	pCodeBuf[off++] = 0xcc; /* int3 poison */
3241
3242	#elif defined(RT_ARCH_ARM64)
3243	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
3244	pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
3245	else
3246	{
3247	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
3248	pCodeBuf[off++] = Armv8A64MkInstrB(-1);
3249	}
3250
3251	#else
3252	# error "Port me!"
3253	#endif
3254	return off;
3255	}
3256
3257
3258	/**
3259	* Emits a JMP rel32 / B imm19 to the given label.
3260	*/
3261	DECL_INLINE_THROW(uint32_t)
3262	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3263	{
3264	#ifdef RT_ARCH_AMD64
3265	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
3266	#elif defined(RT_ARCH_ARM64)
3267	off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
3268	#else
3269	# error "Port me!"
3270	#endif
3271	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3272	return off;
3273	}
3274
3275
3276	/**
3277	* Emits a JMP rel32 / B imm19 to a new undefined label.
3278	*/
3279	DECL_INLINE_THROW(uint32_t)
3280	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3281	{
3282	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3283	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
3284	}
3285
3286	/** Condition type. */
3287	#ifdef RT_ARCH_AMD64
3288	typedef enum IEMNATIVEINSTRCOND : uint8_t
3289	{
3290	kIemNativeInstrCond_o = 0,
3291	kIemNativeInstrCond_no,
3292	kIemNativeInstrCond_c,
3293	kIemNativeInstrCond_nc,
3294	kIemNativeInstrCond_e,
3295	kIemNativeInstrCond_ne,
3296	kIemNativeInstrCond_be,
3297	kIemNativeInstrCond_nbe,
3298	kIemNativeInstrCond_s,
3299	kIemNativeInstrCond_ns,
3300	kIemNativeInstrCond_p,
3301	kIemNativeInstrCond_np,
3302	kIemNativeInstrCond_l,
3303	kIemNativeInstrCond_nl,
3304	kIemNativeInstrCond_le,
3305	kIemNativeInstrCond_nle
3306	} IEMNATIVEINSTRCOND;
3307	#elif defined(RT_ARCH_ARM64)
3308	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
3309	#else
3310	# error "Port me!"
3311	#endif
3312
3313
3314	/**
3315	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
3316	*/
3317	DECL_FORCE_INLINE_THROW(uint32_t)
3318	iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
3319	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
3320	{
3321	Assert(idxLabel < pReNative->cLabels);
3322
3323	#ifdef RT_ARCH_AMD64
3324	/* jcc rel32 */
3325	pCodeBuf[off++] = 0x0f;
3326	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
3327	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
3328	pCodeBuf[off++] = 0x00;
3329	pCodeBuf[off++] = 0x00;
3330	pCodeBuf[off++] = 0x00;
3331	pCodeBuf[off++] = 0x00;
3332
3333	#elif defined(RT_ARCH_ARM64)
3334	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3335	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
3336
3337	#else
3338	# error "Port me!"
3339	#endif
3340	return off;
3341	}
3342
3343
3344	/**
3345	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
3346	*/
3347	DECL_INLINE_THROW(uint32_t)
3348	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
3349	{
3350	#ifdef RT_ARCH_AMD64
3351	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
3352	#elif defined(RT_ARCH_ARM64)
3353	off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
3354	#else
3355	# error "Port me!"
3356	#endif
3357	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3358	return off;
3359	}
3360
3361
3362	/**
3363	* Emits a Jcc rel32 / B.cc imm19 to a new label.
3364	*/
3365	DECL_INLINE_THROW(uint32_t)
3366	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3367	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
3368	{
3369	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3370	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
3371	}
3372
3373
3374	/**
3375	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
3376	*/
3377	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3378	{
3379	#ifdef RT_ARCH_AMD64
3380	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
3381	#elif defined(RT_ARCH_ARM64)
3382	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
3383	#else
3384	# error "Port me!"
3385	#endif
3386	}
3387
3388	/**
3389	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
3390	*/
3391	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3392	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3393	{
3394	#ifdef RT_ARCH_AMD64
3395	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
3396	#elif defined(RT_ARCH_ARM64)
3397	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
3398	#else
3399	# error "Port me!"
3400	#endif
3401	}
3402
3403
3404	/**
3405	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
3406	*/
3407	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3408	{
3409	#ifdef RT_ARCH_AMD64
3410	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
3411	#elif defined(RT_ARCH_ARM64)
3412	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
3413	#else
3414	# error "Port me!"
3415	#endif
3416	}
3417
3418	/**
3419	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
3420	*/
3421	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3422	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3423	{
3424	#ifdef RT_ARCH_AMD64
3425	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
3426	#elif defined(RT_ARCH_ARM64)
3427	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
3428	#else
3429	# error "Port me!"
3430	#endif
3431	}
3432
3433
3434	/**
3435	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
3436	*/
3437	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3438	{
3439	#ifdef RT_ARCH_AMD64
3440	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
3441	#elif defined(RT_ARCH_ARM64)
3442	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
3443	#else
3444	# error "Port me!"
3445	#endif
3446	}
3447
3448	/**
3449	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
3450	*/
3451	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3452	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3453	{
3454	#ifdef RT_ARCH_AMD64
3455	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
3456	#elif defined(RT_ARCH_ARM64)
3457	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
3458	#else
3459	# error "Port me!"
3460	#endif
3461	}
3462
3463
3464	/**
3465	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
3466	*/
3467	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3468	{
3469	#ifdef RT_ARCH_AMD64
3470	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
3471	#elif defined(RT_ARCH_ARM64)
3472	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
3473	#else
3474	# error "Port me!"
3475	#endif
3476	}
3477
3478	/**
3479	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
3480	*/
3481	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3482	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3483	{
3484	#ifdef RT_ARCH_AMD64
3485	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
3486	#elif defined(RT_ARCH_ARM64)
3487	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
3488	#else
3489	# error "Port me!"
3490	#endif
3491	}
3492
3493
3494	/**
3495	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
3496	*/
3497	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
3498	{
3499	#ifdef RT_ARCH_AMD64
3500	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
3501	#elif defined(RT_ARCH_ARM64)
3502	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
3503	#else
3504	# error "Port me!"
3505	#endif
3506	}
3507
3508	/**
3509	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
3510	*/
3511	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3512	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3513	{
3514	#ifdef RT_ARCH_AMD64
3515	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
3516	#elif defined(RT_ARCH_ARM64)
3517	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
3518	#else
3519	# error "Port me!"
3520	#endif
3521	}
3522
3523
3524	/**
3525	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
3526	*
3527	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
3528	*
3529	* Only use hardcoded jumps forward when emitting for exactly one
3530	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
3531	* the right target address on all platforms!
3532	*
3533	* Please also note that on x86 it is necessary pass off + 256 or higher
3534	* for @a offTarget one believe the intervening code is more than 127
3535	* bytes long.
3536	*/
3537	DECL_FORCE_INLINE(uint32_t)
3538	iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
3539	{
3540	#ifdef RT_ARCH_AMD64
3541	/* jcc rel8 / rel32 */
3542	int32_t offDisp = (int32_t)(offTarget - (off + 2));
3543	if (offDisp < 128 && offDisp >= -128)
3544	{
3545	pCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
3546	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3547	}
3548	else
3549	{
3550	offDisp -= 4;
3551	pCodeBuf[off++] = 0x0f;
3552	pCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
3553	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3554	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3555	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3556	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3557	}
3558
3559	#elif defined(RT_ARCH_ARM64)
3560	pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
3561
3562	#else
3563	# error "Port me!"
3564	#endif
3565	return off;
3566	}
3567
3568
3569	/**
3570	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
3571	*
3572	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
3573	*
3574	* Only use hardcoded jumps forward when emitting for exactly one
3575	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
3576	* the right target address on all platforms!
3577	*
3578	* Please also note that on x86 it is necessary pass off + 256 or higher
3579	* for @a offTarget one believe the intervening code is more than 127
3580	* bytes long.
3581	*/
3582	DECL_INLINE_THROW(uint32_t)
3583	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
3584	{
3585	#ifdef RT_ARCH_AMD64
3586	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
3587	#elif defined(RT_ARCH_ARM64)
3588	off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
3589	#else
3590	# error "Port me!"
3591	#endif
3592	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3593	return off;
3594	}
3595
3596
3597	/**
3598	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
3599	*
3600	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3601	*/
3602	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3603	{
3604	#ifdef RT_ARCH_AMD64
3605	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
3606	#elif defined(RT_ARCH_ARM64)
3607	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
3608	#else
3609	# error "Port me!"
3610	#endif
3611	}
3612
3613
3614	/**
3615	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
3616	*
3617	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3618	*/
3619	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3620	{
3621	#ifdef RT_ARCH_AMD64
3622	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
3623	#elif defined(RT_ARCH_ARM64)
3624	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
3625	#else
3626	# error "Port me!"
3627	#endif
3628	}
3629
3630
3631	/**
3632	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
3633	*
3634	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3635	*/
3636	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3637	{
3638	#ifdef RT_ARCH_AMD64
3639	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
3640	#elif defined(RT_ARCH_ARM64)
3641	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
3642	#else
3643	# error "Port me!"
3644	#endif
3645	}
3646
3647
3648	/**
3649	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
3650	*
3651	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3652	*/
3653	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3654	{
3655	#ifdef RT_ARCH_AMD64
3656	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3657	#elif defined(RT_ARCH_ARM64)
3658	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3659	#else
3660	# error "Port me!"
3661	#endif
3662	}
3663
3664
3665	/**
3666	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
3667	*
3668	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3669	*/
3670	DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
3671	{
3672	#ifdef RT_ARCH_AMD64
3673	/* jmp rel8 or rel32 */
3674	int32_t offDisp = offTarget - (off + 2);
3675	if (offDisp < 128 && offDisp >= -128)
3676	{
3677	pCodeBuf[off++] = 0xeb;
3678	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3679	}
3680	else
3681	{
3682	offDisp -= 3;
3683	pCodeBuf[off++] = 0xe9;
3684	pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3685	pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3686	pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3687	pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3688	}
3689
3690	#elif defined(RT_ARCH_ARM64)
3691	pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
3692
3693	#else
3694	# error "Port me!"
3695	#endif
3696	return off;
3697	}
3698
3699
3700	/**
3701	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
3702	*
3703	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3704	*/
3705	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3706	{
3707	#ifdef RT_ARCH_AMD64
3708	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
3709	#elif defined(RT_ARCH_ARM64)
3710	off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
3711	#else
3712	# error "Port me!"
3713	#endif
3714	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3715	return off;
3716	}
3717
3718
3719	/**
3720	* Fixes up a conditional jump to a fixed label.
3721	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
3722	* iemNativeEmitJzToFixed, ...
3723	*/
3724	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3725	{
3726	#ifdef RT_ARCH_AMD64
3727	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3728	uint8_t const bOpcode = pbCodeBuf[offFixup];
3729	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
3730	{
3731	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3732	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
3733	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
3734	}
3735	else
3736	{
3737	if (bOpcode != 0x0f)
3738	Assert(bOpcode == 0xe9);
3739	else
3740	{
3741	offFixup += 1;
3742	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
3743	}
3744	uint32_t const offRel32 = offTarget - (offFixup + 5);
3745	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
3746	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
3747	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
3748	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
3749	}
3750
3751	#elif defined(RT_ARCH_ARM64)
3752	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3753	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
3754	{
3755	/* B.COND + BC.COND */
3756	int32_t const offDisp = offTarget - offFixup;
3757	Assert(offDisp >= -262144 && offDisp < 262144);
3758	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
3759	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
3760	}
3761	else
3762	{
3763	/* B imm26 */
3764	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
3765	int32_t const offDisp = offTarget - offFixup;
3766	Assert(offDisp >= -33554432 && offDisp < 33554432);
3767	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
3768	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
3769	}
3770
3771	#else
3772	# error "Port me!"
3773	#endif
3774	}
3775
3776
3777	/**
3778	* Internal helper, don't call directly.
3779	*/
3780	DECL_INLINE_THROW(uint32_t)
3781	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3782	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3783	{
3784	Assert(iBitNo < 64);
3785	#ifdef RT_ARCH_AMD64
3786	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3787	if (iBitNo < 8)
3788	{
3789	/* test Eb, imm8 */
3790	if (iGprSrc >= 4)
3791	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3792	pbCodeBuf[off++] = 0xf6;
3793	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3794	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3795	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3796	}
3797	else
3798	{
3799	/* bt Ev, imm8 */
3800	if (iBitNo >= 32)
3801	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3802	else if (iGprSrc >= 8)
3803	pbCodeBuf[off++] = X86_OP_REX_B;
3804	pbCodeBuf[off++] = 0x0f;
3805	pbCodeBuf[off++] = 0xba;
3806	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3807	pbCodeBuf[off++] = iBitNo;
3808	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3809	}
3810
3811	#elif defined(RT_ARCH_ARM64)
3812	/* Use the TBNZ instruction here. */
3813	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3814	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3815	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3816
3817	#else
3818	# error "Port me!"
3819	#endif
3820	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3821	return off;
3822	}
3823
3824
3825	/**
3826	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3827	* @a iGprSrc.
3828	*
3829	* @note On ARM64 the range is only +/-8191 instructions.
3830	*/
3831	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3832	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3833	{
3834	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
3835	}
3836
3837
3838	/**
3839	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3840	* _set_ in @a iGprSrc.
3841	*
3842	* @note On ARM64 the range is only +/-8191 instructions.
3843	*/
3844	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3845	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3846	{
3847	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
3848	}
3849
3850
3851	/**
3852	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3853	* flags accordingly.
3854	*/
3855	DECL_INLINE_THROW(uint32_t)
3856	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3857	{
3858	Assert(fBits != 0);
3859	#ifdef RT_ARCH_AMD64
3860
3861	if (fBits >= UINT32_MAX)
3862	{
3863	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3864
3865	/* test Ev,Gv */
3866	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3867	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3868	pbCodeBuf[off++] = 0x85;
3869	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3870
3871	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3872	}
3873	else if (fBits <= UINT32_MAX)
3874	{
3875	/* test Eb, imm8 or test Ev, imm32 */
3876	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3877	if (fBits <= UINT8_MAX)
3878	{
3879	if (iGprSrc >= 4)
3880	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3881	pbCodeBuf[off++] = 0xf6;
3882	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3883	pbCodeBuf[off++] = (uint8_t)fBits;
3884	}
3885	else
3886	{
3887	if (iGprSrc >= 8)
3888	pbCodeBuf[off++] = X86_OP_REX_B;
3889	pbCodeBuf[off++] = 0xf7;
3890	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3891	pbCodeBuf[off++] = RT_BYTE1(fBits);
3892	pbCodeBuf[off++] = RT_BYTE2(fBits);
3893	pbCodeBuf[off++] = RT_BYTE3(fBits);
3894	pbCodeBuf[off++] = RT_BYTE4(fBits);
3895	}
3896	}
3897	/** @todo implement me. */
3898	else
3899	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3900
3901	#elif defined(RT_ARCH_ARM64)
3902
3903	if (false)
3904	{
3905	/** @todo figure out how to work the immr / N:imms constants. */
3906	}
3907	else
3908	{
3909	/* ands Zr, iGprSrc, iTmpReg */
3910	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3911	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3912	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3913	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3914	}
3915
3916	#else
3917	# error "Port me!"
3918	#endif
3919	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3920	return off;
3921	}
3922
3923
3924	/**
3925	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
3926	* @a iGprSrc, setting CPU flags accordingly.
3927	*
3928	* @note For ARM64 this only supports @a fBits values that can be expressed
3929	* using the two 6-bit immediates of the ANDS instruction. The caller
3930	* must make sure this is possible!
3931	*/
3932	DECL_FORCE_INLINE_THROW(uint32_t)
3933	iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
3934	{
3935	Assert(fBits != 0);
3936
3937	/* test Eb, imm8 */
3938	#ifdef RT_ARCH_AMD64
3939	if (iGprSrc >= 4)
3940	pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3941	pCodeBuf[off++] = 0xf6;
3942	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3943	pCodeBuf[off++] = fBits;
3944
3945	#elif defined(RT_ARCH_ARM64)
3946	/* ands xzr, src, [tmp\|#imm] */
3947	uint32_t uImmR = 0;
3948	uint32_t uImmNandS = 0;
3949	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
3950	pCodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3951	else
3952	# ifdef IEM_WITH_THROW_CATCH
3953	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
3954	# else
3955	AssertReleaseFailedStmt(off = UINT32_MAX);
3956	# endif
3957
3958	#else
3959	# error "Port me!"
3960	#endif
3961	return off;
3962	}
3963
3964
3965	/**
3966	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
3967	* @a iGprSrc, setting CPU flags accordingly.
3968	*/
3969	DECL_INLINE_THROW(uint32_t)
3970	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
3971	{
3972	Assert(fBits != 0);
3973
3974	#ifdef RT_ARCH_AMD64
3975	off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
3976
3977	#elif defined(RT_ARCH_ARM64)
3978	/* ands xzr, src, [tmp\|#imm] */
3979	uint32_t uImmR = 0;
3980	uint32_t uImmNandS = 0;
3981	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
3982	{
3983	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3984	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3985	}
3986	else
3987	{
3988	/* Use temporary register for the 64-bit immediate. */
3989	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3990	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3991	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
3992	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3993	}
3994
3995	#else
3996	# error "Port me!"
3997	#endif
3998	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3999	return off;
4000	}
4001
4002
4003	/**
4004	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
4005	* are set in @a iGprSrc.
4006	*/
4007	DECL_INLINE_THROW(uint32_t)
4008	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4009	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
4010	{
4011	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
4012
4013	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
4014	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4015
4016	return off;
4017	}
4018
4019
4020	/**
4021	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
4022	* are set in @a iGprSrc.
4023	*/
4024	DECL_INLINE_THROW(uint32_t)
4025	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4026	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
4027	{
4028	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
4029
4030	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
4031	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
4032
4033	return off;
4034	}
4035
4036
4037	/**
4038	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
4039	*
4040	* The operand size is given by @a f64Bit.
4041	*/
4042	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4043	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
4044	{
4045	Assert(idxLabel < pReNative->cLabels);
4046
4047	#ifdef RT_ARCH_AMD64
4048	/* test reg32,reg32 / test reg64,reg64 */
4049	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4050	if (f64Bit)
4051	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
4052	else if (iGprSrc >= 8)
4053	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
4054	pbCodeBuf[off++] = 0x85;
4055	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
4056	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4057
4058	/* jz idxLabel */
4059	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
4060
4061	#elif defined(RT_ARCH_ARM64)
4062	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4063	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4064	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
4065	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4066
4067	#else
4068	# error "Port me!"
4069	#endif
4070	return off;
4071	}
4072
4073
4074	/**
4075	* Emits code that jumps to a new label if @a iGprSrc is zero.
4076	*
4077	* The operand size is given by @a f64Bit.
4078	*/
4079	DECL_INLINE_THROW(uint32_t)
4080	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
4081	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4082	{
4083	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4084	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
4085	}
4086
4087
4088	/**
4089	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
4090	*
4091	* The operand size is given by @a f64Bit.
4092	*/
4093	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4094	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
4095	{
4096	Assert(idxLabel < pReNative->cLabels);
4097
4098	#ifdef RT_ARCH_AMD64
4099	/* test reg32,reg32 / test reg64,reg64 */
4100	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
4101	if (f64Bit)
4102	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
4103	else if (iGprSrc >= 8)
4104	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
4105	pbCodeBuf[off++] = 0x85;
4106	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
4107	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4108
4109	/* jnz idxLabel */
4110	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4111
4112	#elif defined(RT_ARCH_ARM64)
4113	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4114	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
4115	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(true /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
4116	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4117
4118	#else
4119	# error "Port me!"
4120	#endif
4121	return off;
4122	}
4123
4124
4125	/**
4126	* Emits code that jumps to a new label if @a iGprSrc is not zero.
4127	*
4128	* The operand size is given by @a f64Bit.
4129	*/
4130	DECL_INLINE_THROW(uint32_t)
4131	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
4132	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4133	{
4134	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4135	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
4136	}
4137
4138
4139	/**
4140	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
4141	* differs.
4142	*/
4143	DECL_INLINE_THROW(uint32_t)
4144	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4145	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
4146	{
4147	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
4148	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4149	return off;
4150	}
4151
4152
4153	/**
4154	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
4155	*/
4156	DECL_INLINE_THROW(uint32_t)
4157	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4158	uint8_t iGprLeft, uint8_t iGprRight,
4159	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4160	{
4161	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4162	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
4163	}
4164
4165
4166	/**
4167	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
4168	*/
4169	DECL_INLINE_THROW(uint32_t)
4170	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4171	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
4172	{
4173	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
4174	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4175	return off;
4176	}
4177
4178
4179	/**
4180	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
4181	*/
4182	DECL_INLINE_THROW(uint32_t)
4183	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4184	uint8_t iGprSrc, uint64_t uImm,
4185	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4186	{
4187	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4188	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
4189	}
4190
4191
4192	/**
4193	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
4194	* @a uImm.
4195	*/
4196	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4197	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
4198	{
4199	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
4200	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
4201	return off;
4202	}
4203
4204
4205	/**
4206	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
4207	* @a uImm.
4208	*/
4209	DECL_INLINE_THROW(uint32_t)
4210	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
4211	uint8_t iGprSrc, uint32_t uImm,
4212	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
4213	{
4214	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
4215	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
4216	}
4217
4218
4219	/*********************************************************************************************************************************
4220	* Calls. *
4221	*********************************************************************************************************************************/
4222
4223	/**
4224	* Emits a call to a 64-bit address.
4225	*/
4226	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
4227	{
4228	#ifdef RT_ARCH_AMD64
4229	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
4230
4231	/* call rax */
4232	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
4233	pbCodeBuf[off++] = 0xff;
4234	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
4235
4236	#elif defined(RT_ARCH_ARM64)
4237	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
4238
4239	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
4240	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
4241
4242	#else
4243	# error "port me"
4244	#endif
4245	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
4246	return off;
4247	}
4248
4249
4250	/**
4251	* Emits code to load a stack variable into an argument GPR.
4252	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
4253	*/
4254	DECL_FORCE_INLINE_THROW(uint32_t)
4255	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4256	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
4257	{
4258	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4259	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
4260	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
4261
4262	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
4263	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
4264	{
4265	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK) \|\| fVarAllowInVolatileReg);
4266	RT_NOREF(fVarAllowInVolatileReg);
4267	if (!offAddend)
4268	{
4269	if (idxRegArg != idxRegVar)
4270	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
4271	}
4272	else
4273	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
4274	}
4275	else
4276	{
4277	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
4278	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
4279	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
4280	if (offAddend)
4281	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
4282	}
4283	return off;
4284	}
4285
4286
4287	/**
4288	* Emits code to load a stack or immediate variable value into an argument GPR,
4289	* optional with a addend.
4290	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
4291	*/
4292	DECL_FORCE_INLINE_THROW(uint32_t)
4293	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4294	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
4295	{
4296	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4297	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
4298	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
4299	else
4300	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend, fVarAllowInVolatileReg);
4301	return off;
4302	}
4303
4304
4305	/**
4306	* Emits code to load the variable address into an argument GRP.
4307	*
4308	* This only works for uninitialized and stack variables.
4309	*/
4310	DECL_FORCE_INLINE_THROW(uint32_t)
4311	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
4312	bool fFlushShadows)
4313	{
4314	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
4315	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
4316	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
4317	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
4318
4319	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
4320	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
4321
4322	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
4323	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
4324	{
4325	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
4326	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
4327	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
4328	}
4329	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
4330	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
4331
4332	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
4333	}
4334
4335
4336	/** @} */
4337
4338	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
4339

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102720

Download in other formats: