IEMN8veRecompilerEmit.h@ 102687

Last change on this file since 102687 was 102662, checked in by vboxsync, 17 months ago
VMM/IEM: Fixed typo in amd64 part of iemNativeEmitAddTwoGprs and a missing shift in the arm64 part of iemNativeEmitCmpGprWithImm. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 122.0 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102662 2023-12-21 01:39:13Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
88	pbCodeBuf[off++] = 0xcc;
89	RT_NOREF(uInfo);
90
91	#elif defined(RT_ARCH_ARM64)
92	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
93	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
94
95	#else
96	# error "error"
97	#endif
98	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
99	return off;
100	}
101
102
103	/*********************************************************************************************************************************
104	* Loads, Stores and Related Stuff. *
105	*********************************************************************************************************************************/
106
107	#ifdef RT_ARCH_AMD64
108	/**
109	* Common bit of iemNativeEmitLoadGprByGpr and friends.
110	*/
111	DECL_FORCE_INLINE(uint32_t)
112	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
113	{
114	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
115	{
116	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
117	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
118	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
119	}
120	else if (offDisp == (int8_t)offDisp)
121	{
122	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
123	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
124	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
125	pbCodeBuf[off++] = (uint8_t)offDisp;
126	}
127	else
128	{
129	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
130	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
131	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
132	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
133	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
134	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
135	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
136	}
137	return off;
138	}
139	#endif /* RT_ARCH_AMD64 */
140
141	/**
142	* Emits setting a GPR to zero.
143	*/
144	DECL_INLINE_THROW(uint32_t)
145	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
146	{
147	#ifdef RT_ARCH_AMD64
148	/* xor gpr32, gpr32 */
149	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
150	if (iGpr >= 8)
151	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
152	pbCodeBuf[off++] = 0x33;
153	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
154
155	#elif defined(RT_ARCH_ARM64)
156	/* mov gpr, #0x0 */
157	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
158	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
159
160	#else
161	# error "port me"
162	#endif
163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
164	return off;
165	}
166
167
168	/**
169	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
170	* buffer space.
171	*
172	* Max buffer consumption:
173	* - AMD64: 10 instruction bytes.
174	* - ARM64: 4 instruction words (16 bytes).
175	*/
176	DECLINLINE(uint32_t) iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
177	{
178	#ifdef RT_ARCH_AMD64
179	if (uImm64 == 0)
180	{
181	/* xor gpr, gpr */
182	if (iGpr >= 8)
183	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
184	pCodeBuf[off++] = 0x33;
185	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
186	}
187	else if (uImm64 <= UINT32_MAX)
188	{
189	/* mov gpr, imm32 */
190	if (iGpr >= 8)
191	pCodeBuf[off++] = X86_OP_REX_B;
192	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
193	pCodeBuf[off++] = RT_BYTE1(uImm64);
194	pCodeBuf[off++] = RT_BYTE2(uImm64);
195	pCodeBuf[off++] = RT_BYTE3(uImm64);
196	pCodeBuf[off++] = RT_BYTE4(uImm64);
197	}
198	else
199	{
200	/* mov gpr, imm64 */
201	if (iGpr < 8)
202	pCodeBuf[off++] = X86_OP_REX_W;
203	else
204	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
205	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
206	pCodeBuf[off++] = RT_BYTE1(uImm64);
207	pCodeBuf[off++] = RT_BYTE2(uImm64);
208	pCodeBuf[off++] = RT_BYTE3(uImm64);
209	pCodeBuf[off++] = RT_BYTE4(uImm64);
210	pCodeBuf[off++] = RT_BYTE5(uImm64);
211	pCodeBuf[off++] = RT_BYTE6(uImm64);
212	pCodeBuf[off++] = RT_BYTE7(uImm64);
213	pCodeBuf[off++] = RT_BYTE8(uImm64);
214	}
215
216	#elif defined(RT_ARCH_ARM64)
217	/*
218	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
219	* supply remaining bits using 'movk grp, imm16, lsl #x'.
220	*
221	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
222	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
223	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
224	* after the first non-zero immediate component so we switch to movk for
225	* the remainder.
226	*/
227	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
228	+ !((uImm64 >> 16) & UINT16_MAX)
229	+ !((uImm64 >> 32) & UINT16_MAX)
230	+ !((uImm64 >> 48) & UINT16_MAX);
231	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
232	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
233	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
234	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
235	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
236	if (cFfffHalfWords <= cZeroHalfWords)
237	{
238	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
239
240	/* movz gpr, imm16 */
241	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
242	if (uImmPart \|\| cZeroHalfWords == 4)
243	{
244	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
245	fMovBase \|= RT_BIT_32(29);
246	}
247	/* mov[z/k] gpr, imm16, lsl #16 */
248	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
249	if (uImmPart)
250	{
251	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
252	fMovBase \|= RT_BIT_32(29);
253	}
254	/* mov[z/k] gpr, imm16, lsl #32 */
255	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
256	if (uImmPart)
257	{
258	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
259	fMovBase \|= RT_BIT_32(29);
260	}
261	/* mov[z/k] gpr, imm16, lsl #48 */
262	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
263	if (uImmPart)
264	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
265	}
266	else
267	{
268	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
269
270	/* find the first half-word that isn't UINT16_MAX. */
271	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
272	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
273	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
274
275	/* movn gpr, imm16, lsl #iHwNotFfff16 /
276	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
277	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
278	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
279	/* movk gpr, imm16 */
280	if (iHwNotFfff != 0)
281	{
282	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
283	if (uImmPart != UINT32_C(0xffff))
284	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
285	}
286	/* movk gpr, imm16, lsl #16 */
287	if (iHwNotFfff != 1)
288	{
289	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
290	if (uImmPart != UINT32_C(0xffff))
291	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
292	}
293	/* movk gpr, imm16, lsl #32 */
294	if (iHwNotFfff != 2)
295	{
296	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
297	if (uImmPart != UINT32_C(0xffff))
298	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
299	}
300	/* movk gpr, imm16, lsl #48 */
301	if (iHwNotFfff != 3)
302	{
303	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
304	if (uImmPart != UINT32_C(0xffff))
305	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
306	}
307	}
308
309	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
310	* clang 12.x does that, only to use the 'x' version for the
311	* addressing in the following ldr). */
312
313	#else
314	# error "port me"
315	#endif
316	return off;
317	}
318
319
320	/**
321	* Emits loading a constant into a 64-bit GPR
322	*/
323	DECL_INLINE_THROW(uint32_t)
324	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
325	{
326	#ifdef RT_ARCH_AMD64
327	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
328	#elif defined(RT_ARCH_ARM64)
329	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
330	#else
331	# error "port me"
332	#endif
333	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
334	return off;
335	}
336
337
338	/**
339	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
340	* buffer space.
341	*
342	* Max buffer consumption:
343	* - AMD64: 6 instruction bytes.
344	* - ARM64: 2 instruction words (8 bytes).
345	*
346	* @note The top 32 bits will be cleared.
347	*/
348	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
349	{
350	#ifdef RT_ARCH_AMD64
351	if (uImm32 == 0)
352	{
353	/* xor gpr, gpr */
354	if (iGpr >= 8)
355	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
356	pCodeBuf[off++] = 0x33;
357	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
358	}
359	else
360	{
361	/* mov gpr, imm32 */
362	if (iGpr >= 8)
363	pCodeBuf[off++] = X86_OP_REX_B;
364	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
365	pCodeBuf[off++] = RT_BYTE1(uImm32);
366	pCodeBuf[off++] = RT_BYTE2(uImm32);
367	pCodeBuf[off++] = RT_BYTE3(uImm32);
368	pCodeBuf[off++] = RT_BYTE4(uImm32);
369	}
370
371	#elif defined(RT_ARCH_ARM64)
372	if ((uImm32 >> 16) == 0)
373	/* movz gpr, imm16 */
374	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
375	else if ((uImm32 & UINT32_C(0xffff)) == 0)
376	/* movz gpr, imm16, lsl #16 */
377	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
378	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
379	/* movn gpr, imm16, lsl #16 */
380	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
381	else if ((uImm32 >> 16) == UINT32_C(0xffff))
382	/* movn gpr, imm16 */
383	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
384	else
385	{
386	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
387	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
388	}
389
390	#else
391	# error "port me"
392	#endif
393	return off;
394	}
395
396
397	/**
398	* Emits loading a constant into a 32-bit GPR.
399	* @note The top 32 bits will be cleared.
400	*/
401	DECL_INLINE_THROW(uint32_t)
402	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
403	{
404	#ifdef RT_ARCH_AMD64
405	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
406	#elif defined(RT_ARCH_ARM64)
407	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
408	#else
409	# error "port me"
410	#endif
411	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
412	return off;
413	}
414
415
416	/**
417	* Emits loading a constant into a 8-bit GPR
418	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
419	* only the ARM64 version does that.
420	*/
421	DECL_INLINE_THROW(uint32_t)
422	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
423	{
424	#ifdef RT_ARCH_AMD64
425	/* mov gpr, imm8 */
426	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
427	if (iGpr >= 8)
428	pbCodeBuf[off++] = X86_OP_REX_B;
429	else if (iGpr >= 4)
430	pbCodeBuf[off++] = X86_OP_REX;
431	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
432	pbCodeBuf[off++] = RT_BYTE1(uImm8);
433
434	#elif defined(RT_ARCH_ARM64)
435	/* movz gpr, imm16, lsl #0 */
436	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
437	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
438
439	#else
440	# error "port me"
441	#endif
442	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
443	return off;
444	}
445
446
447	#ifdef RT_ARCH_AMD64
448	/**
449	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
450	*/
451	DECL_FORCE_INLINE(uint32_t)
452	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
453	{
454	if (offVCpu < 128)
455	{
456	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
457	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
458	}
459	else
460	{
461	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
462	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
463	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
464	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
465	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
466	}
467	return off;
468	}
469	#elif defined(RT_ARCH_ARM64)
470	/**
471	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
472	*/
473	DECL_FORCE_INLINE_THROW(uint32_t)
474	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
475	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
476	{
477	/*
478	* There are a couple of ldr variants that takes an immediate offset, so
479	* try use those if we can, otherwise we have to use the temporary register
480	* help with the addressing.
481	*/
482	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
483	{
484	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
485	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
486	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
487	}
488	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
489	{
490	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
491	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
492	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
493	}
494	else
495	{
496	/* The offset is too large, so we must load it into a register and use
497	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
498	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
499	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
500
501	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
502	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
503	IEMNATIVE_REG_FIXED_TMP0);
504	}
505	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
506	return off;
507	}
508	#endif
509
510
511	/**
512	* Emits a 64-bit GPR load of a VCpu value.
513	*/
514	DECL_INLINE_THROW(uint32_t)
515	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
516	{
517	#ifdef RT_ARCH_AMD64
518	/* mov reg64, mem64 */
519	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
520	if (iGpr < 8)
521	pbCodeBuf[off++] = X86_OP_REX_W;
522	else
523	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
524	pbCodeBuf[off++] = 0x8b;
525	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
526	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
527
528	#elif defined(RT_ARCH_ARM64)
529	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
530
531	#else
532	# error "port me"
533	#endif
534	return off;
535	}
536
537
538	/**
539	* Emits a 32-bit GPR load of a VCpu value.
540	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
541	*/
542	DECL_INLINE_THROW(uint32_t)
543	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
544	{
545	#ifdef RT_ARCH_AMD64
546	/* mov reg32, mem32 */
547	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
548	if (iGpr >= 8)
549	pbCodeBuf[off++] = X86_OP_REX_R;
550	pbCodeBuf[off++] = 0x8b;
551	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
552	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
553
554	#elif defined(RT_ARCH_ARM64)
555	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
556
557	#else
558	# error "port me"
559	#endif
560	return off;
561	}
562
563
564	/**
565	* Emits a 16-bit GPR load of a VCpu value.
566	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
567	*/
568	DECL_INLINE_THROW(uint32_t)
569	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
570	{
571	#ifdef RT_ARCH_AMD64
572	/* movzx reg32, mem16 */
573	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
574	if (iGpr >= 8)
575	pbCodeBuf[off++] = X86_OP_REX_R;
576	pbCodeBuf[off++] = 0x0f;
577	pbCodeBuf[off++] = 0xb7;
578	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
579	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
580
581	#elif defined(RT_ARCH_ARM64)
582	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
583
584	#else
585	# error "port me"
586	#endif
587	return off;
588	}
589
590
591	/**
592	* Emits a 8-bit GPR load of a VCpu value.
593	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
594	*/
595	DECL_INLINE_THROW(uint32_t)
596	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
597	{
598	#ifdef RT_ARCH_AMD64
599	/* movzx reg32, mem8 */
600	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
601	if (iGpr >= 8)
602	pbCodeBuf[off++] = X86_OP_REX_R;
603	pbCodeBuf[off++] = 0x0f;
604	pbCodeBuf[off++] = 0xb6;
605	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
606	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
607
608	#elif defined(RT_ARCH_ARM64)
609	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
610
611	#else
612	# error "port me"
613	#endif
614	return off;
615	}
616
617
618	/**
619	* Emits a store of a GPR value to a 64-bit VCpu field.
620	*/
621	DECL_INLINE_THROW(uint32_t)
622	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
623	{
624	#ifdef RT_ARCH_AMD64
625	/* mov mem64, reg64 */
626	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
627	if (iGpr < 8)
628	pbCodeBuf[off++] = X86_OP_REX_W;
629	else
630	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
631	pbCodeBuf[off++] = 0x89;
632	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
633	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
634
635	#elif defined(RT_ARCH_ARM64)
636	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
637
638	#else
639	# error "port me"
640	#endif
641	return off;
642	}
643
644
645	/**
646	* Emits a store of a GPR value to a 32-bit VCpu field.
647	*/
648	DECL_INLINE_THROW(uint32_t)
649	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
650	{
651	#ifdef RT_ARCH_AMD64
652	/* mov mem32, reg32 */
653	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
654	if (iGpr >= 8)
655	pbCodeBuf[off++] = X86_OP_REX_R;
656	pbCodeBuf[off++] = 0x89;
657	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
658	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
659
660	#elif defined(RT_ARCH_ARM64)
661	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
662
663	#else
664	# error "port me"
665	#endif
666	return off;
667	}
668
669
670	/**
671	* Emits a store of a GPR value to a 16-bit VCpu field.
672	*/
673	DECL_INLINE_THROW(uint32_t)
674	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
675	{
676	#ifdef RT_ARCH_AMD64
677	/* mov mem16, reg16 */
678	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
679	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
680	if (iGpr >= 8)
681	pbCodeBuf[off++] = X86_OP_REX_R;
682	pbCodeBuf[off++] = 0x89;
683	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
684	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
685
686	#elif defined(RT_ARCH_ARM64)
687	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
688
689	#else
690	# error "port me"
691	#endif
692	return off;
693	}
694
695
696	/**
697	* Emits a store of a GPR value to a 8-bit VCpu field.
698	*/
699	DECL_INLINE_THROW(uint32_t)
700	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
701	{
702	#ifdef RT_ARCH_AMD64
703	/* mov mem8, reg8 */
704	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
705	if (iGpr >= 8)
706	pbCodeBuf[off++] = X86_OP_REX_R;
707	pbCodeBuf[off++] = 0x88;
708	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
709	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
710
711	#elif defined(RT_ARCH_ARM64)
712	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
713
714	#else
715	# error "port me"
716	#endif
717	return off;
718	}
719
720
721	/**
722	* Emits a store of an immediate value to a 8-bit VCpu field.
723	*/
724	DECL_INLINE_THROW(uint32_t)
725	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
726	{
727	#ifdef RT_ARCH_AMD64
728	/* mov mem8, imm8 */
729	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
730	pbCodeBuf[off++] = 0xc6;
731	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
732	pbCodeBuf[off++] = bImm;
733	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
734
735	#elif defined(RT_ARCH_ARM64)
736	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
737	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
738	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
739	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
740
741	#else
742	# error "port me"
743	#endif
744	return off;
745	}
746
747
748	/**
749	* Emits a load effective address to a GRP of a VCpu field.
750	*/
751	DECL_INLINE_THROW(uint32_t)
752	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
753	{
754	#ifdef RT_ARCH_AMD64
755	/* lea gprdst, [rbx + offDisp] */
756	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
757	if (iGprDst < 8)
758	pbCodeBuf[off++] = X86_OP_REX_W;
759	else
760	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
761	pbCodeBuf[off++] = 0x8d;
762	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
763
764	#elif defined(RT_ARCH_ARM64)
765	if (offVCpu < (unsigned)_4K)
766	{
767	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
768	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
769	}
770	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
771	{
772	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
773	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
774	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
775	}
776	else
777	{
778	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
779	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
780	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
781	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
782	}
783
784	#else
785	# error "port me"
786	#endif
787	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
788	return off;
789	}
790
791
792	/**
793	* Emits a gprdst = gprsrc load.
794	*/
795	DECL_INLINE_THROW(uint32_t)
796	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
797	{
798	#ifdef RT_ARCH_AMD64
799	/* mov gprdst, gprsrc */
800	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
801	if ((iGprDst \| iGprSrc) >= 8)
802	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
803	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
804	: X86_OP_REX_W \| X86_OP_REX_R;
805	else
806	pbCodeBuf[off++] = X86_OP_REX_W;
807	pbCodeBuf[off++] = 0x8b;
808	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
809
810	#elif defined(RT_ARCH_ARM64)
811	/* mov dst, src; alias for: orr dst, xzr, src */
812	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
813	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
814
815	#else
816	# error "port me"
817	#endif
818	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
819	return off;
820	}
821
822
823	/**
824	* Emits a gprdst = gprsrc[31:0] load.
825	* @note Bits 63 thru 32 are cleared.
826	*/
827	DECL_INLINE_THROW(uint32_t)
828	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
829	{
830	#ifdef RT_ARCH_AMD64
831	/* mov gprdst, gprsrc */
832	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
833	if ((iGprDst \| iGprSrc) >= 8)
834	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
835	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
836	: X86_OP_REX_R;
837	pbCodeBuf[off++] = 0x8b;
838	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
839
840	#elif defined(RT_ARCH_ARM64)
841	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
842	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
843	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
844
845	#else
846	# error "port me"
847	#endif
848	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
849	return off;
850	}
851
852
853	/**
854	* Emits a gprdst = gprsrc[15:0] load.
855	* @note Bits 63 thru 15 are cleared.
856	*/
857	DECL_INLINE_THROW(uint32_t)
858	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
859	{
860	#ifdef RT_ARCH_AMD64
861	/* movzx Gv,Ew */
862	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
863	if ((iGprDst \| iGprSrc) >= 8)
864	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
865	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
866	: X86_OP_REX_R;
867	pbCodeBuf[off++] = 0x0f;
868	pbCodeBuf[off++] = 0xb7;
869	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
870
871	#elif defined(RT_ARCH_ARM64)
872	/* and gprdst, gprsrc, #0xffff */
873	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
874	# if 1
875	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
876	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
877	# else
878	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
879	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
880	# endif
881
882	#else
883	# error "port me"
884	#endif
885	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
886	return off;
887	}
888
889
890	/**
891	* Emits a gprdst = gprsrc[7:0] load.
892	* @note Bits 63 thru 8 are cleared.
893	*/
894	DECL_INLINE_THROW(uint32_t)
895	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
896	{
897	#ifdef RT_ARCH_AMD64
898	/* movzx Gv,Eb */
899	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
900	if (iGprDst >= 8 \|\| iGprSrc >= 8)
901	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
902	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
903	: X86_OP_REX_R;
904	else if (iGprSrc >= 4)
905	pbCodeBuf[off++] = X86_OP_REX;
906	pbCodeBuf[off++] = 0x0f;
907	pbCodeBuf[off++] = 0xb6;
908	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
909
910	#elif defined(RT_ARCH_ARM64)
911	/* and gprdst, gprsrc, #0xff */
912	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
913	# if 1
914	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
915	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
916	# else
917	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
918	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
919	# endif
920
921	#else
922	# error "port me"
923	#endif
924	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
925	return off;
926	}
927
928
929	/**
930	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
931	* @note Bits 63 thru 8 are cleared.
932	*/
933	DECL_INLINE_THROW(uint32_t)
934	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
935	{
936	#ifdef RT_ARCH_AMD64
937	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
938
939	/* movzx Gv,Ew */
940	if ((iGprDst \| iGprSrc) >= 8)
941	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
942	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
943	: X86_OP_REX_R;
944	pbCodeBuf[off++] = 0x0f;
945	pbCodeBuf[off++] = 0xb7;
946	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
947
948	/* shr Ev,8 */
949	if (iGprDst >= 8)
950	pbCodeBuf[off++] = X86_OP_REX_B;
951	pbCodeBuf[off++] = 0xc1;
952	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
953	pbCodeBuf[off++] = 8;
954
955	#elif defined(RT_ARCH_ARM64)
956	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
957	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
958	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
959
960	#else
961	# error "port me"
962	#endif
963	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
964	return off;
965	}
966
967
968	/**
969	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
970	*/
971	DECL_INLINE_THROW(uint32_t)
972	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
973	{
974	#ifdef RT_ARCH_AMD64
975	/* movsxd r64, r/m32 */
976	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
977	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
978	pbCodeBuf[off++] = 0x63;
979	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
980
981	#elif defined(RT_ARCH_ARM64)
982	/* sxtw dst, src */
983	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
984	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
985
986	#else
987	# error "port me"
988	#endif
989	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
990	return off;
991	}
992
993
994	/**
995	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
996	*/
997	DECL_INLINE_THROW(uint32_t)
998	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
999	{
1000	#ifdef RT_ARCH_AMD64
1001	/* movsx r64, r/m16 */
1002	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1003	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1004	pbCodeBuf[off++] = 0x0f;
1005	pbCodeBuf[off++] = 0xbf;
1006	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1007
1008	#elif defined(RT_ARCH_ARM64)
1009	/* sxth dst, src */
1010	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1011	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1012
1013	#else
1014	# error "port me"
1015	#endif
1016	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1017	return off;
1018	}
1019
1020
1021	/**
1022	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1023	*/
1024	DECL_INLINE_THROW(uint32_t)
1025	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1026	{
1027	#ifdef RT_ARCH_AMD64
1028	/* movsx r64, r/m16 */
1029	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1030	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1031	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1032	pbCodeBuf[off++] = 0x0f;
1033	pbCodeBuf[off++] = 0xbf;
1034	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1035
1036	#elif defined(RT_ARCH_ARM64)
1037	/* sxth dst32, src */
1038	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1039	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1040
1041	#else
1042	# error "port me"
1043	#endif
1044	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1045	return off;
1046	}
1047
1048
1049	/**
1050	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1051	*/
1052	DECL_INLINE_THROW(uint32_t)
1053	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1054	{
1055	#ifdef RT_ARCH_AMD64
1056	/* movsx r64, r/m8 */
1057	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1058	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1059	pbCodeBuf[off++] = 0x0f;
1060	pbCodeBuf[off++] = 0xbe;
1061	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1062
1063	#elif defined(RT_ARCH_ARM64)
1064	/* sxtb dst, src */
1065	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1066	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1067
1068	#else
1069	# error "port me"
1070	#endif
1071	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1072	return off;
1073	}
1074
1075
1076	/**
1077	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1078	* @note Bits 63 thru 32 are cleared.
1079	*/
1080	DECL_INLINE_THROW(uint32_t)
1081	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1082	{
1083	#ifdef RT_ARCH_AMD64
1084	/* movsx r32, r/m8 */
1085	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1086	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1087	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1088	pbCodeBuf[off++] = 0x0f;
1089	pbCodeBuf[off++] = 0xbe;
1090	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1091
1092	#elif defined(RT_ARCH_ARM64)
1093	/* sxtb dst32, src32 */
1094	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1095	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1096
1097	#else
1098	# error "port me"
1099	#endif
1100	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1101	return off;
1102	}
1103
1104
1105	/**
1106	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1107	* @note Bits 63 thru 16 are cleared.
1108	*/
1109	DECL_INLINE_THROW(uint32_t)
1110	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1111	{
1112	#ifdef RT_ARCH_AMD64
1113	/* movsx r16, r/m8 */
1114	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1115	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1116	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1117	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1118	pbCodeBuf[off++] = 0x0f;
1119	pbCodeBuf[off++] = 0xbe;
1120	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1121
1122	/* movzx r32, r/m16 */
1123	if (iGprDst >= 8)
1124	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1125	pbCodeBuf[off++] = 0x0f;
1126	pbCodeBuf[off++] = 0xb7;
1127	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1128
1129	#elif defined(RT_ARCH_ARM64)
1130	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1131	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1132	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1133	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1134	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1135
1136	#else
1137	# error "port me"
1138	#endif
1139	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1140	return off;
1141	}
1142
1143
1144	/**
1145	* Emits a gprdst = gprsrc + addend load.
1146	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1147	*/
1148	#ifdef RT_ARCH_AMD64
1149	DECL_INLINE_THROW(uint32_t)
1150	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1151	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1152	{
1153	Assert(iAddend != 0);
1154
1155	/* lea gprdst, [gprsrc + iAddend] */
1156	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1157	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1158	pbCodeBuf[off++] = 0x8d;
1159	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1160	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1161	return off;
1162	}
1163
1164	#elif defined(RT_ARCH_ARM64)
1165	DECL_INLINE_THROW(uint32_t)
1166	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1167	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1168	{
1169	if ((uint32_t)iAddend < 4096)
1170	{
1171	/* add dst, src, uimm12 */
1172	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1173	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1174	}
1175	else if ((uint32_t)-iAddend < 4096)
1176	{
1177	/* sub dst, src, uimm12 */
1178	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1179	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1180	}
1181	else
1182	{
1183	Assert(iGprSrc != iGprDst);
1184	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1185	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1186	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1187	}
1188	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1189	return off;
1190	}
1191	#else
1192	# error "port me"
1193	#endif
1194
1195	/**
1196	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1197	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1198	*/
1199	#ifdef RT_ARCH_AMD64
1200	DECL_INLINE_THROW(uint32_t)
1201	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1202	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1203	#else
1204	DECL_INLINE_THROW(uint32_t)
1205	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1206	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1207	#endif
1208	{
1209	if (iAddend != 0)
1210	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1211	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1212	}
1213
1214
1215	/**
1216	* Emits a gprdst = gprsrc32 + addend load.
1217	* @note Bits 63 thru 32 are cleared.
1218	*/
1219	DECL_INLINE_THROW(uint32_t)
1220	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1221	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1222	{
1223	Assert(iAddend != 0);
1224
1225	#ifdef RT_ARCH_AMD64
1226	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1227	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1228	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1229	if ((iGprDst \| iGprSrc) >= 8)
1230	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1231	pbCodeBuf[off++] = 0x8d;
1232	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1233
1234	#elif defined(RT_ARCH_ARM64)
1235	if ((uint32_t)iAddend < 4096)
1236	{
1237	/* add dst, src, uimm12 */
1238	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1239	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1240	}
1241	else if ((uint32_t)-iAddend < 4096)
1242	{
1243	/* sub dst, src, uimm12 */
1244	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1245	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1246	}
1247	else
1248	{
1249	Assert(iGprSrc != iGprDst);
1250	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1251	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1252	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1253	}
1254
1255	#else
1256	# error "port me"
1257	#endif
1258	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1259	return off;
1260	}
1261
1262
1263	/**
1264	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1265	*/
1266	DECL_INLINE_THROW(uint32_t)
1267	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1268	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1269	{
1270	if (iAddend != 0)
1271	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1272	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1273	}
1274
1275
1276
1277	#ifdef RT_ARCH_AMD64
1278	/**
1279	* Common bit of iemNativeEmitLoadGprByBp and friends.
1280	*/
1281	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1282	PIEMRECOMPILERSTATE pReNativeAssert)
1283	{
1284	if (offDisp < 128 && offDisp >= -128)
1285	{
1286	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1287	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1288	}
1289	else
1290	{
1291	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1292	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1293	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1294	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1295	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1296	}
1297	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1298	return off;
1299	}
1300	#elif defined(RT_ARCH_ARM64)
1301	/**
1302	* Common bit of iemNativeEmitLoadGprByBp and friends.
1303	*/
1304	DECL_FORCE_INLINE_THROW(uint32_t)
1305	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1306	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1307	{
1308	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1309	{
1310	/* str w/ unsigned imm12 (scaled) */
1311	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1312	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1313	}
1314	else if (offDisp >= -256 && offDisp <= 256)
1315	{
1316	/* stur w/ signed imm9 (unscaled) */
1317	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1318	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1319	}
1320	else
1321	{
1322	/* Use temporary indexing register. */
1323	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1324	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1325	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1326	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1327	}
1328	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1329	return off;
1330	}
1331	#endif
1332
1333
1334	/**
1335	* Emits a 64-bit GRP load instruction with an BP relative source address.
1336	*/
1337	DECL_INLINE_THROW(uint32_t)
1338	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1339	{
1340	#ifdef RT_ARCH_AMD64
1341	/* mov gprdst, qword [rbp + offDisp] */
1342	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1343	if (iGprDst < 8)
1344	pbCodeBuf[off++] = X86_OP_REX_W;
1345	else
1346	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1347	pbCodeBuf[off++] = 0x8b;
1348	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1349
1350	#elif defined(RT_ARCH_ARM64)
1351	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1352
1353	#else
1354	# error "port me"
1355	#endif
1356	}
1357
1358
1359	/**
1360	* Emits a 32-bit GRP load instruction with an BP relative source address.
1361	* @note Bits 63 thru 32 of the GPR will be cleared.
1362	*/
1363	DECL_INLINE_THROW(uint32_t)
1364	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1365	{
1366	#ifdef RT_ARCH_AMD64
1367	/* mov gprdst, dword [rbp + offDisp] */
1368	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1369	if (iGprDst >= 8)
1370	pbCodeBuf[off++] = X86_OP_REX_R;
1371	pbCodeBuf[off++] = 0x8b;
1372	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1373
1374	#elif defined(RT_ARCH_ARM64)
1375	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1376
1377	#else
1378	# error "port me"
1379	#endif
1380	}
1381
1382
1383	/**
1384	* Emits a 16-bit GRP load instruction with an BP relative source address.
1385	* @note Bits 63 thru 16 of the GPR will be cleared.
1386	*/
1387	DECL_INLINE_THROW(uint32_t)
1388	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1389	{
1390	#ifdef RT_ARCH_AMD64
1391	/* movzx gprdst, word [rbp + offDisp] */
1392	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1393	if (iGprDst >= 8)
1394	pbCodeBuf[off++] = X86_OP_REX_R;
1395	pbCodeBuf[off++] = 0x0f;
1396	pbCodeBuf[off++] = 0xb7;
1397	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1398
1399	#elif defined(RT_ARCH_ARM64)
1400	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1401
1402	#else
1403	# error "port me"
1404	#endif
1405	}
1406
1407
1408	/**
1409	* Emits a 8-bit GRP load instruction with an BP relative source address.
1410	* @note Bits 63 thru 8 of the GPR will be cleared.
1411	*/
1412	DECL_INLINE_THROW(uint32_t)
1413	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1414	{
1415	#ifdef RT_ARCH_AMD64
1416	/* movzx gprdst, byte [rbp + offDisp] */
1417	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1418	if (iGprDst >= 8)
1419	pbCodeBuf[off++] = X86_OP_REX_R;
1420	pbCodeBuf[off++] = 0x0f;
1421	pbCodeBuf[off++] = 0xb6;
1422	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1423
1424	#elif defined(RT_ARCH_ARM64)
1425	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1426
1427	#else
1428	# error "port me"
1429	#endif
1430	}
1431
1432
1433	/**
1434	* Emits a load effective address to a GRP with an BP relative source address.
1435	*/
1436	DECL_INLINE_THROW(uint32_t)
1437	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1438	{
1439	#ifdef RT_ARCH_AMD64
1440	/* lea gprdst, [rbp + offDisp] */
1441	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1442	if (iGprDst < 8)
1443	pbCodeBuf[off++] = X86_OP_REX_W;
1444	else
1445	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1446	pbCodeBuf[off++] = 0x8d;
1447	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1448
1449	#elif defined(RT_ARCH_ARM64)
1450	if ((uint32_t)offDisp < (unsigned)_4K)
1451	{
1452	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1453	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1454	}
1455	else if ((uint32_t)-offDisp < (unsigned)_4K)
1456	{
1457	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1458	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1459	}
1460	else
1461	{
1462	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1463	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1464	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1465	if (offDisp >= 0)
1466	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1467	else
1468	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1469	}
1470
1471	#else
1472	# error "port me"
1473	#endif
1474
1475	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1476	return off;
1477	}
1478
1479
1480	/**
1481	* Emits a 64-bit GPR store with an BP relative destination address.
1482	*
1483	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1484	*/
1485	DECL_INLINE_THROW(uint32_t)
1486	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1487	{
1488	#ifdef RT_ARCH_AMD64
1489	/* mov qword [rbp + offDisp], gprdst */
1490	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1491	if (iGprSrc < 8)
1492	pbCodeBuf[off++] = X86_OP_REX_W;
1493	else
1494	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1495	pbCodeBuf[off++] = 0x89;
1496	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1497
1498	#elif defined(RT_ARCH_ARM64)
1499	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1500	{
1501	/* str w/ unsigned imm12 (scaled) */
1502	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1503	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1504	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1505	}
1506	else if (offDisp >= -256 && offDisp <= 256)
1507	{
1508	/* stur w/ signed imm9 (unscaled) */
1509	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1510	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1511	}
1512	else if ((uint32_t)-offDisp < (unsigned)_4K)
1513	{
1514	/* Use temporary indexing register w/ sub uimm12. */
1515	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1516	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1517	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1518	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1519	}
1520	else
1521	{
1522	/* Use temporary indexing register. */
1523	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1524	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1525	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1526	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1527	}
1528	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1529	return off;
1530
1531	#else
1532	# error "Port me!"
1533	#endif
1534	}
1535
1536
1537	/**
1538	* Emits a 64-bit immediate store with an BP relative destination address.
1539	*
1540	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1541	*/
1542	DECL_INLINE_THROW(uint32_t)
1543	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1544	{
1545	#ifdef RT_ARCH_AMD64
1546	if ((int64_t)uImm64 == (int32_t)uImm64)
1547	{
1548	/* mov qword [rbp + offDisp], imm32 - sign extended */
1549	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1550	pbCodeBuf[off++] = X86_OP_REX_W;
1551	pbCodeBuf[off++] = 0xc7;
1552	if (offDisp < 128 && offDisp >= -128)
1553	{
1554	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1555	pbCodeBuf[off++] = (uint8_t)offDisp;
1556	}
1557	else
1558	{
1559	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1560	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1561	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1562	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1563	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1564	}
1565	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1566	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1567	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1568	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1569	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1570	return off;
1571	}
1572	#endif
1573
1574	/* Load tmp0, imm64; Store tmp to bp+disp. */
1575	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1576	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1577	}
1578
1579
1580	#if defined(RT_ARCH_ARM64)
1581	/**
1582	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1583	*/
1584	DECL_FORCE_INLINE_THROW(uint32_t)
1585	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1586	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1587	{
1588	/*
1589	* There are a couple of ldr variants that takes an immediate offset, so
1590	* try use those if we can, otherwise we have to use the temporary register
1591	* help with the addressing.
1592	*/
1593	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1594	{
1595	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1596	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1597	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1598	}
1599	else
1600	{
1601	/* The offset is too large, so we must load it into a register and use
1602	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1603	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1604	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1605
1606	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1607	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1608
1609	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1610	}
1611	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1612	return off;
1613	}
1614	#endif
1615
1616
1617	/**
1618	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1619	*/
1620	DECL_INLINE_THROW(uint32_t)
1621	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1622	{
1623	#ifdef RT_ARCH_AMD64
1624	/* mov reg64, mem64 */
1625	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1626	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1627	pbCodeBuf[off++] = 0x8b;
1628	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1629	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1630
1631	#elif defined(RT_ARCH_ARM64)
1632	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1633
1634	#else
1635	# error "port me"
1636	#endif
1637	return off;
1638	}
1639
1640
1641	/**
1642	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1643	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1644	*/
1645	DECL_INLINE_THROW(uint32_t)
1646	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1647	{
1648	#ifdef RT_ARCH_AMD64
1649	/* mov reg32, mem32 */
1650	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1651	if (iGprDst >= 8 \|\| iGprBase >= 8)
1652	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1653	pbCodeBuf[off++] = 0x8b;
1654	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1655	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1656
1657	#elif defined(RT_ARCH_ARM64)
1658	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1659
1660	#else
1661	# error "port me"
1662	#endif
1663	return off;
1664	}
1665
1666
1667	/*********************************************************************************************************************************
1668	* Subtraction and Additions *
1669	*********************************************************************************************************************************/
1670
1671	/**
1672	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
1673	* @note The AMD64 version sets flags.
1674	*/
1675	DECL_INLINE_THROW(uint32_t)
1676	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1677	{
1678	#if defined(RT_ARCH_AMD64)
1679	/* sub Gv,Ev */
1680	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1681	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1682	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1683	pbCodeBuf[off++] = 0x2b;
1684	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1685
1686	#elif defined(RT_ARCH_ARM64)
1687	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1688	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
1689
1690	#else
1691	# error "Port me"
1692	#endif
1693	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1694	return off;
1695	}
1696
1697
1698	#ifdef RT_ARCH_AMD64
1699	/**
1700	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1701	*/
1702	DECL_INLINE_THROW(uint32_t)
1703	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1704	{
1705	/* sub gprdst, imm8/imm32 */
1706	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1707	if (iGprDst < 8)
1708	pbCodeBuf[off++] = X86_OP_REX_W;
1709	else
1710	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1711	if (iSubtrahend < 128 && iSubtrahend >= -128)
1712	{
1713	pbCodeBuf[off++] = 0x83;
1714	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1715	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1716	}
1717	else
1718	{
1719	pbCodeBuf[off++] = 0x81;
1720	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1721	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1722	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1723	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1724	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1725	}
1726	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1727	return off;
1728	}
1729	#endif
1730
1731
1732	/**
1733	* Emits adding a 64-bit GPR to another, storing the result in the first.
1734	* @note The AMD64 version sets flags.
1735	*/
1736	DECL_INLINE_THROW(uint32_t)
1737	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1738	{
1739	#if defined(RT_ARCH_AMD64)
1740	/* add Gv,Ev */
1741	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1742	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1743	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1744	pbCodeBuf[off++] = 0x03;
1745	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1746
1747	#elif defined(RT_ARCH_ARM64)
1748	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1749	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1750
1751	#else
1752	# error "Port me"
1753	#endif
1754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1755	return off;
1756	}
1757
1758
1759	/**
1760	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1761	*/
1762	DECL_INLINE_THROW(uint32_t)
1763	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1764	{
1765	#if defined(RT_ARCH_AMD64)
1766	/* add or inc */
1767	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1768	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1769	if (iImm8 != 1)
1770	{
1771	pbCodeBuf[off++] = 0x83;
1772	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1773	pbCodeBuf[off++] = (uint8_t)iImm8;
1774	}
1775	else
1776	{
1777	pbCodeBuf[off++] = 0xff;
1778	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1779	}
1780
1781	#elif defined(RT_ARCH_ARM64)
1782	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1783	if (iImm8 >= 0)
1784	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1785	else
1786	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1787
1788	#else
1789	# error "Port me"
1790	#endif
1791	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1792	return off;
1793	}
1794
1795
1796	/**
1797	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1798	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1799	*/
1800	DECL_INLINE_THROW(uint32_t)
1801	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1802	{
1803	#if defined(RT_ARCH_AMD64)
1804	/* add or inc */
1805	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1806	if (iGprDst >= 8)
1807	pbCodeBuf[off++] = X86_OP_REX_B;
1808	if (iImm8 != 1)
1809	{
1810	pbCodeBuf[off++] = 0x83;
1811	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1812	pbCodeBuf[off++] = (uint8_t)iImm8;
1813	}
1814	else
1815	{
1816	pbCodeBuf[off++] = 0xff;
1817	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1818	}
1819
1820	#elif defined(RT_ARCH_ARM64)
1821	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1822	if (iImm8 >= 0)
1823	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1824	else
1825	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1826
1827	#else
1828	# error "Port me"
1829	#endif
1830	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1831	return off;
1832	}
1833
1834
1835	/**
1836	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1837	*/
1838	DECL_INLINE_THROW(uint32_t)
1839	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1840	{
1841	#if defined(RT_ARCH_AMD64)
1842	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1843	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1844
1845	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1846	{
1847	/* add grp, imm32 */
1848	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1849	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1850	pbCodeBuf[off++] = 0x81;
1851	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1852	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1853	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1854	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1855	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1856	}
1857	else
1858	{
1859	/* Best to use a temporary register to deal with this in the simplest way: */
1860	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1861
1862	/* add dst, tmpreg */
1863	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1864	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1865	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1866	pbCodeBuf[off++] = 0x03;
1867	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1868
1869	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1870	}
1871
1872	#elif defined(RT_ARCH_ARM64)
1873	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1874	{
1875	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1876	if (iAddend >= 0)
1877	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1878	else
1879	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1880	}
1881	else
1882	{
1883	/* Use temporary register for the immediate. */
1884	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1885
1886	/* add gprdst, gprdst, tmpreg */
1887	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1888	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1889
1890	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1891	}
1892
1893	#else
1894	# error "Port me"
1895	#endif
1896	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1897	return off;
1898	}
1899
1900
1901	/**
1902	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1903	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1904	*/
1905	DECL_INLINE_THROW(uint32_t)
1906	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1907	{
1908	#if defined(RT_ARCH_AMD64)
1909	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1910	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1911
1912	/* add grp, imm32 */
1913	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1914	if (iGprDst >= 8)
1915	pbCodeBuf[off++] = X86_OP_REX_B;
1916	pbCodeBuf[off++] = 0x81;
1917	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1918	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1919	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1920	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1921	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1922
1923	#elif defined(RT_ARCH_ARM64)
1924	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1925	{
1926	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1927	if (iAddend >= 0)
1928	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1929	else
1930	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1931	}
1932	else
1933	{
1934	/* Use temporary register for the immediate. */
1935	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1936
1937	/* add gprdst, gprdst, tmpreg */
1938	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1939	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1940
1941	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1942	}
1943
1944	#else
1945	# error "Port me"
1946	#endif
1947	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1948	return off;
1949	}
1950
1951
1952	/*********************************************************************************************************************************
1953	* Unary Operations *
1954	*********************************************************************************************************************************/
1955
1956	/**
1957	* Emits code for clearing bits 16 thru 63 in the GPR.
1958	*/
1959	DECL_INLINE_THROW(uint32_t)
1960	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1961	{
1962	#if defined(RT_ARCH_AMD64)
1963	/* neg Ev */
1964	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1965	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1966	pbCodeBuf[off++] = 0xf7;
1967	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
1968
1969	#elif defined(RT_ARCH_ARM64)
1970	/* sub dst, xzr, dst */
1971	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1972	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
1973
1974	#else
1975	# error "Port me"
1976	#endif
1977	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1978	return off;
1979	}
1980
1981
1982
1983	/*********************************************************************************************************************************
1984	* Bit Operations *
1985	*********************************************************************************************************************************/
1986
1987	/**
1988	* Emits code for clearing bits 16 thru 63 in the GPR.
1989	*/
1990	DECL_INLINE_THROW(uint32_t)
1991	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1992	{
1993	#if defined(RT_ARCH_AMD64)
1994	/* movzx Gv,Ew */
1995	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1996	if (iGprDst >= 8)
1997	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1998	pbCodeBuf[off++] = 0x0f;
1999	pbCodeBuf[off++] = 0xb7;
2000	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2001
2002	#elif defined(RT_ARCH_ARM64)
2003	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2004	# if 1
2005	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2006	# else
2007	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2008	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2009	# endif
2010	#else
2011	# error "Port me"
2012	#endif
2013	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2014	return off;
2015	}
2016
2017
2018	/**
2019	* Emits code for AND'ing two 64-bit GPRs.
2020	*
2021	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2022	* and ARM64 hosts.
2023	*/
2024	DECL_INLINE_THROW(uint32_t)
2025	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2026	{
2027	#if defined(RT_ARCH_AMD64)
2028	/* and Gv, Ev */
2029	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2030	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2031	pbCodeBuf[off++] = 0x23;
2032	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2033	RT_NOREF(fSetFlags);
2034
2035	#elif defined(RT_ARCH_ARM64)
2036	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2037	if (!fSetFlags)
2038	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2039	else
2040	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2041
2042	#else
2043	# error "Port me"
2044	#endif
2045	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2046	return off;
2047	}
2048
2049
2050	/**
2051	* Emits code for AND'ing two 32-bit GPRs.
2052	*/
2053	DECL_INLINE_THROW(uint32_t)
2054	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2055	{
2056	#if defined(RT_ARCH_AMD64)
2057	/* and Gv, Ev */
2058	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2059	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2060	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2061	pbCodeBuf[off++] = 0x23;
2062	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2063	RT_NOREF(fSetFlags);
2064
2065	#elif defined(RT_ARCH_ARM64)
2066	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2067	if (!fSetFlags)
2068	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2069	else
2070	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2071
2072	#else
2073	# error "Port me"
2074	#endif
2075	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2076	return off;
2077	}
2078
2079
2080	/**
2081	* Emits code for AND'ing a 64-bit GPRs with a constant.
2082	*
2083	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2084	* and ARM64 hosts.
2085	*/
2086	DECL_INLINE_THROW(uint32_t)
2087	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2088	{
2089	#if defined(RT_ARCH_AMD64)
2090	if ((int64_t)uImm == (int8_t)uImm)
2091	{
2092	/* and Ev, imm8 */
2093	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2094	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2095	pbCodeBuf[off++] = 0x83;
2096	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2097	pbCodeBuf[off++] = (uint8_t)uImm;
2098	}
2099	else if ((int64_t)uImm == (int32_t)uImm)
2100	{
2101	/* and Ev, imm32 */
2102	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2103	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2104	pbCodeBuf[off++] = 0x81;
2105	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2106	pbCodeBuf[off++] = RT_BYTE1(uImm);
2107	pbCodeBuf[off++] = RT_BYTE2(uImm);
2108	pbCodeBuf[off++] = RT_BYTE3(uImm);
2109	pbCodeBuf[off++] = RT_BYTE4(uImm);
2110	}
2111	else
2112	{
2113	/* Use temporary register for the 64-bit immediate. */
2114	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2115	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2116	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2117	}
2118	RT_NOREF(fSetFlags);
2119
2120	#elif defined(RT_ARCH_ARM64)
2121	uint32_t uImmR = 0;
2122	uint32_t uImmNandS = 0;
2123	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
2124	{
2125	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2126	if (!fSetFlags)
2127	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2128	else
2129	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2130	}
2131	else
2132	{
2133	/* Use temporary register for the 64-bit immediate. */
2134	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2135	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2136	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2137	}
2138
2139	#else
2140	# error "Port me"
2141	#endif
2142	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2143	return off;
2144	}
2145
2146
2147	/**
2148	* Emits code for AND'ing an 32-bit GPRs with a constant.
2149	*/
2150	DECL_INLINE_THROW(uint32_t)
2151	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2152	{
2153	#if defined(RT_ARCH_AMD64)
2154	/* and Ev, imm */
2155	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2156	if (iGprDst >= 8)
2157	pbCodeBuf[off++] = X86_OP_REX_B;
2158	if ((int32_t)uImm == (int8_t)uImm)
2159	{
2160	pbCodeBuf[off++] = 0x83;
2161	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2162	pbCodeBuf[off++] = (uint8_t)uImm;
2163	}
2164	else
2165	{
2166	pbCodeBuf[off++] = 0x81;
2167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2168	pbCodeBuf[off++] = RT_BYTE1(uImm);
2169	pbCodeBuf[off++] = RT_BYTE2(uImm);
2170	pbCodeBuf[off++] = RT_BYTE3(uImm);
2171	pbCodeBuf[off++] = RT_BYTE4(uImm);
2172	}
2173	RT_NOREF(fSetFlags);
2174
2175	#elif defined(RT_ARCH_ARM64)
2176	uint32_t uImmR = 0;
2177	uint32_t uImmNandS = 0;
2178	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2179	{
2180	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2181	if (!fSetFlags)
2182	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2183	else
2184	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2185	}
2186	else
2187	{
2188	/* Use temporary register for the 64-bit immediate. */
2189	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2190	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2191	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2192	}
2193
2194	#else
2195	# error "Port me"
2196	#endif
2197	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2198	return off;
2199	}
2200
2201
2202	/**
2203	* Emits code for XOR'ing two 64-bit GPRs.
2204	*/
2205	DECL_INLINE_THROW(uint32_t)
2206	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2207	{
2208	#if defined(RT_ARCH_AMD64)
2209	/* and Gv, Ev */
2210	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2211	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2212	pbCodeBuf[off++] = 0x33;
2213	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2214
2215	#elif defined(RT_ARCH_ARM64)
2216	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2217	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2218
2219	#else
2220	# error "Port me"
2221	#endif
2222	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2223	return off;
2224	}
2225
2226
2227	/**
2228	* Emits code for XOR'ing two 32-bit GPRs.
2229	*/
2230	DECL_INLINE_THROW(uint32_t)
2231	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2232	{
2233	#if defined(RT_ARCH_AMD64)
2234	/* and Gv, Ev */
2235	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2236	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2237	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2238	pbCodeBuf[off++] = 0x33;
2239	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2240
2241	#elif defined(RT_ARCH_ARM64)
2242	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2243	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2244
2245	#else
2246	# error "Port me"
2247	#endif
2248	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2249	return off;
2250	}
2251
2252
2253	/*********************************************************************************************************************************
2254	* Shifting *
2255	*********************************************************************************************************************************/
2256
2257	/**
2258	* Emits code for shifting a GPR a fixed number of bits to the left.
2259	*/
2260	DECL_INLINE_THROW(uint32_t)
2261	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2262	{
2263	Assert(cShift > 0 && cShift < 64);
2264
2265	#if defined(RT_ARCH_AMD64)
2266	/* shl dst, cShift */
2267	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2268	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2269	if (cShift != 1)
2270	{
2271	pbCodeBuf[off++] = 0xc1;
2272	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2273	pbCodeBuf[off++] = cShift;
2274	}
2275	else
2276	{
2277	pbCodeBuf[off++] = 0xd1;
2278	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2279	}
2280
2281	#elif defined(RT_ARCH_ARM64)
2282	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2283	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2284
2285	#else
2286	# error "Port me"
2287	#endif
2288	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2289	return off;
2290	}
2291
2292
2293	/**
2294	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2295	*/
2296	DECL_INLINE_THROW(uint32_t)
2297	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2298	{
2299	Assert(cShift > 0 && cShift < 32);
2300
2301	#if defined(RT_ARCH_AMD64)
2302	/* shl dst, cShift */
2303	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2304	if (iGprDst >= 8)
2305	pbCodeBuf[off++] = X86_OP_REX_B;
2306	if (cShift != 1)
2307	{
2308	pbCodeBuf[off++] = 0xc1;
2309	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2310	pbCodeBuf[off++] = cShift;
2311	}
2312	else
2313	{
2314	pbCodeBuf[off++] = 0xd1;
2315	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2316	}
2317
2318	#elif defined(RT_ARCH_ARM64)
2319	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2320	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2321
2322	#else
2323	# error "Port me"
2324	#endif
2325	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2326	return off;
2327	}
2328
2329
2330	/**
2331	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2332	*/
2333	DECL_INLINE_THROW(uint32_t)
2334	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2335	{
2336	Assert(cShift > 0 && cShift < 64);
2337
2338	#if defined(RT_ARCH_AMD64)
2339	/* shr dst, cShift */
2340	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2341	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2342	if (cShift != 1)
2343	{
2344	pbCodeBuf[off++] = 0xc1;
2345	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2346	pbCodeBuf[off++] = cShift;
2347	}
2348	else
2349	{
2350	pbCodeBuf[off++] = 0xd1;
2351	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2352	}
2353
2354	#elif defined(RT_ARCH_ARM64)
2355	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2356	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2357
2358	#else
2359	# error "Port me"
2360	#endif
2361	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2362	return off;
2363	}
2364
2365
2366	/**
2367	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2368	* right.
2369	*/
2370	DECL_INLINE_THROW(uint32_t)
2371	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2372	{
2373	Assert(cShift > 0 && cShift < 32);
2374
2375	#if defined(RT_ARCH_AMD64)
2376	/* shr dst, cShift */
2377	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2378	if (iGprDst >= 8)
2379	pbCodeBuf[off++] = X86_OP_REX_B;
2380	if (cShift != 1)
2381	{
2382	pbCodeBuf[off++] = 0xc1;
2383	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2384	pbCodeBuf[off++] = cShift;
2385	}
2386	else
2387	{
2388	pbCodeBuf[off++] = 0xd1;
2389	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2390	}
2391
2392	#elif defined(RT_ARCH_ARM64)
2393	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2394	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2395
2396	#else
2397	# error "Port me"
2398	#endif
2399	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2400	return off;
2401	}
2402
2403
2404
2405	/*********************************************************************************************************************************
2406	* Compare and Testing *
2407	*********************************************************************************************************************************/
2408
2409
2410	#ifdef RT_ARCH_ARM64
2411	/**
2412	* Emits an ARM64 compare instruction.
2413	*/
2414	DECL_INLINE_THROW(uint32_t)
2415	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2416	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2417	{
2418	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2419	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2420	f64Bit, true /fSetFlags/, cShift, enmShift);
2421	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2422	return off;
2423	}
2424	#endif
2425
2426
2427	/**
2428	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2429	* with conditional instruction.
2430	*/
2431	DECL_INLINE_THROW(uint32_t)
2432	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2433	{
2434	#ifdef RT_ARCH_AMD64
2435	/* cmp Gv, Ev */
2436	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2437	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2438	pbCodeBuf[off++] = 0x3b;
2439	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2441
2442	#elif defined(RT_ARCH_ARM64)
2443	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2444
2445	#else
2446	# error "Port me!"
2447	#endif
2448	return off;
2449	}
2450
2451
2452	/**
2453	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2454	* with conditional instruction.
2455	*/
2456	DECL_INLINE_THROW(uint32_t)
2457	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2458	{
2459	#ifdef RT_ARCH_AMD64
2460	/* cmp Gv, Ev */
2461	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2462	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2463	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2464	pbCodeBuf[off++] = 0x3b;
2465	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2466	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2467
2468	#elif defined(RT_ARCH_ARM64)
2469	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2470
2471	#else
2472	# error "Port me!"
2473	#endif
2474	return off;
2475	}
2476
2477
2478	/**
2479	* Emits a compare of a 64-bit GPR with a constant value, settings status
2480	* flags/whatever for use with conditional instruction.
2481	*/
2482	DECL_INLINE_THROW(uint32_t)
2483	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2484	{
2485	#ifdef RT_ARCH_AMD64
2486	if (uImm <= UINT32_C(0xff))
2487	{
2488	/* cmp Ev, Ib */
2489	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2490	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2491	pbCodeBuf[off++] = 0x83;
2492	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2493	pbCodeBuf[off++] = (uint8_t)uImm;
2494	}
2495	else if ((int64_t)uImm == (int32_t)uImm)
2496	{
2497	/* cmp Ev, imm */
2498	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2499	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2500	pbCodeBuf[off++] = 0x81;
2501	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2502	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2503	pbCodeBuf[off++] = RT_BYTE1(uImm);
2504	pbCodeBuf[off++] = RT_BYTE2(uImm);
2505	pbCodeBuf[off++] = RT_BYTE3(uImm);
2506	pbCodeBuf[off++] = RT_BYTE4(uImm);
2507	}
2508	else
2509	{
2510	/* Use temporary register for the immediate. */
2511	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2512	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2513	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2514	}
2515
2516	#elif defined(RT_ARCH_ARM64)
2517	/** @todo guess there are clevere things we can do here... */
2518	if (uImm < _4K)
2519	{
2520	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2521	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2522	true /64Bit/, true /fSetFlags/);
2523	}
2524	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2525	{
2526	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2527	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
2528	true /64Bit/, true /fSetFlags/, true /fShift12/);
2529	}
2530	else
2531	{
2532	/* Use temporary register for the immediate. */
2533	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2534	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2535	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2536	}
2537
2538	#else
2539	# error "Port me!"
2540	#endif
2541
2542	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2543	return off;
2544	}
2545
2546
2547	/**
2548	* Emits a compare of a 32-bit GPR with a constant value, settings status
2549	* flags/whatever for use with conditional instruction.
2550	*/
2551	DECL_INLINE_THROW(uint32_t)
2552	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2553	{
2554	#ifdef RT_ARCH_AMD64
2555	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2556	if (iGprLeft >= 8)
2557	pbCodeBuf[off++] = X86_OP_REX_B;
2558	if (uImm <= UINT32_C(0xff))
2559	{
2560	/* cmp Ev, Ib */
2561	pbCodeBuf[off++] = 0x83;
2562	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2563	pbCodeBuf[off++] = (uint8_t)uImm;
2564	}
2565	else
2566	{
2567	/* cmp Ev, imm */
2568	pbCodeBuf[off++] = 0x81;
2569	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2570	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2571	pbCodeBuf[off++] = RT_BYTE1(uImm);
2572	pbCodeBuf[off++] = RT_BYTE2(uImm);
2573	pbCodeBuf[off++] = RT_BYTE3(uImm);
2574	pbCodeBuf[off++] = RT_BYTE4(uImm);
2575	}
2576
2577	#elif defined(RT_ARCH_ARM64)
2578	/** @todo guess there are clevere things we can do here... */
2579	if (uImm < _4K)
2580	{
2581	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2582	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2583	false /64Bit/, true /fSetFlags/);
2584	}
2585	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2586	{
2587	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2588	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2589	false /64Bit/, true /fSetFlags/, true /fShift12/);
2590	}
2591	else
2592	{
2593	/* Use temporary register for the immediate. */
2594	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2595	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2596	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2597	}
2598
2599	#else
2600	# error "Port me!"
2601	#endif
2602
2603	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2604	return off;
2605	}
2606
2607
2608
2609	/*********************************************************************************************************************************
2610	* Branching *
2611	*********************************************************************************************************************************/
2612
2613	/**
2614	* Emits a JMP rel32 / B imm19 to the given label.
2615	*/
2616	DECL_INLINE_THROW(uint32_t)
2617	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2618	{
2619	Assert(idxLabel < pReNative->cLabels);
2620
2621	#ifdef RT_ARCH_AMD64
2622	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2623	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2624	{
2625	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2626	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2627	{
2628	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2629	pbCodeBuf[off++] = (uint8_t)offRel;
2630	}
2631	else
2632	{
2633	offRel -= 3;
2634	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2635	pbCodeBuf[off++] = RT_BYTE1(offRel);
2636	pbCodeBuf[off++] = RT_BYTE2(offRel);
2637	pbCodeBuf[off++] = RT_BYTE3(offRel);
2638	pbCodeBuf[off++] = RT_BYTE4(offRel);
2639	}
2640	}
2641	else
2642	{
2643	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2644	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2645	pbCodeBuf[off++] = 0xfe;
2646	pbCodeBuf[off++] = 0xff;
2647	pbCodeBuf[off++] = 0xff;
2648	pbCodeBuf[off++] = 0xff;
2649	}
2650	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2651
2652	#elif defined(RT_ARCH_ARM64)
2653	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2654	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2655	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2656	else
2657	{
2658	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2659	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2660	}
2661
2662	#else
2663	# error "Port me!"
2664	#endif
2665	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2666	return off;
2667	}
2668
2669
2670	/**
2671	* Emits a JMP rel32 / B imm19 to a new undefined label.
2672	*/
2673	DECL_INLINE_THROW(uint32_t)
2674	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2675	{
2676	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2677	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2678	}
2679
2680	/** Condition type. */
2681	#ifdef RT_ARCH_AMD64
2682	typedef enum IEMNATIVEINSTRCOND : uint8_t
2683	{
2684	kIemNativeInstrCond_o = 0,
2685	kIemNativeInstrCond_no,
2686	kIemNativeInstrCond_c,
2687	kIemNativeInstrCond_nc,
2688	kIemNativeInstrCond_e,
2689	kIemNativeInstrCond_ne,
2690	kIemNativeInstrCond_be,
2691	kIemNativeInstrCond_nbe,
2692	kIemNativeInstrCond_s,
2693	kIemNativeInstrCond_ns,
2694	kIemNativeInstrCond_p,
2695	kIemNativeInstrCond_np,
2696	kIemNativeInstrCond_l,
2697	kIemNativeInstrCond_nl,
2698	kIemNativeInstrCond_le,
2699	kIemNativeInstrCond_nle
2700	} IEMNATIVEINSTRCOND;
2701	#elif defined(RT_ARCH_ARM64)
2702	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2703	#else
2704	# error "Port me!"
2705	#endif
2706
2707
2708	/**
2709	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2710	*/
2711	DECL_INLINE_THROW(uint32_t)
2712	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2713	{
2714	Assert(idxLabel < pReNative->cLabels);
2715
2716	#ifdef RT_ARCH_AMD64
2717	/* jcc rel32 */
2718	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2719	pbCodeBuf[off++] = 0x0f;
2720	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2721	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2722	pbCodeBuf[off++] = 0x00;
2723	pbCodeBuf[off++] = 0x00;
2724	pbCodeBuf[off++] = 0x00;
2725	pbCodeBuf[off++] = 0x00;
2726
2727	#elif defined(RT_ARCH_ARM64)
2728	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2729	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2730	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2731
2732	#else
2733	# error "Port me!"
2734	#endif
2735	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2736	return off;
2737	}
2738
2739
2740	/**
2741	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2742	*/
2743	DECL_INLINE_THROW(uint32_t)
2744	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2745	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2746	{
2747	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2748	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2749	}
2750
2751
2752	/**
2753	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2754	*/
2755	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2756	{
2757	#ifdef RT_ARCH_AMD64
2758	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2759	#elif defined(RT_ARCH_ARM64)
2760	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2761	#else
2762	# error "Port me!"
2763	#endif
2764	}
2765
2766	/**
2767	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2768	*/
2769	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2770	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2771	{
2772	#ifdef RT_ARCH_AMD64
2773	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2774	#elif defined(RT_ARCH_ARM64)
2775	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2776	#else
2777	# error "Port me!"
2778	#endif
2779	}
2780
2781
2782	/**
2783	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2784	*/
2785	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2786	{
2787	#ifdef RT_ARCH_AMD64
2788	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2789	#elif defined(RT_ARCH_ARM64)
2790	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2791	#else
2792	# error "Port me!"
2793	#endif
2794	}
2795
2796	/**
2797	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2798	*/
2799	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2800	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2801	{
2802	#ifdef RT_ARCH_AMD64
2803	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2804	#elif defined(RT_ARCH_ARM64)
2805	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2806	#else
2807	# error "Port me!"
2808	#endif
2809	}
2810
2811
2812	/**
2813	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2814	*/
2815	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2816	{
2817	#ifdef RT_ARCH_AMD64
2818	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2819	#elif defined(RT_ARCH_ARM64)
2820	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2821	#else
2822	# error "Port me!"
2823	#endif
2824	}
2825
2826	/**
2827	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2828	*/
2829	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2830	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2831	{
2832	#ifdef RT_ARCH_AMD64
2833	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2834	#elif defined(RT_ARCH_ARM64)
2835	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2836	#else
2837	# error "Port me!"
2838	#endif
2839	}
2840
2841
2842	/**
2843	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2844	*/
2845	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2846	{
2847	#ifdef RT_ARCH_AMD64
2848	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2849	#elif defined(RT_ARCH_ARM64)
2850	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2851	#else
2852	# error "Port me!"
2853	#endif
2854	}
2855
2856	/**
2857	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2858	*/
2859	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2860	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2861	{
2862	#ifdef RT_ARCH_AMD64
2863	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2864	#elif defined(RT_ARCH_ARM64)
2865	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2866	#else
2867	# error "Port me!"
2868	#endif
2869	}
2870
2871
2872	/**
2873	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
2874	*/
2875	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2876	{
2877	#ifdef RT_ARCH_AMD64
2878	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
2879	#elif defined(RT_ARCH_ARM64)
2880	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
2881	#else
2882	# error "Port me!"
2883	#endif
2884	}
2885
2886	/**
2887	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
2888	*/
2889	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2890	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2891	{
2892	#ifdef RT_ARCH_AMD64
2893	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
2894	#elif defined(RT_ARCH_ARM64)
2895	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
2896	#else
2897	# error "Port me!"
2898	#endif
2899	}
2900
2901
2902	/**
2903	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2904	*
2905	* The @a offTarget is applied x86-style, so zero means the next instruction.
2906	* The unit is IEMNATIVEINSTR.
2907	*/
2908	DECL_INLINE_THROW(uint32_t)
2909	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2910	{
2911	#ifdef RT_ARCH_AMD64
2912	/* jcc rel32 */
2913	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2914	if (offTarget < 128 && offTarget >= -128)
2915	{
2916	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2917	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2918	}
2919	else
2920	{
2921	pbCodeBuf[off++] = 0x0f;
2922	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2923	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2924	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2925	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2926	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2927	}
2928
2929	#elif defined(RT_ARCH_ARM64)
2930	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2931	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget + 1);
2932
2933	#else
2934	# error "Port me!"
2935	#endif
2936	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2937	return off;
2938	}
2939
2940
2941	/**
2942	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2943	*
2944	* The @a offTarget is applied x86-style, so zero means the next instruction.
2945	* The unit is IEMNATIVEINSTR.
2946	*/
2947	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2948	{
2949	#ifdef RT_ARCH_AMD64
2950	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2951	#elif defined(RT_ARCH_ARM64)
2952	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2953	#else
2954	# error "Port me!"
2955	#endif
2956	}
2957
2958
2959	/**
2960	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2961	*
2962	* The @a offTarget is applied x86-style, so zero means the next instruction.
2963	* The unit is IEMNATIVEINSTR.
2964	*/
2965	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2966	{
2967	#ifdef RT_ARCH_AMD64
2968	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2969	#elif defined(RT_ARCH_ARM64)
2970	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2971	#else
2972	# error "Port me!"
2973	#endif
2974	}
2975
2976
2977	/**
2978	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2979	*
2980	* The @a offTarget is applied x86-style, so zero means the next instruction.
2981	* The unit is IEMNATIVEINSTR.
2982	*/
2983	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2984	{
2985	#ifdef RT_ARCH_AMD64
2986	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2987	#elif defined(RT_ARCH_ARM64)
2988	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2989	#else
2990	# error "Port me!"
2991	#endif
2992	}
2993
2994
2995	/**
2996	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
2997	*
2998	* The @a offTarget is applied x86-style, so zero means the next instruction.
2999	* The unit is IEMNATIVEINSTR.
3000	*/
3001	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3002	{
3003	#ifdef RT_ARCH_AMD64
3004	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3005	#elif defined(RT_ARCH_ARM64)
3006	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3007	#else
3008	# error "Port me!"
3009	#endif
3010	}
3011
3012
3013	/**
3014	* Fixes up a conditional jump to a fixed label.
3015	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
3016	*/
3017	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3018	{
3019	# if defined(RT_ARCH_AMD64)
3020	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3021	if (pbCodeBuf[offFixup] != 0x0f)
3022	{
3023	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
3024	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3025	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
3026	}
3027	else
3028	{
3029	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
3030	uint32_t const offRel32 = offTarget - (offFixup + 6);
3031	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
3032	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
3033	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
3034	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
3035	}
3036
3037	# elif defined(RT_ARCH_ARM64)
3038	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3039
3040	int32_t const offDisp = offTarget - offFixup;
3041	Assert(offDisp >= -262144 && offDisp < 262144);
3042	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
3043
3044	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
3045	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
3046
3047	# endif
3048	}
3049
3050
3051	/**
3052	* Internal helper, don't call directly.
3053	*/
3054	DECL_INLINE_THROW(uint32_t)
3055	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3056	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3057	{
3058	Assert(iBitNo < 64);
3059	#ifdef RT_ARCH_AMD64
3060	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3061	if (iBitNo < 8)
3062	{
3063	/* test Eb, imm8 */
3064	if (iGprSrc >= 4)
3065	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3066	pbCodeBuf[off++] = 0xf6;
3067	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3068	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3069	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3070	}
3071	else
3072	{
3073	/* bt Ev, imm8 */
3074	if (iBitNo >= 32)
3075	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3076	else if (iGprSrc >= 8)
3077	pbCodeBuf[off++] = X86_OP_REX_B;
3078	pbCodeBuf[off++] = 0x0f;
3079	pbCodeBuf[off++] = 0xba;
3080	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3081	pbCodeBuf[off++] = iBitNo;
3082	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3083	}
3084
3085	#elif defined(RT_ARCH_ARM64)
3086	/* Use the TBNZ instruction here. */
3087	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3088	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3089	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3090
3091	#else
3092	# error "Port me!"
3093	#endif
3094	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3095	return off;
3096	}
3097
3098
3099	/**
3100	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3101	* @a iGprSrc.
3102	*
3103	* @note On ARM64 the range is only +/-8191 instructions.
3104	*/
3105	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3106	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3107	{
3108	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
3109	}
3110
3111
3112	/**
3113	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3114	* _set_ in @a iGprSrc.
3115	*
3116	* @note On ARM64 the range is only +/-8191 instructions.
3117	*/
3118	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3119	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3120	{
3121	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
3122	}
3123
3124
3125	/**
3126	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3127	* flags accordingly.
3128	*/
3129	DECL_INLINE_THROW(uint32_t)
3130	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3131	{
3132	Assert(fBits != 0);
3133	#ifdef RT_ARCH_AMD64
3134
3135	if (fBits >= UINT32_MAX)
3136	{
3137	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3138
3139	/* test Ev,Gv */
3140	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3141	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3142	pbCodeBuf[off++] = 0x85;
3143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3144
3145	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3146	}
3147	else if (fBits <= UINT32_MAX)
3148	{
3149	/* test Eb, imm8 or test Ev, imm32 */
3150	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3151	if (fBits <= UINT8_MAX)
3152	{
3153	if (iGprSrc >= 4)
3154	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3155	pbCodeBuf[off++] = 0xf6;
3156	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3157	pbCodeBuf[off++] = (uint8_t)fBits;
3158	}
3159	else
3160	{
3161	if (iGprSrc >= 8)
3162	pbCodeBuf[off++] = X86_OP_REX_B;
3163	pbCodeBuf[off++] = 0xf7;
3164	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3165	pbCodeBuf[off++] = RT_BYTE1(fBits);
3166	pbCodeBuf[off++] = RT_BYTE2(fBits);
3167	pbCodeBuf[off++] = RT_BYTE3(fBits);
3168	pbCodeBuf[off++] = RT_BYTE4(fBits);
3169	}
3170	}
3171	/** @todo implement me. */
3172	else
3173	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3174
3175	#elif defined(RT_ARCH_ARM64)
3176
3177	if (false)
3178	{
3179	/** @todo figure out how to work the immr / N:imms constants. */
3180	}
3181	else
3182	{
3183	/* ands Zr, iGprSrc, iTmpReg */
3184	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3185	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3186	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3187	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3188	}
3189
3190	#else
3191	# error "Port me!"
3192	#endif
3193	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3194	return off;
3195	}
3196
3197
3198	/**
3199	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
3200	* @a iGprSrc, setting CPU flags accordingly.
3201	*/
3202	DECL_INLINE_THROW(uint32_t)
3203	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
3204	{
3205	Assert(fBits != 0);
3206
3207	#ifdef RT_ARCH_AMD64
3208	/* test Eb, imm8 */
3209	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3210	if (iGprSrc >= 4)
3211	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3212	pbCodeBuf[off++] = 0xf6;
3213	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3214	pbCodeBuf[off++] = fBits;
3215
3216	#elif defined(RT_ARCH_ARM64)
3217
3218	/* ands xzr, src, [tmp\|#imm] */
3219	uint32_t uImmR = 0;
3220	uint32_t uImmNandS = 0;
3221	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
3222	{
3223	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3224	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3225	}
3226	else
3227	{
3228	/* Use temporary register for the 64-bit immediate. */
3229	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3230	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3231	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
3232	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3233	}
3234
3235	#else
3236	# error "Port me!"
3237	#endif
3238	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3239	return off;
3240	}
3241
3242
3243	/**
3244	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
3245	* are set in @a iGprSrc.
3246	*/
3247	DECL_INLINE_THROW(uint32_t)
3248	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3249	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3250	{
3251	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3252
3253	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3254	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3255
3256	return off;
3257	}
3258
3259
3260	/**
3261	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
3262	* are set in @a iGprSrc.
3263	*/
3264	DECL_INLINE_THROW(uint32_t)
3265	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3266	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3267	{
3268	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3269
3270	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3271	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3272
3273	return off;
3274	}
3275
3276
3277	/**
3278	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
3279	*
3280	* The operand size is given by @a f64Bit.
3281	*/
3282	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3283	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3284	{
3285	Assert(idxLabel < pReNative->cLabels);
3286
3287	#ifdef RT_ARCH_AMD64
3288	/* test reg32,reg32 / test reg64,reg64 */
3289	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3290	if (f64Bit)
3291	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
3292	else if (iGprSrc >= 8)
3293	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
3294	pbCodeBuf[off++] = 0x85;
3295	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3296	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3297
3298	/* jz idxLabel */
3299	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3300
3301	#elif defined(RT_ARCH_ARM64)
3302	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3303	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3304	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
3305	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3306
3307	#else
3308	# error "Port me!"
3309	#endif
3310	return off;
3311	}
3312
3313
3314	/**
3315	* Emits code that jumps to a new label if @a iGprSrc is zero.
3316	*
3317	* The operand size is given by @a f64Bit.
3318	*/
3319	DECL_INLINE_THROW(uint32_t)
3320	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3321	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3322	{
3323	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3324	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3325	}
3326
3327
3328	/**
3329	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3330	* differs.
3331	*/
3332	DECL_INLINE_THROW(uint32_t)
3333	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3334	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3335	{
3336	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3337	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3338	return off;
3339	}
3340
3341
3342	/**
3343	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3344	*/
3345	DECL_INLINE_THROW(uint32_t)
3346	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3347	uint8_t iGprLeft, uint8_t iGprRight,
3348	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3349	{
3350	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3351	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3352	}
3353
3354
3355	/**
3356	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3357	*/
3358	DECL_INLINE_THROW(uint32_t)
3359	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3360	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3361	{
3362	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3363	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3364	return off;
3365	}
3366
3367
3368	/**
3369	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3370	*/
3371	DECL_INLINE_THROW(uint32_t)
3372	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3373	uint8_t iGprSrc, uint64_t uImm,
3374	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3375	{
3376	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3377	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3378	}
3379
3380
3381	/**
3382	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3383	* @a uImm.
3384	*/
3385	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3386	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3387	{
3388	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3389	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3390	return off;
3391	}
3392
3393
3394	/**
3395	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3396	* @a uImm.
3397	*/
3398	DECL_INLINE_THROW(uint32_t)
3399	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3400	uint8_t iGprSrc, uint32_t uImm,
3401	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3402	{
3403	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3404	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3405	}
3406
3407
3408	/*********************************************************************************************************************************
3409	* Calls. *
3410	*********************************************************************************************************************************/
3411
3412	/**
3413	* Emits a call to a 64-bit address.
3414	*/
3415	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3416	{
3417	#ifdef RT_ARCH_AMD64
3418	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3419
3420	/* call rax */
3421	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3422	pbCodeBuf[off++] = 0xff;
3423	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3424
3425	#elif defined(RT_ARCH_ARM64)
3426	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3427
3428	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3429	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3430
3431	#else
3432	# error "port me"
3433	#endif
3434	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3435	return off;
3436	}
3437
3438
3439	/**
3440	* Emits code to load a stack variable into an argument GPR.
3441	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3442	*/
3443	DECL_FORCE_INLINE_THROW(uint32_t)
3444	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3445	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
3446	{
3447	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3448	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3449	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3450
3451	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3452	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3453	{
3454	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK) \|\| fVarAllowInVolatileReg);
3455	RT_NOREF(fVarAllowInVolatileReg);
3456	if (!offAddend)
3457	{
3458	if (idxRegArg != idxRegVar)
3459	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
3460	}
3461	else
3462	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
3463	}
3464	else
3465	{
3466	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
3467	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
3468	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
3469	if (offAddend)
3470	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
3471	}
3472	return off;
3473	}
3474
3475
3476	/**
3477	* Emits code to load a stack or immediate variable value into an argument GPR,
3478	* optional with a addend.
3479	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3480	*/
3481	DECL_FORCE_INLINE_THROW(uint32_t)
3482	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3483	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
3484	{
3485	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3486	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
3487	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
3488	else
3489	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend, fVarAllowInVolatileReg);
3490	return off;
3491	}
3492
3493
3494	/**
3495	* Emits code to load the variable address into an argument GRP.
3496	*
3497	* This only works for uninitialized and stack variables.
3498	*/
3499	DECL_FORCE_INLINE_THROW(uint32_t)
3500	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3501	bool fFlushShadows)
3502	{
3503	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3504	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
3505	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3506	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3507
3508	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
3509	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
3510
3511	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3512	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3513	{
3514	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
3515	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
3516	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3517	}
3518	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
3519	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3520
3521	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
3522	}
3523
3524
3525	/** @} */
3526
3527	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
3528

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102687

Download in other formats: