IEMN8veRecompilerEmit.h@ 102700

Last change on this file since 102700 was 102700, checked in by vboxsync, 16 months ago
VMM/IEM: Native translation of BODY_LOAD_TLB_AFTER_BRANCH. (only tested on amd64) [build fix] bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 126.1 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102700 2023-12-25 22:29:42Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
88	pbCodeBuf[off++] = 0xcc;
89	RT_NOREF(uInfo);
90
91	#elif defined(RT_ARCH_ARM64)
92	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
93	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
94
95	#else
96	# error "error"
97	#endif
98	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
99	return off;
100	}
101
102
103	/*********************************************************************************************************************************
104	* Loads, Stores and Related Stuff. *
105	*********************************************************************************************************************************/
106
107	#ifdef RT_ARCH_AMD64
108	/**
109	* Common bit of iemNativeEmitLoadGprByGpr and friends.
110	*/
111	DECL_FORCE_INLINE(uint32_t)
112	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
113	{
114	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
115	{
116	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
117	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
118	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
119	}
120	else if (offDisp == (int8_t)offDisp)
121	{
122	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
123	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
124	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
125	pbCodeBuf[off++] = (uint8_t)offDisp;
126	}
127	else
128	{
129	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, iGprBase & 7);
130	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
131	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
132	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
133	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
134	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
135	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
136	}
137	return off;
138	}
139	#endif /* RT_ARCH_AMD64 */
140
141	/**
142	* Emits setting a GPR to zero.
143	*/
144	DECL_INLINE_THROW(uint32_t)
145	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
146	{
147	#ifdef RT_ARCH_AMD64
148	/* xor gpr32, gpr32 */
149	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
150	if (iGpr >= 8)
151	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
152	pbCodeBuf[off++] = 0x33;
153	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
154
155	#elif defined(RT_ARCH_ARM64)
156	/* mov gpr, #0x0 */
157	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
158	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
159
160	#else
161	# error "port me"
162	#endif
163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
164	return off;
165	}
166
167
168	/**
169	* Variant of iemNativeEmitLoadGprImm64 where the caller ensures sufficent
170	* buffer space.
171	*
172	* Max buffer consumption:
173	* - AMD64: 10 instruction bytes.
174	* - ARM64: 4 instruction words (16 bytes).
175	*/
176	DECLINLINE(uint32_t) iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
177	{
178	#ifdef RT_ARCH_AMD64
179	if (uImm64 == 0)
180	{
181	/* xor gpr, gpr */
182	if (iGpr >= 8)
183	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
184	pCodeBuf[off++] = 0x33;
185	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
186	}
187	else if (uImm64 <= UINT32_MAX)
188	{
189	/* mov gpr, imm32 */
190	if (iGpr >= 8)
191	pCodeBuf[off++] = X86_OP_REX_B;
192	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
193	pCodeBuf[off++] = RT_BYTE1(uImm64);
194	pCodeBuf[off++] = RT_BYTE2(uImm64);
195	pCodeBuf[off++] = RT_BYTE3(uImm64);
196	pCodeBuf[off++] = RT_BYTE4(uImm64);
197	}
198	else
199	{
200	/* mov gpr, imm64 */
201	if (iGpr < 8)
202	pCodeBuf[off++] = X86_OP_REX_W;
203	else
204	pCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
205	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
206	pCodeBuf[off++] = RT_BYTE1(uImm64);
207	pCodeBuf[off++] = RT_BYTE2(uImm64);
208	pCodeBuf[off++] = RT_BYTE3(uImm64);
209	pCodeBuf[off++] = RT_BYTE4(uImm64);
210	pCodeBuf[off++] = RT_BYTE5(uImm64);
211	pCodeBuf[off++] = RT_BYTE6(uImm64);
212	pCodeBuf[off++] = RT_BYTE7(uImm64);
213	pCodeBuf[off++] = RT_BYTE8(uImm64);
214	}
215
216	#elif defined(RT_ARCH_ARM64)
217	/*
218	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
219	* supply remaining bits using 'movk grp, imm16, lsl #x'.
220	*
221	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
222	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
223	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
224	* after the first non-zero immediate component so we switch to movk for
225	* the remainder.
226	*/
227	unsigned cZeroHalfWords = !( uImm64 & UINT16_MAX)
228	+ !((uImm64 >> 16) & UINT16_MAX)
229	+ !((uImm64 >> 32) & UINT16_MAX)
230	+ !((uImm64 >> 48) & UINT16_MAX);
231	unsigned cFfffHalfWords = cZeroHalfWords >= 2 ? 0 /* skip */
232	: ( (uImm64 & UINT16_MAX) == UINT16_MAX)
233	+ (((uImm64 >> 16) & UINT16_MAX) == UINT16_MAX)
234	+ (((uImm64 >> 32) & UINT16_MAX) == UINT16_MAX)
235	+ (((uImm64 >> 48) & UINT16_MAX) == UINT16_MAX);
236	if (cFfffHalfWords <= cZeroHalfWords)
237	{
238	uint32_t fMovBase = UINT32_C(0xd2800000) \| iGpr;
239
240	/* movz gpr, imm16 */
241	uint32_t uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
242	if (uImmPart \|\| cZeroHalfWords == 4)
243	{
244	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
245	fMovBase \|= RT_BIT_32(29);
246	}
247	/* mov[z/k] gpr, imm16, lsl #16 */
248	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
249	if (uImmPart)
250	{
251	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
252	fMovBase \|= RT_BIT_32(29);
253	}
254	/* mov[z/k] gpr, imm16, lsl #32 */
255	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
256	if (uImmPart)
257	{
258	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
259	fMovBase \|= RT_BIT_32(29);
260	}
261	/* mov[z/k] gpr, imm16, lsl #48 */
262	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
263	if (uImmPart)
264	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
265	}
266	else
267	{
268	uint32_t fMovBase = UINT32_C(0x92800000) \| iGpr;
269
270	/* find the first half-word that isn't UINT16_MAX. */
271	uint32_t const iHwNotFfff = (uImm64 & UINT16_MAX) != UINT16_MAX ? 0
272	: ((uImm64 >> 16) & UINT16_MAX) != UINT16_MAX ? 1
273	: ((uImm64 >> 32) & UINT16_MAX) != UINT16_MAX ? 2 : 3;
274
275	/* movn gpr, imm16, lsl #iHwNotFfff16 /
276	uint32_t uImmPart = (uint32_t)(~(uImm64 >> (iHwNotFfff * 16)) & UINT32_C(0xffff)) << 5;
277	pCodeBuf[off++] = fMovBase \| (iHwNotFfff << 21) \| uImmPart;
278	fMovBase \|= RT_BIT_32(30) \| RT_BIT_32(29); /* -> movk */
279	/* movk gpr, imm16 */
280	if (iHwNotFfff != 0)
281	{
282	uImmPart = (uint32_t)((uImm64 >> 0) & UINT32_C(0xffff));
283	if (uImmPart != UINT32_C(0xffff))
284	pCodeBuf[off++] = fMovBase \| (UINT32_C(0) << 21) \| (uImmPart << 5);
285	}
286	/* movk gpr, imm16, lsl #16 */
287	if (iHwNotFfff != 1)
288	{
289	uImmPart = (uint32_t)((uImm64 >> 16) & UINT32_C(0xffff));
290	if (uImmPart != UINT32_C(0xffff))
291	pCodeBuf[off++] = fMovBase \| (UINT32_C(1) << 21) \| (uImmPart << 5);
292	}
293	/* movk gpr, imm16, lsl #32 */
294	if (iHwNotFfff != 2)
295	{
296	uImmPart = (uint32_t)((uImm64 >> 32) & UINT32_C(0xffff));
297	if (uImmPart != UINT32_C(0xffff))
298	pCodeBuf[off++] = fMovBase \| (UINT32_C(2) << 21) \| (uImmPart << 5);
299	}
300	/* movk gpr, imm16, lsl #48 */
301	if (iHwNotFfff != 3)
302	{
303	uImmPart = (uint32_t)((uImm64 >> 48) & UINT32_C(0xffff));
304	if (uImmPart != UINT32_C(0xffff))
305	pCodeBuf[off++] = fMovBase \| (UINT32_C(3) << 21) \| (uImmPart << 5);
306	}
307	}
308
309	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
310	* clang 12.x does that, only to use the 'x' version for the
311	* addressing in the following ldr). */
312
313	#else
314	# error "port me"
315	#endif
316	return off;
317	}
318
319
320	/**
321	* Emits loading a constant into a 64-bit GPR
322	*/
323	DECL_INLINE_THROW(uint32_t)
324	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
325	{
326	#ifdef RT_ARCH_AMD64
327	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 10), off, iGpr, uImm64);
328	#elif defined(RT_ARCH_ARM64)
329	off = iemNativeEmitLoadGprImmEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGpr, uImm64);
330	#else
331	# error "port me"
332	#endif
333	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
334	return off;
335	}
336
337
338	/**
339	* Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
340	* buffer space.
341	*
342	* Max buffer consumption:
343	* - AMD64: 6 instruction bytes.
344	* - ARM64: 2 instruction words (8 bytes).
345	*
346	* @note The top 32 bits will be cleared.
347	*/
348	DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t uImm32)
349	{
350	#ifdef RT_ARCH_AMD64
351	if (uImm32 == 0)
352	{
353	/* xor gpr, gpr */
354	if (iGpr >= 8)
355	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
356	pCodeBuf[off++] = 0x33;
357	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
358	}
359	else
360	{
361	/* mov gpr, imm32 */
362	if (iGpr >= 8)
363	pCodeBuf[off++] = X86_OP_REX_B;
364	pCodeBuf[off++] = 0xb8 + (iGpr & 7);
365	pCodeBuf[off++] = RT_BYTE1(uImm32);
366	pCodeBuf[off++] = RT_BYTE2(uImm32);
367	pCodeBuf[off++] = RT_BYTE3(uImm32);
368	pCodeBuf[off++] = RT_BYTE4(uImm32);
369	}
370
371	#elif defined(RT_ARCH_ARM64)
372	if ((uImm32 >> 16) == 0)
373	/* movz gpr, imm16 */
374	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32, 0, false /f64Bit/);
375	else if ((uImm32 & UINT32_C(0xffff)) == 0)
376	/* movz gpr, imm16, lsl #16 */
377	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 >> 16, 1, false /f64Bit/);
378	else if ((uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
379	/* movn gpr, imm16, lsl #16 */
380	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32 >> 16, 1, false /f64Bit/);
381	else if ((uImm32 >> 16) == UINT32_C(0xffff))
382	/* movn gpr, imm16 */
383	pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~uImm32, 0, false /f64Bit/);
384	else
385	{
386	pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /f64Bit/);
387	pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16, 1, false /f64Bit/);
388	}
389
390	#else
391	# error "port me"
392	#endif
393	return off;
394	}
395
396
397	/**
398	* Emits loading a constant into a 32-bit GPR.
399	* @note The top 32 bits will be cleared.
400	*/
401	DECL_INLINE_THROW(uint32_t)
402	iemNativeEmitLoadGprImm32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t uImm32)
403	{
404	#ifdef RT_ARCH_AMD64
405	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, iGpr, uImm32);
406	#elif defined(RT_ARCH_ARM64)
407	off = iemNativeEmitLoadGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 2), off, iGpr, uImm32);
408	#else
409	# error "port me"
410	#endif
411	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
412	return off;
413	}
414
415
416	/**
417	* Emits loading a constant into a 8-bit GPR
418	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
419	* only the ARM64 version does that.
420	*/
421	DECL_INLINE_THROW(uint32_t)
422	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
423	{
424	#ifdef RT_ARCH_AMD64
425	/* mov gpr, imm8 */
426	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
427	if (iGpr >= 8)
428	pbCodeBuf[off++] = X86_OP_REX_B;
429	else if (iGpr >= 4)
430	pbCodeBuf[off++] = X86_OP_REX;
431	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
432	pbCodeBuf[off++] = RT_BYTE1(uImm8);
433
434	#elif defined(RT_ARCH_ARM64)
435	/* movz gpr, imm16, lsl #0 */
436	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
437	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
438
439	#else
440	# error "port me"
441	#endif
442	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
443	return off;
444	}
445
446
447	#ifdef RT_ARCH_AMD64
448	/**
449	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
450	*/
451	DECL_FORCE_INLINE(uint32_t)
452	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
453	{
454	if (offVCpu < 128)
455	{
456	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
457	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
458	}
459	else
460	{
461	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
462	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
463	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
464	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
465	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
466	}
467	return off;
468	}
469	#elif defined(RT_ARCH_ARM64)
470	/**
471	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
472	*/
473	DECL_FORCE_INLINE_THROW(uint32_t)
474	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
475	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
476	{
477	/*
478	* There are a couple of ldr variants that takes an immediate offset, so
479	* try use those if we can, otherwise we have to use the temporary register
480	* help with the addressing.
481	*/
482	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
483	{
484	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
485	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
486	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
487	}
488	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
489	{
490	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
491	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
492	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
493	}
494	else
495	{
496	/* The offset is too large, so we must load it into a register and use
497	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
498	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
499	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
500
501	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
502	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
503	IEMNATIVE_REG_FIXED_TMP0);
504	}
505	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
506	return off;
507	}
508	#endif
509
510
511	/**
512	* Emits a 64-bit GPR load of a VCpu value.
513	*/
514	DECL_INLINE_THROW(uint32_t)
515	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
516	{
517	#ifdef RT_ARCH_AMD64
518	/* mov reg64, mem64 */
519	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
520	if (iGpr < 8)
521	pbCodeBuf[off++] = X86_OP_REX_W;
522	else
523	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
524	pbCodeBuf[off++] = 0x8b;
525	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
526	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
527
528	#elif defined(RT_ARCH_ARM64)
529	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
530
531	#else
532	# error "port me"
533	#endif
534	return off;
535	}
536
537
538	/**
539	* Emits a 32-bit GPR load of a VCpu value.
540	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
541	*/
542	DECL_INLINE_THROW(uint32_t)
543	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
544	{
545	#ifdef RT_ARCH_AMD64
546	/* mov reg32, mem32 */
547	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
548	if (iGpr >= 8)
549	pbCodeBuf[off++] = X86_OP_REX_R;
550	pbCodeBuf[off++] = 0x8b;
551	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
552	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
553
554	#elif defined(RT_ARCH_ARM64)
555	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
556
557	#else
558	# error "port me"
559	#endif
560	return off;
561	}
562
563
564	/**
565	* Emits a 16-bit GPR load of a VCpu value.
566	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
567	*/
568	DECL_INLINE_THROW(uint32_t)
569	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
570	{
571	#ifdef RT_ARCH_AMD64
572	/* movzx reg32, mem16 */
573	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
574	if (iGpr >= 8)
575	pbCodeBuf[off++] = X86_OP_REX_R;
576	pbCodeBuf[off++] = 0x0f;
577	pbCodeBuf[off++] = 0xb7;
578	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
579	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
580
581	#elif defined(RT_ARCH_ARM64)
582	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
583
584	#else
585	# error "port me"
586	#endif
587	return off;
588	}
589
590
591	/**
592	* Emits a 8-bit GPR load of a VCpu value.
593	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
594	*/
595	DECL_INLINE_THROW(uint32_t)
596	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
597	{
598	#ifdef RT_ARCH_AMD64
599	/* movzx reg32, mem8 */
600	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
601	if (iGpr >= 8)
602	pbCodeBuf[off++] = X86_OP_REX_R;
603	pbCodeBuf[off++] = 0x0f;
604	pbCodeBuf[off++] = 0xb6;
605	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
606	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
607
608	#elif defined(RT_ARCH_ARM64)
609	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
610
611	#else
612	# error "port me"
613	#endif
614	return off;
615	}
616
617
618	/**
619	* Emits a store of a GPR value to a 64-bit VCpu field.
620	*/
621	DECL_INLINE_THROW(uint32_t)
622	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
623	{
624	#ifdef RT_ARCH_AMD64
625	/* mov mem64, reg64 */
626	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
627	if (iGpr < 8)
628	pbCodeBuf[off++] = X86_OP_REX_W;
629	else
630	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
631	pbCodeBuf[off++] = 0x89;
632	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
633	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
634
635	#elif defined(RT_ARCH_ARM64)
636	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
637
638	#else
639	# error "port me"
640	#endif
641	return off;
642	}
643
644
645	/**
646	* Emits a store of a GPR value to a 32-bit VCpu field.
647	*/
648	DECL_INLINE_THROW(uint32_t)
649	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
650	{
651	#ifdef RT_ARCH_AMD64
652	/* mov mem32, reg32 */
653	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
654	if (iGpr >= 8)
655	pbCodeBuf[off++] = X86_OP_REX_R;
656	pbCodeBuf[off++] = 0x89;
657	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
658	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
659
660	#elif defined(RT_ARCH_ARM64)
661	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
662
663	#else
664	# error "port me"
665	#endif
666	return off;
667	}
668
669
670	/**
671	* Emits a store of a GPR value to a 16-bit VCpu field.
672	*/
673	DECL_INLINE_THROW(uint32_t)
674	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
675	{
676	#ifdef RT_ARCH_AMD64
677	/* mov mem16, reg16 */
678	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
679	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
680	if (iGpr >= 8)
681	pbCodeBuf[off++] = X86_OP_REX_R;
682	pbCodeBuf[off++] = 0x89;
683	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
684	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
685
686	#elif defined(RT_ARCH_ARM64)
687	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
688
689	#else
690	# error "port me"
691	#endif
692	return off;
693	}
694
695
696	/**
697	* Emits a store of a GPR value to a 8-bit VCpu field.
698	*/
699	DECL_INLINE_THROW(uint32_t)
700	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
701	{
702	#ifdef RT_ARCH_AMD64
703	/* mov mem8, reg8 */
704	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
705	if (iGpr >= 8)
706	pbCodeBuf[off++] = X86_OP_REX_R;
707	pbCodeBuf[off++] = 0x88;
708	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
709	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
710
711	#elif defined(RT_ARCH_ARM64)
712	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
713
714	#else
715	# error "port me"
716	#endif
717	return off;
718	}
719
720
721	/**
722	* Emits a store of an immediate value to a 8-bit VCpu field.
723	*/
724	DECL_INLINE_THROW(uint32_t)
725	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
726	{
727	#ifdef RT_ARCH_AMD64
728	/* mov mem8, imm8 */
729	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
730	pbCodeBuf[off++] = 0xc6;
731	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
732	pbCodeBuf[off++] = bImm;
733	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
734
735	#elif defined(RT_ARCH_ARM64)
736	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
737	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
738	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
739	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
740
741	#else
742	# error "port me"
743	#endif
744	return off;
745	}
746
747
748	/**
749	* Emits a load effective address to a GRP of a VCpu field.
750	*/
751	DECL_INLINE_THROW(uint32_t)
752	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
753	{
754	#ifdef RT_ARCH_AMD64
755	/* lea gprdst, [rbx + offDisp] */
756	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
757	if (iGprDst < 8)
758	pbCodeBuf[off++] = X86_OP_REX_W;
759	else
760	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
761	pbCodeBuf[off++] = 0x8d;
762	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
763
764	#elif defined(RT_ARCH_ARM64)
765	if (offVCpu < (unsigned)_4K)
766	{
767	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
768	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
769	}
770	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
771	{
772	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
773	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
774	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
775	}
776	else
777	{
778	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
779	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
780	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
781	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
782	}
783
784	#else
785	# error "port me"
786	#endif
787	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
788	return off;
789	}
790
791
792	/**
793	* Emits a gprdst = gprsrc load.
794	*/
795	DECL_INLINE_THROW(uint32_t)
796	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
797	{
798	#ifdef RT_ARCH_AMD64
799	/* mov gprdst, gprsrc */
800	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
801	if ((iGprDst \| iGprSrc) >= 8)
802	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
803	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
804	: X86_OP_REX_W \| X86_OP_REX_R;
805	else
806	pbCodeBuf[off++] = X86_OP_REX_W;
807	pbCodeBuf[off++] = 0x8b;
808	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
809
810	#elif defined(RT_ARCH_ARM64)
811	/* mov dst, src; alias for: orr dst, xzr, src */
812	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
813	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
814
815	#else
816	# error "port me"
817	#endif
818	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
819	return off;
820	}
821
822
823	/**
824	* Emits a gprdst = gprsrc[31:0] load.
825	* @note Bits 63 thru 32 are cleared.
826	*/
827	DECL_INLINE_THROW(uint32_t)
828	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
829	{
830	#ifdef RT_ARCH_AMD64
831	/* mov gprdst, gprsrc */
832	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
833	if ((iGprDst \| iGprSrc) >= 8)
834	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
835	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
836	: X86_OP_REX_R;
837	pbCodeBuf[off++] = 0x8b;
838	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
839
840	#elif defined(RT_ARCH_ARM64)
841	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
842	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
843	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
844
845	#else
846	# error "port me"
847	#endif
848	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
849	return off;
850	}
851
852
853	/**
854	* Emits a gprdst = gprsrc[15:0] load.
855	* @note Bits 63 thru 15 are cleared.
856	*/
857	DECL_INLINE_THROW(uint32_t)
858	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
859	{
860	#ifdef RT_ARCH_AMD64
861	/* movzx Gv,Ew */
862	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
863	if ((iGprDst \| iGprSrc) >= 8)
864	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
865	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
866	: X86_OP_REX_R;
867	pbCodeBuf[off++] = 0x0f;
868	pbCodeBuf[off++] = 0xb7;
869	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
870
871	#elif defined(RT_ARCH_ARM64)
872	/* and gprdst, gprsrc, #0xffff */
873	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
874	# if 1
875	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
876	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
877	# else
878	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
879	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
880	# endif
881
882	#else
883	# error "port me"
884	#endif
885	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
886	return off;
887	}
888
889
890	/**
891	* Emits a gprdst = gprsrc[7:0] load.
892	* @note Bits 63 thru 8 are cleared.
893	*/
894	DECL_INLINE_THROW(uint32_t)
895	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
896	{
897	#ifdef RT_ARCH_AMD64
898	/* movzx Gv,Eb */
899	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
900	if (iGprDst >= 8 \|\| iGprSrc >= 8)
901	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
902	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
903	: X86_OP_REX_R;
904	else if (iGprSrc >= 4)
905	pbCodeBuf[off++] = X86_OP_REX;
906	pbCodeBuf[off++] = 0x0f;
907	pbCodeBuf[off++] = 0xb6;
908	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
909
910	#elif defined(RT_ARCH_ARM64)
911	/* and gprdst, gprsrc, #0xff */
912	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
913	# if 1
914	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
915	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
916	# else
917	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
918	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
919	# endif
920
921	#else
922	# error "port me"
923	#endif
924	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
925	return off;
926	}
927
928
929	/**
930	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
931	* @note Bits 63 thru 8 are cleared.
932	*/
933	DECL_INLINE_THROW(uint32_t)
934	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
935	{
936	#ifdef RT_ARCH_AMD64
937	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
938
939	/* movzx Gv,Ew */
940	if ((iGprDst \| iGprSrc) >= 8)
941	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
942	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
943	: X86_OP_REX_R;
944	pbCodeBuf[off++] = 0x0f;
945	pbCodeBuf[off++] = 0xb7;
946	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
947
948	/* shr Ev,8 */
949	if (iGprDst >= 8)
950	pbCodeBuf[off++] = X86_OP_REX_B;
951	pbCodeBuf[off++] = 0xc1;
952	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
953	pbCodeBuf[off++] = 8;
954
955	#elif defined(RT_ARCH_ARM64)
956	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
957	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
958	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
959
960	#else
961	# error "port me"
962	#endif
963	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
964	return off;
965	}
966
967
968	/**
969	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
970	*/
971	DECL_INLINE_THROW(uint32_t)
972	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
973	{
974	#ifdef RT_ARCH_AMD64
975	/* movsxd r64, r/m32 */
976	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
977	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
978	pbCodeBuf[off++] = 0x63;
979	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
980
981	#elif defined(RT_ARCH_ARM64)
982	/* sxtw dst, src */
983	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
984	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
985
986	#else
987	# error "port me"
988	#endif
989	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
990	return off;
991	}
992
993
994	/**
995	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
996	*/
997	DECL_INLINE_THROW(uint32_t)
998	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
999	{
1000	#ifdef RT_ARCH_AMD64
1001	/* movsx r64, r/m16 */
1002	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1003	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1004	pbCodeBuf[off++] = 0x0f;
1005	pbCodeBuf[off++] = 0xbf;
1006	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1007
1008	#elif defined(RT_ARCH_ARM64)
1009	/* sxth dst, src */
1010	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1011	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
1012
1013	#else
1014	# error "port me"
1015	#endif
1016	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1017	return off;
1018	}
1019
1020
1021	/**
1022	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1023	*/
1024	DECL_INLINE_THROW(uint32_t)
1025	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1026	{
1027	#ifdef RT_ARCH_AMD64
1028	/* movsx r64, r/m16 */
1029	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1030	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1031	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1032	pbCodeBuf[off++] = 0x0f;
1033	pbCodeBuf[off++] = 0xbf;
1034	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1035
1036	#elif defined(RT_ARCH_ARM64)
1037	/* sxth dst32, src */
1038	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1039	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
1040
1041	#else
1042	# error "port me"
1043	#endif
1044	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1045	return off;
1046	}
1047
1048
1049	/**
1050	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
1051	*/
1052	DECL_INLINE_THROW(uint32_t)
1053	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1054	{
1055	#ifdef RT_ARCH_AMD64
1056	/* movsx r64, r/m8 */
1057	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1058	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1059	pbCodeBuf[off++] = 0x0f;
1060	pbCodeBuf[off++] = 0xbe;
1061	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1062
1063	#elif defined(RT_ARCH_ARM64)
1064	/* sxtb dst, src */
1065	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1066	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
1067
1068	#else
1069	# error "port me"
1070	#endif
1071	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1072	return off;
1073	}
1074
1075
1076	/**
1077	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
1078	* @note Bits 63 thru 32 are cleared.
1079	*/
1080	DECL_INLINE_THROW(uint32_t)
1081	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1082	{
1083	#ifdef RT_ARCH_AMD64
1084	/* movsx r32, r/m8 */
1085	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1086	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1087	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1088	pbCodeBuf[off++] = 0x0f;
1089	pbCodeBuf[off++] = 0xbe;
1090	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1091
1092	#elif defined(RT_ARCH_ARM64)
1093	/* sxtb dst32, src32 */
1094	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1095	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1096
1097	#else
1098	# error "port me"
1099	#endif
1100	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1101	return off;
1102	}
1103
1104
1105	/**
1106	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
1107	* @note Bits 63 thru 16 are cleared.
1108	*/
1109	DECL_INLINE_THROW(uint32_t)
1110	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1111	{
1112	#ifdef RT_ARCH_AMD64
1113	/* movsx r16, r/m8 */
1114	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1115	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1116	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1117	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1118	pbCodeBuf[off++] = 0x0f;
1119	pbCodeBuf[off++] = 0xbe;
1120	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1121
1122	/* movzx r32, r/m16 */
1123	if (iGprDst >= 8)
1124	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
1125	pbCodeBuf[off++] = 0x0f;
1126	pbCodeBuf[off++] = 0xb7;
1127	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1128
1129	#elif defined(RT_ARCH_ARM64)
1130	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
1131	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1132	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
1133	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
1134	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
1135
1136	#else
1137	# error "port me"
1138	#endif
1139	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1140	return off;
1141	}
1142
1143
1144	/**
1145	* Emits a gprdst = gprsrc + addend load.
1146	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1147	*/
1148	#ifdef RT_ARCH_AMD64
1149	DECL_INLINE_THROW(uint32_t)
1150	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1151	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1152	{
1153	Assert(iAddend != 0);
1154
1155	/* lea gprdst, [gprsrc + iAddend] */
1156	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1157	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1158	pbCodeBuf[off++] = 0x8d;
1159	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1160	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1161	return off;
1162	}
1163
1164	#elif defined(RT_ARCH_ARM64)
1165	DECL_INLINE_THROW(uint32_t)
1166	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1167	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1168	{
1169	if ((uint32_t)iAddend < 4096)
1170	{
1171	/* add dst, src, uimm12 */
1172	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1173	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
1174	}
1175	else if ((uint32_t)-iAddend < 4096)
1176	{
1177	/* sub dst, src, uimm12 */
1178	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1179	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1180	}
1181	else
1182	{
1183	Assert(iGprSrc != iGprDst);
1184	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, iAddend);
1185	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1186	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1187	}
1188	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1189	return off;
1190	}
1191	#else
1192	# error "port me"
1193	#endif
1194
1195	/**
1196	* Emits a gprdst = gprsrc + addend load, accepting iAddend == 0.
1197	* @note The added is 32-bit for AMD64 and 64-bit for ARM64.
1198	*/
1199	#ifdef RT_ARCH_AMD64
1200	DECL_INLINE_THROW(uint32_t)
1201	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1202	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1203	#else
1204	DECL_INLINE_THROW(uint32_t)
1205	iemNativeEmitLoadGprFromGprWithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1206	uint8_t iGprDst, uint8_t iGprSrc, int64_t iAddend)
1207	#endif
1208	{
1209	if (iAddend != 0)
1210	return iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1211	return iemNativeEmitLoadGprFromGpr(pReNative, off, iGprDst, iGprSrc);
1212	}
1213
1214
1215	/**
1216	* Emits a gprdst = gprsrc32 + addend load.
1217	* @note Bits 63 thru 32 are cleared.
1218	*/
1219	DECL_INLINE_THROW(uint32_t)
1220	iemNativeEmitLoadGprFromGpr32WithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1221	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1222	{
1223	Assert(iAddend != 0);
1224
1225	#ifdef RT_ARCH_AMD64
1226	/* a32 o32 lea gprdst, [gprsrc + iAddend] */
1227	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
1228	pbCodeBuf[off++] = X86_OP_PRF_SIZE_ADDR;
1229	if ((iGprDst \| iGprSrc) >= 8)
1230	pbCodeBuf[off++] = (iGprDst >= 8 ? X86_OP_REX_R : 0) \| (iGprSrc >= 8 ? X86_OP_REX_B : 0);
1231	pbCodeBuf[off++] = 0x8d;
1232	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprSrc, iAddend);
1233
1234	#elif defined(RT_ARCH_ARM64)
1235	if ((uint32_t)iAddend < 4096)
1236	{
1237	/* add dst, src, uimm12 */
1238	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1239	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend, false /f64Bit/);
1240	}
1241	else if ((uint32_t)-iAddend < 4096)
1242	{
1243	/* sub dst, src, uimm12 */
1244	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1245	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend, false /f64Bit/);
1246	}
1247	else
1248	{
1249	Assert(iGprSrc != iGprDst);
1250	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1251	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1252	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst, false /f64Bit/);
1253	}
1254
1255	#else
1256	# error "port me"
1257	#endif
1258	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1259	return off;
1260	}
1261
1262
1263	/**
1264	* Emits a gprdst = gprsrc32 + addend load, accepting iAddend == 0.
1265	*/
1266	DECL_INLINE_THROW(uint32_t)
1267	iemNativeEmitLoadGprFromGpr32WithAddendMaybeZero(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1268	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
1269	{
1270	if (iAddend != 0)
1271	return iemNativeEmitLoadGprFromGpr32WithAddend(pReNative, off, iGprDst, iGprSrc, iAddend);
1272	return iemNativeEmitLoadGprFromGpr32(pReNative, off, iGprDst, iGprSrc);
1273	}
1274
1275
1276
1277	#ifdef RT_ARCH_AMD64
1278	/**
1279	* Common bit of iemNativeEmitLoadGprByBp and friends.
1280	*/
1281	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1282	PIEMRECOMPILERSTATE pReNativeAssert)
1283	{
1284	if (offDisp < 128 && offDisp >= -128)
1285	{
1286	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1287	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1288	}
1289	else
1290	{
1291	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1292	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1293	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1294	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1295	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1296	}
1297	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1298	return off;
1299	}
1300	#elif defined(RT_ARCH_ARM64)
1301	/**
1302	* Common bit of iemNativeEmitLoadGprByBp and friends.
1303	*/
1304	DECL_FORCE_INLINE_THROW(uint32_t)
1305	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1306	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1307	{
1308	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1309	{
1310	/* str w/ unsigned imm12 (scaled) */
1311	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1312	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1313	}
1314	else if (offDisp >= -256 && offDisp <= 256)
1315	{
1316	/* stur w/ signed imm9 (unscaled) */
1317	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1318	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1319	}
1320	else
1321	{
1322	/* Use temporary indexing register. */
1323	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1324	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1325	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1326	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1327	}
1328	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1329	return off;
1330	}
1331	#endif
1332
1333
1334	/**
1335	* Emits a 64-bit GRP load instruction with an BP relative source address.
1336	*/
1337	DECL_INLINE_THROW(uint32_t)
1338	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1339	{
1340	#ifdef RT_ARCH_AMD64
1341	/* mov gprdst, qword [rbp + offDisp] */
1342	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1343	if (iGprDst < 8)
1344	pbCodeBuf[off++] = X86_OP_REX_W;
1345	else
1346	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1347	pbCodeBuf[off++] = 0x8b;
1348	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1349
1350	#elif defined(RT_ARCH_ARM64)
1351	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1352
1353	#else
1354	# error "port me"
1355	#endif
1356	}
1357
1358
1359	/**
1360	* Emits a 32-bit GRP load instruction with an BP relative source address.
1361	* @note Bits 63 thru 32 of the GPR will be cleared.
1362	*/
1363	DECL_INLINE_THROW(uint32_t)
1364	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1365	{
1366	#ifdef RT_ARCH_AMD64
1367	/* mov gprdst, dword [rbp + offDisp] */
1368	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1369	if (iGprDst >= 8)
1370	pbCodeBuf[off++] = X86_OP_REX_R;
1371	pbCodeBuf[off++] = 0x8b;
1372	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1373
1374	#elif defined(RT_ARCH_ARM64)
1375	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1376
1377	#else
1378	# error "port me"
1379	#endif
1380	}
1381
1382
1383	/**
1384	* Emits a 16-bit GRP load instruction with an BP relative source address.
1385	* @note Bits 63 thru 16 of the GPR will be cleared.
1386	*/
1387	DECL_INLINE_THROW(uint32_t)
1388	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1389	{
1390	#ifdef RT_ARCH_AMD64
1391	/* movzx gprdst, word [rbp + offDisp] */
1392	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1393	if (iGprDst >= 8)
1394	pbCodeBuf[off++] = X86_OP_REX_R;
1395	pbCodeBuf[off++] = 0x0f;
1396	pbCodeBuf[off++] = 0xb7;
1397	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1398
1399	#elif defined(RT_ARCH_ARM64)
1400	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1401
1402	#else
1403	# error "port me"
1404	#endif
1405	}
1406
1407
1408	/**
1409	* Emits a 8-bit GRP load instruction with an BP relative source address.
1410	* @note Bits 63 thru 8 of the GPR will be cleared.
1411	*/
1412	DECL_INLINE_THROW(uint32_t)
1413	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1414	{
1415	#ifdef RT_ARCH_AMD64
1416	/* movzx gprdst, byte [rbp + offDisp] */
1417	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1418	if (iGprDst >= 8)
1419	pbCodeBuf[off++] = X86_OP_REX_R;
1420	pbCodeBuf[off++] = 0x0f;
1421	pbCodeBuf[off++] = 0xb6;
1422	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1423
1424	#elif defined(RT_ARCH_ARM64)
1425	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1426
1427	#else
1428	# error "port me"
1429	#endif
1430	}
1431
1432
1433	/**
1434	* Emits a load effective address to a GRP with an BP relative source address.
1435	*/
1436	DECL_INLINE_THROW(uint32_t)
1437	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1438	{
1439	#ifdef RT_ARCH_AMD64
1440	/* lea gprdst, [rbp + offDisp] */
1441	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1442	if (iGprDst < 8)
1443	pbCodeBuf[off++] = X86_OP_REX_W;
1444	else
1445	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1446	pbCodeBuf[off++] = 0x8d;
1447	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1448
1449	#elif defined(RT_ARCH_ARM64)
1450	if ((uint32_t)offDisp < (unsigned)_4K)
1451	{
1452	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1453	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1454	}
1455	else if ((uint32_t)-offDisp < (unsigned)_4K)
1456	{
1457	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1458	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1459	}
1460	else
1461	{
1462	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1463	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1464	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1465	if (offDisp >= 0)
1466	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1467	else
1468	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1469	}
1470
1471	#else
1472	# error "port me"
1473	#endif
1474
1475	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1476	return off;
1477	}
1478
1479
1480	/**
1481	* Emits a 64-bit GPR store with an BP relative destination address.
1482	*
1483	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1484	*/
1485	DECL_INLINE_THROW(uint32_t)
1486	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1487	{
1488	#ifdef RT_ARCH_AMD64
1489	/* mov qword [rbp + offDisp], gprdst */
1490	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1491	if (iGprSrc < 8)
1492	pbCodeBuf[off++] = X86_OP_REX_W;
1493	else
1494	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1495	pbCodeBuf[off++] = 0x89;
1496	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1497
1498	#elif defined(RT_ARCH_ARM64)
1499	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1500	{
1501	/* str w/ unsigned imm12 (scaled) */
1502	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1503	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1504	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1505	}
1506	else if (offDisp >= -256 && offDisp <= 256)
1507	{
1508	/* stur w/ signed imm9 (unscaled) */
1509	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1510	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1511	}
1512	else if ((uint32_t)-offDisp < (unsigned)_4K)
1513	{
1514	/* Use temporary indexing register w/ sub uimm12. */
1515	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1516	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1517	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1518	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1519	}
1520	else
1521	{
1522	/* Use temporary indexing register. */
1523	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1524	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1525	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1526	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1527	}
1528	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1529	return off;
1530
1531	#else
1532	# error "Port me!"
1533	#endif
1534	}
1535
1536
1537	/**
1538	* Emits a 64-bit immediate store with an BP relative destination address.
1539	*
1540	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1541	*/
1542	DECL_INLINE_THROW(uint32_t)
1543	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1544	{
1545	#ifdef RT_ARCH_AMD64
1546	if ((int64_t)uImm64 == (int32_t)uImm64)
1547	{
1548	/* mov qword [rbp + offDisp], imm32 - sign extended */
1549	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1550	pbCodeBuf[off++] = X86_OP_REX_W;
1551	pbCodeBuf[off++] = 0xc7;
1552	if (offDisp < 128 && offDisp >= -128)
1553	{
1554	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1555	pbCodeBuf[off++] = (uint8_t)offDisp;
1556	}
1557	else
1558	{
1559	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1560	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1561	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1562	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1563	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1564	}
1565	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1566	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1567	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1568	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1569	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1570	return off;
1571	}
1572	#endif
1573
1574	/* Load tmp0, imm64; Store tmp to bp+disp. */
1575	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1576	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1577	}
1578
1579
1580	#if defined(RT_ARCH_ARM64)
1581	/**
1582	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1583	*/
1584	DECL_FORCE_INLINE_THROW(uint32_t)
1585	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1586	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1587	{
1588	/*
1589	* There are a couple of ldr variants that takes an immediate offset, so
1590	* try use those if we can, otherwise we have to use the temporary register
1591	* help with the addressing.
1592	*/
1593	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1594	{
1595	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1596	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1597	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1598	}
1599	else
1600	{
1601	/* The offset is too large, so we must load it into a register and use
1602	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1603	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1604	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1605
1606	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1607	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1608
1609	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1610	}
1611	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1612	return off;
1613	}
1614	#endif
1615
1616
1617	/**
1618	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1619	*/
1620	DECL_INLINE_THROW(uint32_t)
1621	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1622	{
1623	#ifdef RT_ARCH_AMD64
1624	/* mov reg64, mem64 */
1625	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1626	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1627	pbCodeBuf[off++] = 0x8b;
1628	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1629	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1630
1631	#elif defined(RT_ARCH_ARM64)
1632	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1633
1634	#else
1635	# error "port me"
1636	#endif
1637	return off;
1638	}
1639
1640
1641	/**
1642	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1643	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1644	*/
1645	DECL_INLINE_THROW(uint32_t)
1646	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1647	{
1648	#ifdef RT_ARCH_AMD64
1649	/* mov reg32, mem32 */
1650	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1651	if (iGprDst >= 8 \|\| iGprBase >= 8)
1652	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1653	pbCodeBuf[off++] = 0x8b;
1654	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1655	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1656
1657	#elif defined(RT_ARCH_ARM64)
1658	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1659
1660	#else
1661	# error "port me"
1662	#endif
1663	return off;
1664	}
1665
1666
1667	/*********************************************************************************************************************************
1668	* Subtraction and Additions *
1669	*********************************************************************************************************************************/
1670
1671	/**
1672	* Emits subtracting a 64-bit GPR from another, storing the result in the first.
1673	* @note The AMD64 version sets flags.
1674	*/
1675	DECL_INLINE_THROW(uint32_t)
1676	iemNativeEmitSubTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
1677	{
1678	#if defined(RT_ARCH_AMD64)
1679	/* sub Gv,Ev */
1680	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1681	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1682	\| (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
1683	pbCodeBuf[off++] = 0x2b;
1684	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
1685
1686	#elif defined(RT_ARCH_ARM64)
1687	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1688	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend);
1689
1690	#else
1691	# error "Port me"
1692	#endif
1693	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1694	return off;
1695	}
1696
1697
1698	#ifdef RT_ARCH_AMD64
1699	/**
1700	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1701	*/
1702	DECL_INLINE_THROW(uint32_t)
1703	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1704	{
1705	/* sub gprdst, imm8/imm32 */
1706	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1707	if (iGprDst < 8)
1708	pbCodeBuf[off++] = X86_OP_REX_W;
1709	else
1710	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1711	if (iSubtrahend < 128 && iSubtrahend >= -128)
1712	{
1713	pbCodeBuf[off++] = 0x83;
1714	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1715	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1716	}
1717	else
1718	{
1719	pbCodeBuf[off++] = 0x81;
1720	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1721	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1722	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1723	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1724	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1725	}
1726	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1727	return off;
1728	}
1729	#endif
1730
1731
1732	/**
1733	* Emits adding a 64-bit GPR to another, storing the result in the first.
1734	* @note The AMD64 version sets flags.
1735	*/
1736	DECL_INLINE_THROW(uint32_t)
1737	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1738	{
1739	#if defined(RT_ARCH_AMD64)
1740	/* add Gv,Ev */
1741	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1742	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1743	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1744	pbCodeBuf[off++] = 0x03;
1745	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1746
1747	#elif defined(RT_ARCH_ARM64)
1748	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1749	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1750
1751	#else
1752	# error "Port me"
1753	#endif
1754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1755	return off;
1756	}
1757
1758
1759	/**
1760	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1761	*/
1762	DECL_INLINE_THROW(uint32_t)
1763	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1764	{
1765	#if defined(RT_ARCH_AMD64)
1766	/* add or inc */
1767	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1768	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1769	if (iImm8 != 1)
1770	{
1771	pbCodeBuf[off++] = 0x83;
1772	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1773	pbCodeBuf[off++] = (uint8_t)iImm8;
1774	}
1775	else
1776	{
1777	pbCodeBuf[off++] = 0xff;
1778	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1779	}
1780
1781	#elif defined(RT_ARCH_ARM64)
1782	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1783	if (iImm8 >= 0)
1784	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1785	else
1786	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1787
1788	#else
1789	# error "Port me"
1790	#endif
1791	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1792	return off;
1793	}
1794
1795
1796	/**
1797	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1798	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1799	*/
1800	DECL_INLINE_THROW(uint32_t)
1801	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1802	{
1803	#if defined(RT_ARCH_AMD64)
1804	/* add or inc */
1805	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1806	if (iGprDst >= 8)
1807	pbCodeBuf[off++] = X86_OP_REX_B;
1808	if (iImm8 != 1)
1809	{
1810	pbCodeBuf[off++] = 0x83;
1811	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1812	pbCodeBuf[off++] = (uint8_t)iImm8;
1813	}
1814	else
1815	{
1816	pbCodeBuf[off++] = 0xff;
1817	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1818	}
1819
1820	#elif defined(RT_ARCH_ARM64)
1821	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1822	if (iImm8 >= 0)
1823	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1824	else
1825	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1826
1827	#else
1828	# error "Port me"
1829	#endif
1830	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1831	return off;
1832	}
1833
1834
1835	/**
1836	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1837	*/
1838	DECL_INLINE_THROW(uint32_t)
1839	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1840	{
1841	#if defined(RT_ARCH_AMD64)
1842	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1843	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1844
1845	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1846	{
1847	/* add grp, imm32 */
1848	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1849	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1850	pbCodeBuf[off++] = 0x81;
1851	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1852	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1853	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1854	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1855	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1856	}
1857	else
1858	{
1859	/* Best to use a temporary register to deal with this in the simplest way: */
1860	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1861
1862	/* add dst, tmpreg */
1863	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1864	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1865	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1866	pbCodeBuf[off++] = 0x03;
1867	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1868
1869	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1870	}
1871
1872	#elif defined(RT_ARCH_ARM64)
1873	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1874	{
1875	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1876	if (iAddend >= 0)
1877	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1878	else
1879	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1880	}
1881	else
1882	{
1883	/* Use temporary register for the immediate. */
1884	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1885
1886	/* add gprdst, gprdst, tmpreg */
1887	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1888	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1889
1890	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1891	}
1892
1893	#else
1894	# error "Port me"
1895	#endif
1896	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1897	return off;
1898	}
1899
1900
1901	/**
1902	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1903	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1904	*/
1905	DECL_INLINE_THROW(uint32_t)
1906	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1907	{
1908	#if defined(RT_ARCH_AMD64)
1909	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1910	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1911
1912	/* add grp, imm32 */
1913	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1914	if (iGprDst >= 8)
1915	pbCodeBuf[off++] = X86_OP_REX_B;
1916	pbCodeBuf[off++] = 0x81;
1917	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1918	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1919	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1920	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1921	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1922
1923	#elif defined(RT_ARCH_ARM64)
1924	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1925	{
1926	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1927	if (iAddend >= 0)
1928	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1929	else
1930	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1931	}
1932	else
1933	{
1934	/* Use temporary register for the immediate. */
1935	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1936
1937	/* add gprdst, gprdst, tmpreg */
1938	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1939	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1940
1941	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1942	}
1943
1944	#else
1945	# error "Port me"
1946	#endif
1947	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1948	return off;
1949	}
1950
1951
1952	/*********************************************************************************************************************************
1953	* Unary Operations *
1954	*********************************************************************************************************************************/
1955
1956	/**
1957	* Emits code for clearing bits 16 thru 63 in the GPR.
1958	*/
1959	DECL_INLINE_THROW(uint32_t)
1960	iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1961	{
1962	#if defined(RT_ARCH_AMD64)
1963	/* neg Ev */
1964	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1965	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1966	pbCodeBuf[off++] = 0xf7;
1967	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
1968
1969	#elif defined(RT_ARCH_ARM64)
1970	/* sub dst, xzr, dst */
1971	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1972	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
1973
1974	#else
1975	# error "Port me"
1976	#endif
1977	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1978	return off;
1979	}
1980
1981
1982
1983	/*********************************************************************************************************************************
1984	* Bit Operations *
1985	*********************************************************************************************************************************/
1986
1987	/**
1988	* Emits code for clearing bits 16 thru 63 in the GPR.
1989	*/
1990	DECL_INLINE_THROW(uint32_t)
1991	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1992	{
1993	#if defined(RT_ARCH_AMD64)
1994	/* movzx Gv,Ew */
1995	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1996	if (iGprDst >= 8)
1997	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1998	pbCodeBuf[off++] = 0x0f;
1999	pbCodeBuf[off++] = 0xb7;
2000	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2001
2002	#elif defined(RT_ARCH_ARM64)
2003	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2004	# if 1
2005	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2006	# else
2007	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2008	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2009	# endif
2010	#else
2011	# error "Port me"
2012	#endif
2013	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2014	return off;
2015	}
2016
2017
2018	/**
2019	* Emits code for AND'ing two 64-bit GPRs.
2020	*
2021	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2022	* and ARM64 hosts.
2023	*/
2024	DECL_INLINE_THROW(uint32_t)
2025	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2026	{
2027	#if defined(RT_ARCH_AMD64)
2028	/* and Gv, Ev */
2029	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2030	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2031	pbCodeBuf[off++] = 0x23;
2032	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2033	RT_NOREF(fSetFlags);
2034
2035	#elif defined(RT_ARCH_ARM64)
2036	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2037	if (!fSetFlags)
2038	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2039	else
2040	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2041
2042	#else
2043	# error "Port me"
2044	#endif
2045	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2046	return off;
2047	}
2048
2049
2050	/**
2051	* Emits code for AND'ing two 32-bit GPRs.
2052	*/
2053	DECL_INLINE_THROW(uint32_t)
2054	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2055	{
2056	#if defined(RT_ARCH_AMD64)
2057	/* and Gv, Ev */
2058	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2059	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2060	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2061	pbCodeBuf[off++] = 0x23;
2062	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2063	RT_NOREF(fSetFlags);
2064
2065	#elif defined(RT_ARCH_ARM64)
2066	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2067	if (!fSetFlags)
2068	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2069	else
2070	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2071
2072	#else
2073	# error "Port me"
2074	#endif
2075	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2076	return off;
2077	}
2078
2079
2080	/**
2081	* Emits code for AND'ing a 64-bit GPRs with a constant.
2082	*
2083	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2084	* and ARM64 hosts.
2085	*/
2086	DECL_INLINE_THROW(uint32_t)
2087	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2088	{
2089	#if defined(RT_ARCH_AMD64)
2090	if ((int64_t)uImm == (int8_t)uImm)
2091	{
2092	/* and Ev, imm8 */
2093	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2094	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2095	pbCodeBuf[off++] = 0x83;
2096	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2097	pbCodeBuf[off++] = (uint8_t)uImm;
2098	}
2099	else if ((int64_t)uImm == (int32_t)uImm)
2100	{
2101	/* and Ev, imm32 */
2102	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2103	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
2104	pbCodeBuf[off++] = 0x81;
2105	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2106	pbCodeBuf[off++] = RT_BYTE1(uImm);
2107	pbCodeBuf[off++] = RT_BYTE2(uImm);
2108	pbCodeBuf[off++] = RT_BYTE3(uImm);
2109	pbCodeBuf[off++] = RT_BYTE4(uImm);
2110	}
2111	else
2112	{
2113	/* Use temporary register for the 64-bit immediate. */
2114	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2115	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2116	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2117	}
2118	RT_NOREF(fSetFlags);
2119
2120	#elif defined(RT_ARCH_ARM64)
2121	uint32_t uImmR = 0;
2122	uint32_t uImmNandS = 0;
2123	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
2124	{
2125	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2126	if (!fSetFlags)
2127	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2128	else
2129	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2130	}
2131	else
2132	{
2133	/* Use temporary register for the 64-bit immediate. */
2134	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2135	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2136	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2137	}
2138
2139	#else
2140	# error "Port me"
2141	#endif
2142	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2143	return off;
2144	}
2145
2146
2147	/**
2148	* Emits code for AND'ing an 32-bit GPRs with a constant.
2149	*/
2150	DECL_INLINE_THROW(uint32_t)
2151	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2152	{
2153	#if defined(RT_ARCH_AMD64)
2154	/* and Ev, imm */
2155	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2156	if (iGprDst >= 8)
2157	pbCodeBuf[off++] = X86_OP_REX_B;
2158	if ((int32_t)uImm == (int8_t)uImm)
2159	{
2160	pbCodeBuf[off++] = 0x83;
2161	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2162	pbCodeBuf[off++] = (uint8_t)uImm;
2163	}
2164	else
2165	{
2166	pbCodeBuf[off++] = 0x81;
2167	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2168	pbCodeBuf[off++] = RT_BYTE1(uImm);
2169	pbCodeBuf[off++] = RT_BYTE2(uImm);
2170	pbCodeBuf[off++] = RT_BYTE3(uImm);
2171	pbCodeBuf[off++] = RT_BYTE4(uImm);
2172	}
2173	RT_NOREF(fSetFlags);
2174
2175	#elif defined(RT_ARCH_ARM64)
2176	uint32_t uImmR = 0;
2177	uint32_t uImmNandS = 0;
2178	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2179	{
2180	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2181	if (!fSetFlags)
2182	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2183	else
2184	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2185	}
2186	else
2187	{
2188	/* Use temporary register for the 64-bit immediate. */
2189	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2190	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2191	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2192	}
2193
2194	#else
2195	# error "Port me"
2196	#endif
2197	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2198	return off;
2199	}
2200
2201
2202	/**
2203	* Emits code for XOR'ing two 64-bit GPRs.
2204	*/
2205	DECL_INLINE_THROW(uint32_t)
2206	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2207	{
2208	#if defined(RT_ARCH_AMD64)
2209	/* and Gv, Ev */
2210	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2211	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2212	pbCodeBuf[off++] = 0x33;
2213	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2214
2215	#elif defined(RT_ARCH_ARM64)
2216	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2217	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2218
2219	#else
2220	# error "Port me"
2221	#endif
2222	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2223	return off;
2224	}
2225
2226
2227	/**
2228	* Emits code for XOR'ing two 32-bit GPRs.
2229	*/
2230	DECL_INLINE_THROW(uint32_t)
2231	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2232	{
2233	#if defined(RT_ARCH_AMD64)
2234	/* and Gv, Ev */
2235	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2236	if (iGprDst >= 8 \|\| iGprSrc >= 8)
2237	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2238	pbCodeBuf[off++] = 0x33;
2239	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2240
2241	#elif defined(RT_ARCH_ARM64)
2242	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2243	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
2244
2245	#else
2246	# error "Port me"
2247	#endif
2248	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2249	return off;
2250	}
2251
2252
2253	/*********************************************************************************************************************************
2254	* Shifting *
2255	*********************************************************************************************************************************/
2256
2257	/**
2258	* Emits code for shifting a GPR a fixed number of bits to the left.
2259	*/
2260	DECL_INLINE_THROW(uint32_t)
2261	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2262	{
2263	Assert(cShift > 0 && cShift < 64);
2264
2265	#if defined(RT_ARCH_AMD64)
2266	/* shl dst, cShift */
2267	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2268	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2269	if (cShift != 1)
2270	{
2271	pbCodeBuf[off++] = 0xc1;
2272	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2273	pbCodeBuf[off++] = cShift;
2274	}
2275	else
2276	{
2277	pbCodeBuf[off++] = 0xd1;
2278	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2279	}
2280
2281	#elif defined(RT_ARCH_ARM64)
2282	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2283	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2284
2285	#else
2286	# error "Port me"
2287	#endif
2288	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2289	return off;
2290	}
2291
2292
2293	/**
2294	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2295	*/
2296	DECL_INLINE_THROW(uint32_t)
2297	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2298	{
2299	Assert(cShift > 0 && cShift < 32);
2300
2301	#if defined(RT_ARCH_AMD64)
2302	/* shl dst, cShift */
2303	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2304	if (iGprDst >= 8)
2305	pbCodeBuf[off++] = X86_OP_REX_B;
2306	if (cShift != 1)
2307	{
2308	pbCodeBuf[off++] = 0xc1;
2309	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2310	pbCodeBuf[off++] = cShift;
2311	}
2312	else
2313	{
2314	pbCodeBuf[off++] = 0xd1;
2315	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2316	}
2317
2318	#elif defined(RT_ARCH_ARM64)
2319	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2320	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2321
2322	#else
2323	# error "Port me"
2324	#endif
2325	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2326	return off;
2327	}
2328
2329
2330	/**
2331	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2332	*/
2333	DECL_INLINE_THROW(uint32_t)
2334	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2335	{
2336	Assert(cShift > 0 && cShift < 64);
2337
2338	#if defined(RT_ARCH_AMD64)
2339	/* shr dst, cShift */
2340	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2341	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2342	if (cShift != 1)
2343	{
2344	pbCodeBuf[off++] = 0xc1;
2345	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2346	pbCodeBuf[off++] = cShift;
2347	}
2348	else
2349	{
2350	pbCodeBuf[off++] = 0xd1;
2351	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2352	}
2353
2354	#elif defined(RT_ARCH_ARM64)
2355	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2356	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2357
2358	#else
2359	# error "Port me"
2360	#endif
2361	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2362	return off;
2363	}
2364
2365
2366	/**
2367	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2368	* right.
2369	*/
2370	DECL_INLINE_THROW(uint32_t)
2371	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2372	{
2373	Assert(cShift > 0 && cShift < 32);
2374
2375	#if defined(RT_ARCH_AMD64)
2376	/* shr dst, cShift */
2377	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2378	if (iGprDst >= 8)
2379	pbCodeBuf[off++] = X86_OP_REX_B;
2380	if (cShift != 1)
2381	{
2382	pbCodeBuf[off++] = 0xc1;
2383	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2384	pbCodeBuf[off++] = cShift;
2385	}
2386	else
2387	{
2388	pbCodeBuf[off++] = 0xd1;
2389	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2390	}
2391
2392	#elif defined(RT_ARCH_ARM64)
2393	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2394	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2395
2396	#else
2397	# error "Port me"
2398	#endif
2399	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2400	return off;
2401	}
2402
2403
2404
2405	/*********************************************************************************************************************************
2406	* Compare and Testing *
2407	*********************************************************************************************************************************/
2408
2409
2410	#ifdef RT_ARCH_ARM64
2411	/**
2412	* Emits an ARM64 compare instruction.
2413	*/
2414	DECL_INLINE_THROW(uint32_t)
2415	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2416	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2417	{
2418	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2419	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2420	f64Bit, true /fSetFlags/, cShift, enmShift);
2421	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2422	return off;
2423	}
2424	#endif
2425
2426
2427	/**
2428	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2429	* with conditional instruction.
2430	*/
2431	DECL_INLINE_THROW(uint32_t)
2432	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2433	{
2434	#ifdef RT_ARCH_AMD64
2435	/* cmp Gv, Ev */
2436	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2437	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2438	pbCodeBuf[off++] = 0x3b;
2439	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2440	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2441
2442	#elif defined(RT_ARCH_ARM64)
2443	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2444
2445	#else
2446	# error "Port me!"
2447	#endif
2448	return off;
2449	}
2450
2451
2452	/**
2453	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2454	* with conditional instruction.
2455	*/
2456	DECL_INLINE_THROW(uint32_t)
2457	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2458	{
2459	#ifdef RT_ARCH_AMD64
2460	/* cmp Gv, Ev */
2461	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2462	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2463	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2464	pbCodeBuf[off++] = 0x3b;
2465	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2466	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2467
2468	#elif defined(RT_ARCH_ARM64)
2469	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2470
2471	#else
2472	# error "Port me!"
2473	#endif
2474	return off;
2475	}
2476
2477
2478	/**
2479	* Emits a compare of a 64-bit GPR with a constant value, settings status
2480	* flags/whatever for use with conditional instruction.
2481	*/
2482	DECL_INLINE_THROW(uint32_t)
2483	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2484	{
2485	#ifdef RT_ARCH_AMD64
2486	if (uImm <= UINT32_C(0xff))
2487	{
2488	/* cmp Ev, Ib */
2489	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2490	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2491	pbCodeBuf[off++] = 0x83;
2492	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2493	pbCodeBuf[off++] = (uint8_t)uImm;
2494	}
2495	else if ((int64_t)uImm == (int32_t)uImm)
2496	{
2497	/* cmp Ev, imm */
2498	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2499	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2500	pbCodeBuf[off++] = 0x81;
2501	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2502	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2503	pbCodeBuf[off++] = RT_BYTE1(uImm);
2504	pbCodeBuf[off++] = RT_BYTE2(uImm);
2505	pbCodeBuf[off++] = RT_BYTE3(uImm);
2506	pbCodeBuf[off++] = RT_BYTE4(uImm);
2507	}
2508	else
2509	{
2510	/* Use temporary register for the immediate. */
2511	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2512	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2513	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2514	}
2515
2516	#elif defined(RT_ARCH_ARM64)
2517	/** @todo guess there are clevere things we can do here... */
2518	if (uImm < _4K)
2519	{
2520	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2521	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2522	true /64Bit/, true /fSetFlags/);
2523	}
2524	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2525	{
2526	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2527	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
2528	true /64Bit/, true /fSetFlags/, true /fShift12/);
2529	}
2530	else
2531	{
2532	/* Use temporary register for the immediate. */
2533	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2534	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2535	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2536	}
2537
2538	#else
2539	# error "Port me!"
2540	#endif
2541
2542	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2543	return off;
2544	}
2545
2546
2547	/**
2548	* Emits a compare of a 32-bit GPR with a constant value, settings status
2549	* flags/whatever for use with conditional instruction.
2550	*/
2551	DECL_INLINE_THROW(uint32_t)
2552	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2553	{
2554	#ifdef RT_ARCH_AMD64
2555	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2556	if (iGprLeft >= 8)
2557	pbCodeBuf[off++] = X86_OP_REX_B;
2558	if (uImm <= UINT32_C(0xff))
2559	{
2560	/* cmp Ev, Ib */
2561	pbCodeBuf[off++] = 0x83;
2562	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2563	pbCodeBuf[off++] = (uint8_t)uImm;
2564	}
2565	else
2566	{
2567	/* cmp Ev, imm */
2568	pbCodeBuf[off++] = 0x81;
2569	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2570	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2571	pbCodeBuf[off++] = RT_BYTE1(uImm);
2572	pbCodeBuf[off++] = RT_BYTE2(uImm);
2573	pbCodeBuf[off++] = RT_BYTE3(uImm);
2574	pbCodeBuf[off++] = RT_BYTE4(uImm);
2575	}
2576
2577	#elif defined(RT_ARCH_ARM64)
2578	/** @todo guess there are clevere things we can do here... */
2579	if (uImm < _4K)
2580	{
2581	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2582	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2583	false /64Bit/, true /fSetFlags/);
2584	}
2585	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2586	{
2587	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2588	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2589	false /64Bit/, true /fSetFlags/, true /fShift12/);
2590	}
2591	else
2592	{
2593	/* Use temporary register for the immediate. */
2594	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2595	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2596	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2597	}
2598
2599	#else
2600	# error "Port me!"
2601	#endif
2602
2603	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2604	return off;
2605	}
2606
2607
2608
2609	/*********************************************************************************************************************************
2610	* Branching *
2611	*********************************************************************************************************************************/
2612
2613	/**
2614	* Emits a JMP rel32 / B imm19 to the given label.
2615	*/
2616	DECL_INLINE_THROW(uint32_t)
2617	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2618	{
2619	Assert(idxLabel < pReNative->cLabels);
2620
2621	#ifdef RT_ARCH_AMD64
2622	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2623	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2624	{
2625	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2626	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2627	{
2628	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2629	pbCodeBuf[off++] = (uint8_t)offRel;
2630	}
2631	else
2632	{
2633	offRel -= 3;
2634	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2635	pbCodeBuf[off++] = RT_BYTE1(offRel);
2636	pbCodeBuf[off++] = RT_BYTE2(offRel);
2637	pbCodeBuf[off++] = RT_BYTE3(offRel);
2638	pbCodeBuf[off++] = RT_BYTE4(offRel);
2639	}
2640	}
2641	else
2642	{
2643	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2644	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2645	pbCodeBuf[off++] = 0xfe;
2646	pbCodeBuf[off++] = 0xff;
2647	pbCodeBuf[off++] = 0xff;
2648	pbCodeBuf[off++] = 0xff;
2649	}
2650	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2651
2652	#elif defined(RT_ARCH_ARM64)
2653	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2654	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2655	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2656	else
2657	{
2658	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2659	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2660	}
2661
2662	#else
2663	# error "Port me!"
2664	#endif
2665	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2666	return off;
2667	}
2668
2669
2670	/**
2671	* Emits a JMP rel32 / B imm19 to a new undefined label.
2672	*/
2673	DECL_INLINE_THROW(uint32_t)
2674	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2675	{
2676	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2677	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2678	}
2679
2680	/** Condition type. */
2681	#ifdef RT_ARCH_AMD64
2682	typedef enum IEMNATIVEINSTRCOND : uint8_t
2683	{
2684	kIemNativeInstrCond_o = 0,
2685	kIemNativeInstrCond_no,
2686	kIemNativeInstrCond_c,
2687	kIemNativeInstrCond_nc,
2688	kIemNativeInstrCond_e,
2689	kIemNativeInstrCond_ne,
2690	kIemNativeInstrCond_be,
2691	kIemNativeInstrCond_nbe,
2692	kIemNativeInstrCond_s,
2693	kIemNativeInstrCond_ns,
2694	kIemNativeInstrCond_p,
2695	kIemNativeInstrCond_np,
2696	kIemNativeInstrCond_l,
2697	kIemNativeInstrCond_nl,
2698	kIemNativeInstrCond_le,
2699	kIemNativeInstrCond_nle
2700	} IEMNATIVEINSTRCOND;
2701	#elif defined(RT_ARCH_ARM64)
2702	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2703	#else
2704	# error "Port me!"
2705	#endif
2706
2707
2708	/**
2709	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2710	*/
2711	DECL_INLINE_THROW(uint32_t)
2712	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2713	{
2714	Assert(idxLabel < pReNative->cLabels);
2715
2716	#ifdef RT_ARCH_AMD64
2717	/* jcc rel32 */
2718	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2719	pbCodeBuf[off++] = 0x0f;
2720	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2721	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2722	pbCodeBuf[off++] = 0x00;
2723	pbCodeBuf[off++] = 0x00;
2724	pbCodeBuf[off++] = 0x00;
2725	pbCodeBuf[off++] = 0x00;
2726
2727	#elif defined(RT_ARCH_ARM64)
2728	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2729	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2730	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2731
2732	#else
2733	# error "Port me!"
2734	#endif
2735	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2736	return off;
2737	}
2738
2739
2740	/**
2741	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2742	*/
2743	DECL_INLINE_THROW(uint32_t)
2744	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2745	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2746	{
2747	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2748	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2749	}
2750
2751
2752	/**
2753	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2754	*/
2755	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2756	{
2757	#ifdef RT_ARCH_AMD64
2758	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2759	#elif defined(RT_ARCH_ARM64)
2760	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2761	#else
2762	# error "Port me!"
2763	#endif
2764	}
2765
2766	/**
2767	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2768	*/
2769	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2770	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2771	{
2772	#ifdef RT_ARCH_AMD64
2773	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2774	#elif defined(RT_ARCH_ARM64)
2775	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2776	#else
2777	# error "Port me!"
2778	#endif
2779	}
2780
2781
2782	/**
2783	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2784	*/
2785	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2786	{
2787	#ifdef RT_ARCH_AMD64
2788	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2789	#elif defined(RT_ARCH_ARM64)
2790	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2791	#else
2792	# error "Port me!"
2793	#endif
2794	}
2795
2796	/**
2797	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2798	*/
2799	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2800	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2801	{
2802	#ifdef RT_ARCH_AMD64
2803	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2804	#elif defined(RT_ARCH_ARM64)
2805	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2806	#else
2807	# error "Port me!"
2808	#endif
2809	}
2810
2811
2812	/**
2813	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2814	*/
2815	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2816	{
2817	#ifdef RT_ARCH_AMD64
2818	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2819	#elif defined(RT_ARCH_ARM64)
2820	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2821	#else
2822	# error "Port me!"
2823	#endif
2824	}
2825
2826	/**
2827	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2828	*/
2829	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2830	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2831	{
2832	#ifdef RT_ARCH_AMD64
2833	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2834	#elif defined(RT_ARCH_ARM64)
2835	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2836	#else
2837	# error "Port me!"
2838	#endif
2839	}
2840
2841
2842	/**
2843	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2844	*/
2845	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2846	{
2847	#ifdef RT_ARCH_AMD64
2848	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2849	#elif defined(RT_ARCH_ARM64)
2850	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2851	#else
2852	# error "Port me!"
2853	#endif
2854	}
2855
2856	/**
2857	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2858	*/
2859	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2860	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2861	{
2862	#ifdef RT_ARCH_AMD64
2863	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2864	#elif defined(RT_ARCH_ARM64)
2865	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2866	#else
2867	# error "Port me!"
2868	#endif
2869	}
2870
2871
2872	/**
2873	* Emits a JL/JNGE rel32 / B.LT imm19 to the given label.
2874	*/
2875	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2876	{
2877	#ifdef RT_ARCH_AMD64
2878	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_l);
2879	#elif defined(RT_ARCH_ARM64)
2880	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Lt);
2881	#else
2882	# error "Port me!"
2883	#endif
2884	}
2885
2886	/**
2887	* Emits a JA/JNGE rel32 / B.HI imm19 to a new label.
2888	*/
2889	DECL_INLINE_THROW(uint32_t) iemNativeEmitJlToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2890	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2891	{
2892	#ifdef RT_ARCH_AMD64
2893	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_l);
2894	#elif defined(RT_ARCH_ARM64)
2895	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Lt);
2896	#else
2897	# error "Port me!"
2898	#endif
2899	}
2900
2901
2902	/**
2903	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2904	*
2905	* @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
2906	*
2907	* Only use hardcoded jumps forward when emitting for exactly one
2908	* platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
2909	* the right target address on all platforms!
2910	*
2911	* Please also note that on x86 it is necessary pass off + 256 or higher
2912	* for @a offTarget one believe the intervening code is more than 127
2913	* bytes long.
2914	*/
2915	DECL_INLINE_THROW(uint32_t)
2916	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2917	{
2918	#ifdef RT_ARCH_AMD64
2919	/* jcc rel8 / rel32 */
2920	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2921	int32_t offDisp = (int32_t)(offTarget - (off + 2));
2922	if (offDisp < 128 && offDisp >= -128)
2923	{
2924	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2925	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
2926	}
2927	else
2928	{
2929	offDisp -= 4;
2930	pbCodeBuf[off++] = 0x0f;
2931	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2932	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
2933	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
2934	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
2935	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
2936	}
2937
2938	#elif defined(RT_ARCH_ARM64)
2939	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2940	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
2941
2942	#else
2943	# error "Port me!"
2944	#endif
2945	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2946	return off;
2947	}
2948
2949
2950	/**
2951	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2952	*
2953	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
2954	*/
2955	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
2956	{
2957	#ifdef RT_ARCH_AMD64
2958	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2959	#elif defined(RT_ARCH_ARM64)
2960	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2961	#else
2962	# error "Port me!"
2963	#endif
2964	}
2965
2966
2967	/**
2968	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2969	*
2970	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
2971	*/
2972	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
2973	{
2974	#ifdef RT_ARCH_AMD64
2975	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2976	#elif defined(RT_ARCH_ARM64)
2977	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2978	#else
2979	# error "Port me!"
2980	#endif
2981	}
2982
2983
2984	/**
2985	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2986	*
2987	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
2988	*/
2989	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
2990	{
2991	#ifdef RT_ARCH_AMD64
2992	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2993	#elif defined(RT_ARCH_ARM64)
2994	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2995	#else
2996	# error "Port me!"
2997	#endif
2998	}
2999
3000
3001	/**
3002	* Emits a JA/JNBE rel32 / B.HI imm19 with a fixed displacement.
3003	*
3004	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3005	*/
3006	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3007	{
3008	#ifdef RT_ARCH_AMD64
3009	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3010	#elif defined(RT_ARCH_ARM64)
3011	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3012	#else
3013	# error "Port me!"
3014	#endif
3015	}
3016
3017
3018	/**
3019	* Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
3020	*
3021	* See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
3022	*/
3023	DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
3024	{
3025	#ifdef RT_ARCH_AMD64
3026	/* jmp rel8 or rel32 */
3027	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3028	int32_t offDisp = offTarget - (off + 2);
3029	if (offDisp < 128 && offDisp >= -128)
3030	{
3031	pbCodeBuf[off++] = 0xeb;
3032	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3033	}
3034	else
3035	{
3036	offDisp -= 3;
3037	pbCodeBuf[off++] = 0xe9;
3038	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
3039	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
3040	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
3041	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
3042	}
3043
3044	#elif defined(RT_ARCH_ARM64)
3045	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3046	pu32CodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
3047
3048	#else
3049	# error "Port me!"
3050	#endif
3051	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3052	return off;
3053	}
3054
3055
3056	/**
3057	* Fixes up a conditional jump to a fixed label.
3058	* @see iemNativeEmitJmpToFixed, iemNativeEmitJnzToFixed,
3059	* iemNativeEmitJzToFixed, ...
3060	*/
3061	DECL_INLINE_THROW(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3062	{
3063	#ifdef RT_ARCH_AMD64
3064	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3065	uint8_t const bOpcode = pbCodeBuf[offFixup];
3066	if ((uint8_t)(bOpcode - 0x70) < (uint8_t)0x10 \|\| bOpcode == 0xeb)
3067	{
3068	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3069	AssertStmt(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2),
3070	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_FIXED_JUMP_OUT_OF_RANGE));
3071	}
3072	else
3073	{
3074	if (bOpcode != 0x0f)
3075	Assert(bOpcode == 0xe9);
3076	else
3077	{
3078	offFixup += 1;
3079	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x80) <= 0x10);
3080	}
3081	uint32_t const offRel32 = offTarget - (offFixup + 5);
3082	pbCodeBuf[offFixup + 1] = RT_BYTE1(offRel32);
3083	pbCodeBuf[offFixup + 2] = RT_BYTE2(offRel32);
3084	pbCodeBuf[offFixup + 3] = RT_BYTE3(offRel32);
3085	pbCodeBuf[offFixup + 4] = RT_BYTE4(offRel32);
3086	}
3087
3088	#elif defined(RT_ARCH_ARM64)
3089	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3090	if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
3091	{
3092	/* B.COND + BC.COND */
3093	int32_t const offDisp = offTarget - offFixup;
3094	Assert(offDisp >= -262144 && offDisp < 262144);
3095	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
3096	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
3097	}
3098	else
3099	{
3100	/* B imm26 */
3101	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
3102	int32_t const offDisp = offTarget - offFixup;
3103	Assert(offDisp >= -33554432 && offDisp < 33554432);
3104	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
3105	\| ((uint32_t)offDisp & UINT32_C(0x03ffffff));
3106	}
3107
3108	#else
3109	# error "Port me!"
3110	#endif
3111	}
3112
3113
3114	/**
3115	* Internal helper, don't call directly.
3116	*/
3117	DECL_INLINE_THROW(uint32_t)
3118	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3119	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3120	{
3121	Assert(iBitNo < 64);
3122	#ifdef RT_ARCH_AMD64
3123	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3124	if (iBitNo < 8)
3125	{
3126	/* test Eb, imm8 */
3127	if (iGprSrc >= 4)
3128	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3129	pbCodeBuf[off++] = 0xf6;
3130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3131	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3132	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3133	}
3134	else
3135	{
3136	/* bt Ev, imm8 */
3137	if (iBitNo >= 32)
3138	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3139	else if (iGprSrc >= 8)
3140	pbCodeBuf[off++] = X86_OP_REX_B;
3141	pbCodeBuf[off++] = 0x0f;
3142	pbCodeBuf[off++] = 0xba;
3143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3144	pbCodeBuf[off++] = iBitNo;
3145	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3146	}
3147
3148	#elif defined(RT_ARCH_ARM64)
3149	/* Use the TBNZ instruction here. */
3150	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3151	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3152	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3153
3154	#else
3155	# error "Port me!"
3156	#endif
3157	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3158	return off;
3159	}
3160
3161
3162	/**
3163	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3164	* @a iGprSrc.
3165	*
3166	* @note On ARM64 the range is only +/-8191 instructions.
3167	*/
3168	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3169	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3170	{
3171	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
3172	}
3173
3174
3175	/**
3176	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3177	* _set_ in @a iGprSrc.
3178	*
3179	* @note On ARM64 the range is only +/-8191 instructions.
3180	*/
3181	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3182	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3183	{
3184	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
3185	}
3186
3187
3188	/**
3189	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3190	* flags accordingly.
3191	*/
3192	DECL_INLINE_THROW(uint32_t)
3193	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3194	{
3195	Assert(fBits != 0);
3196	#ifdef RT_ARCH_AMD64
3197
3198	if (fBits >= UINT32_MAX)
3199	{
3200	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3201
3202	/* test Ev,Gv */
3203	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3204	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3205	pbCodeBuf[off++] = 0x85;
3206	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3207
3208	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3209	}
3210	else if (fBits <= UINT32_MAX)
3211	{
3212	/* test Eb, imm8 or test Ev, imm32 */
3213	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3214	if (fBits <= UINT8_MAX)
3215	{
3216	if (iGprSrc >= 4)
3217	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3218	pbCodeBuf[off++] = 0xf6;
3219	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3220	pbCodeBuf[off++] = (uint8_t)fBits;
3221	}
3222	else
3223	{
3224	if (iGprSrc >= 8)
3225	pbCodeBuf[off++] = X86_OP_REX_B;
3226	pbCodeBuf[off++] = 0xf7;
3227	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3228	pbCodeBuf[off++] = RT_BYTE1(fBits);
3229	pbCodeBuf[off++] = RT_BYTE2(fBits);
3230	pbCodeBuf[off++] = RT_BYTE3(fBits);
3231	pbCodeBuf[off++] = RT_BYTE4(fBits);
3232	}
3233	}
3234	/** @todo implement me. */
3235	else
3236	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3237
3238	#elif defined(RT_ARCH_ARM64)
3239
3240	if (false)
3241	{
3242	/** @todo figure out how to work the immr / N:imms constants. */
3243	}
3244	else
3245	{
3246	/* ands Zr, iGprSrc, iTmpReg */
3247	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3248	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3249	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3250	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3251	}
3252
3253	#else
3254	# error "Port me!"
3255	#endif
3256	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3257	return off;
3258	}
3259
3260
3261	/**
3262	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
3263	* @a iGprSrc, setting CPU flags accordingly.
3264	*/
3265	DECL_INLINE_THROW(uint32_t)
3266	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
3267	{
3268	Assert(fBits != 0);
3269
3270	#ifdef RT_ARCH_AMD64
3271	/* test Eb, imm8 */
3272	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
3273	if (iGprSrc >= 4)
3274	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3275	pbCodeBuf[off++] = 0xf6;
3276	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3277	pbCodeBuf[off++] = fBits;
3278
3279	#elif defined(RT_ARCH_ARM64)
3280
3281	/* ands xzr, src, [tmp\|#imm] */
3282	uint32_t uImmR = 0;
3283	uint32_t uImmNandS = 0;
3284	if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
3285	{
3286	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3287	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /f64Bit/);
3288	}
3289	else
3290	{
3291	/* Use temporary register for the 64-bit immediate. */
3292	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3293	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3294	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
3295	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3296	}
3297
3298	#else
3299	# error "Port me!"
3300	#endif
3301	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3302	return off;
3303	}
3304
3305
3306	/**
3307	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
3308	* are set in @a iGprSrc.
3309	*/
3310	DECL_INLINE_THROW(uint32_t)
3311	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3312	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3313	{
3314	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3315
3316	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3317	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3318
3319	return off;
3320	}
3321
3322
3323	/**
3324	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
3325	* are set in @a iGprSrc.
3326	*/
3327	DECL_INLINE_THROW(uint32_t)
3328	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3329	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3330	{
3331	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3332
3333	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3334	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3335
3336	return off;
3337	}
3338
3339
3340	/**
3341	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
3342	*
3343	* The operand size is given by @a f64Bit.
3344	*/
3345	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3346	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3347	{
3348	Assert(idxLabel < pReNative->cLabels);
3349
3350	#ifdef RT_ARCH_AMD64
3351	/* test reg32,reg32 / test reg64,reg64 */
3352	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3353	if (f64Bit)
3354	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
3355	else if (iGprSrc >= 8)
3356	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
3357	pbCodeBuf[off++] = 0x85;
3358	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3359	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3360
3361	/* jz idxLabel */
3362	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3363
3364	#elif defined(RT_ARCH_ARM64)
3365	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3366	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3367	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
3368	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3369
3370	#else
3371	# error "Port me!"
3372	#endif
3373	return off;
3374	}
3375
3376
3377	/**
3378	* Emits code that jumps to a new label if @a iGprSrc is zero.
3379	*
3380	* The operand size is given by @a f64Bit.
3381	*/
3382	DECL_INLINE_THROW(uint32_t)
3383	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3384	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3385	{
3386	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3387	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3388	}
3389
3390
3391	/**
3392	* Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
3393	*
3394	* The operand size is given by @a f64Bit.
3395	*/
3396	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3397	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3398	{
3399	Assert(idxLabel < pReNative->cLabels);
3400
3401	#ifdef RT_ARCH_AMD64
3402	/* test reg32,reg32 / test reg64,reg64 */
3403	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3404	if (f64Bit)
3405	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
3406	else if (iGprSrc >= 8)
3407	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
3408	pbCodeBuf[off++] = 0x85;
3409	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3410	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3411
3412	/* jnz idxLabel */
3413	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3414
3415	#elif defined(RT_ARCH_ARM64)
3416	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3417	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3418	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(true /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
3419	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3420
3421	#else
3422	# error "Port me!"
3423	#endif
3424	return off;
3425	}
3426
3427
3428	/**
3429	* Emits code that jumps to a new label if @a iGprSrc is not zero.
3430	*
3431	* The operand size is given by @a f64Bit.
3432	*/
3433	DECL_INLINE_THROW(uint32_t)
3434	iemNativeEmitTestIfGprIsNotZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3435	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3436	{
3437	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3438	return iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3439	}
3440
3441
3442	/**
3443	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3444	* differs.
3445	*/
3446	DECL_INLINE_THROW(uint32_t)
3447	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3448	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3449	{
3450	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3451	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3452	return off;
3453	}
3454
3455
3456	/**
3457	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3458	*/
3459	DECL_INLINE_THROW(uint32_t)
3460	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3461	uint8_t iGprLeft, uint8_t iGprRight,
3462	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3463	{
3464	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3465	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3466	}
3467
3468
3469	/**
3470	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3471	*/
3472	DECL_INLINE_THROW(uint32_t)
3473	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3474	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3475	{
3476	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3477	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3478	return off;
3479	}
3480
3481
3482	/**
3483	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3484	*/
3485	DECL_INLINE_THROW(uint32_t)
3486	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3487	uint8_t iGprSrc, uint64_t uImm,
3488	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3489	{
3490	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3491	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3492	}
3493
3494
3495	/**
3496	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3497	* @a uImm.
3498	*/
3499	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3500	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3501	{
3502	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3503	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3504	return off;
3505	}
3506
3507
3508	/**
3509	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3510	* @a uImm.
3511	*/
3512	DECL_INLINE_THROW(uint32_t)
3513	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3514	uint8_t iGprSrc, uint32_t uImm,
3515	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3516	{
3517	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3518	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3519	}
3520
3521
3522	/*********************************************************************************************************************************
3523	* Calls. *
3524	*********************************************************************************************************************************/
3525
3526	/**
3527	* Emits a call to a 64-bit address.
3528	*/
3529	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3530	{
3531	#ifdef RT_ARCH_AMD64
3532	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3533
3534	/* call rax */
3535	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3536	pbCodeBuf[off++] = 0xff;
3537	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3538
3539	#elif defined(RT_ARCH_ARM64)
3540	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3541
3542	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3543	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3544
3545	#else
3546	# error "port me"
3547	#endif
3548	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3549	return off;
3550	}
3551
3552
3553	/**
3554	* Emits code to load a stack variable into an argument GPR.
3555	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3556	*/
3557	DECL_FORCE_INLINE_THROW(uint32_t)
3558	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3559	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
3560	{
3561	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3562	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3563	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3564
3565	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3566	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3567	{
3568	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK) \|\| fVarAllowInVolatileReg);
3569	RT_NOREF(fVarAllowInVolatileReg);
3570	if (!offAddend)
3571	{
3572	if (idxRegArg != idxRegVar)
3573	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
3574	}
3575	else
3576	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
3577	}
3578	else
3579	{
3580	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
3581	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
3582	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
3583	if (offAddend)
3584	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
3585	}
3586	return off;
3587	}
3588
3589
3590	/**
3591	* Emits code to load a stack or immediate variable value into an argument GPR,
3592	* optional with a addend.
3593	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3594	*/
3595	DECL_FORCE_INLINE_THROW(uint32_t)
3596	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3597	int32_t offAddend = 0, bool fVarAllowInVolatileReg = false)
3598	{
3599	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3600	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
3601	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
3602	else
3603	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend, fVarAllowInVolatileReg);
3604	return off;
3605	}
3606
3607
3608	/**
3609	* Emits code to load the variable address into an argument GRP.
3610	*
3611	* This only works for uninitialized and stack variables.
3612	*/
3613	DECL_FORCE_INLINE_THROW(uint32_t)
3614	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3615	bool fFlushShadows)
3616	{
3617	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3618	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
3619	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3620	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3621
3622	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
3623	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
3624
3625	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3626	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3627	{
3628	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
3629	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
3630	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3631	}
3632	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
3633	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3634
3635	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
3636	}
3637
3638
3639	/** @} */
3640
3641	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
3642

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102700

Download in other formats: