IEMN8veRecompilerEmit.h@ 102436

Last change on this file since 102436 was 102436, checked in by vboxsync, 15 months ago
VMM/IEM: U8 mem map MCs. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 104.2 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102436 2023-12-02 22:09:36Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
88	pbCodeBuf[off++] = 0xcc;
89	RT_NOREF(uInfo);
90
91	#elif defined(RT_ARCH_ARM64)
92	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
93	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
94
95	#else
96	# error "error"
97	#endif
98	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
99	return off;
100	}
101
102
103	/*********************************************************************************************************************************
104	* Loads, Stores and Related Stuff. *
105	*********************************************************************************************************************************/
106
107	/**
108	* Emits setting a GPR to zero.
109	*/
110	DECL_INLINE_THROW(uint32_t)
111	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
112	{
113	#ifdef RT_ARCH_AMD64
114	/* xor gpr32, gpr32 */
115	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
116	if (iGpr >= 8)
117	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
118	pbCodeBuf[off++] = 0x33;
119	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
120
121	#elif defined(RT_ARCH_ARM64)
122	/* mov gpr, #0x0 */
123	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
124	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
125
126	#else
127	# error "port me"
128	#endif
129	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
130	return off;
131	}
132
133
134	/**
135	* Emits loading a constant into a 64-bit GPR
136	*/
137	DECL_INLINE_THROW(uint32_t)
138	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
139	{
140	if (!uImm64)
141	return iemNativeEmitGprZero(pReNative, off, iGpr);
142
143	#ifdef RT_ARCH_AMD64
144	if (uImm64 <= UINT32_MAX)
145	{
146	/* mov gpr, imm32 */
147	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
148	if (iGpr >= 8)
149	pbCodeBuf[off++] = X86_OP_REX_B;
150	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
151	pbCodeBuf[off++] = RT_BYTE1(uImm64);
152	pbCodeBuf[off++] = RT_BYTE2(uImm64);
153	pbCodeBuf[off++] = RT_BYTE3(uImm64);
154	pbCodeBuf[off++] = RT_BYTE4(uImm64);
155	}
156	else
157	{
158	/* mov gpr, imm64 */
159	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
160	if (iGpr < 8)
161	pbCodeBuf[off++] = X86_OP_REX_W;
162	else
163	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
164	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
165	pbCodeBuf[off++] = RT_BYTE1(uImm64);
166	pbCodeBuf[off++] = RT_BYTE2(uImm64);
167	pbCodeBuf[off++] = RT_BYTE3(uImm64);
168	pbCodeBuf[off++] = RT_BYTE4(uImm64);
169	pbCodeBuf[off++] = RT_BYTE5(uImm64);
170	pbCodeBuf[off++] = RT_BYTE6(uImm64);
171	pbCodeBuf[off++] = RT_BYTE7(uImm64);
172	pbCodeBuf[off++] = RT_BYTE8(uImm64);
173	}
174
175	#elif defined(RT_ARCH_ARM64)
176	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
177
178	/*
179	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
180	* supply remaining bits using 'movk grp, imm16, lsl #x'.
181	*
182	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
183	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
184	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
185	* after the first non-zero immediate component so we switch to movk for
186	* the remainder.
187	*/
188	uint32_t fMovK = 0;
189	/* mov gpr, imm16 */
190	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
191	if (uImmPart)
192	{
193	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
194	fMovK \|= RT_BIT_32(29);
195	}
196	/* mov[k] gpr, imm16, lsl #16 */
197	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
198	if (uImmPart)
199	{
200	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
201	fMovK \|= RT_BIT_32(29);
202	}
203	/* mov[k] gpr, imm16, lsl #32 */
204	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
205	if (uImmPart)
206	{
207	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
208	fMovK \|= RT_BIT_32(29);
209	}
210	/* mov[k] gpr, imm16, lsl #48 */
211	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
212	if (uImmPart)
213	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
214
215	/** @todo there is an inverted mask variant we might want to explore if it
216	* reduces the number of instructions... */
217	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
218	* clang 12.x does that, only to use the 'x' version for the
219	* addressing in the following ldr). */
220
221	#else
222	# error "port me"
223	#endif
224	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
225	return off;
226	}
227
228
229	/**
230	* Emits loading a constant into a 8-bit GPR
231	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
232	* only the ARM64 version does that.
233	*/
234	DECL_INLINE_THROW(uint32_t)
235	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
236	{
237	#ifdef RT_ARCH_AMD64
238	/* mov gpr, imm8 */
239	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
240	if (iGpr >= 8)
241	pbCodeBuf[off++] = X86_OP_REX_B;
242	else if (iGpr >= 4)
243	pbCodeBuf[off++] = X86_OP_REX;
244	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
245	pbCodeBuf[off++] = RT_BYTE1(uImm8);
246
247	#elif defined(RT_ARCH_ARM64)
248	/* movz gpr, imm16, lsl #0 */
249	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
250	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
251
252	#else
253	# error "port me"
254	#endif
255	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
256	return off;
257	}
258
259
260	#ifdef RT_ARCH_AMD64
261	/**
262	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
263	*/
264	DECL_FORCE_INLINE(uint32_t)
265	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
266	{
267	if (offVCpu < 128)
268	{
269	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
270	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
271	}
272	else
273	{
274	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
275	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
276	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
277	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
278	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
279	}
280	return off;
281	}
282	#elif defined(RT_ARCH_ARM64)
283	/**
284	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
285	*/
286	DECL_FORCE_INLINE_THROW(uint32_t)
287	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
288	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
289	{
290	/*
291	* There are a couple of ldr variants that takes an immediate offset, so
292	* try use those if we can, otherwise we have to use the temporary register
293	* help with the addressing.
294	*/
295	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
296	{
297	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
298	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
299	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
300	}
301	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
302	{
303	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
304	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
305	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
306	}
307	else
308	{
309	/* The offset is too large, so we must load it into a register and use
310	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
311	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
312	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
313
314	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
315	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
316	IEMNATIVE_REG_FIXED_TMP0);
317	}
318	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
319	return off;
320	}
321	#endif
322
323
324	/**
325	* Emits a 64-bit GPR load of a VCpu value.
326	*/
327	DECL_INLINE_THROW(uint32_t)
328	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
329	{
330	#ifdef RT_ARCH_AMD64
331	/* mov reg64, mem64 */
332	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
333	if (iGpr < 8)
334	pbCodeBuf[off++] = X86_OP_REX_W;
335	else
336	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
337	pbCodeBuf[off++] = 0x8b;
338	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
339	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
340
341	#elif defined(RT_ARCH_ARM64)
342	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
343
344	#else
345	# error "port me"
346	#endif
347	return off;
348	}
349
350
351	/**
352	* Emits a 32-bit GPR load of a VCpu value.
353	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
354	*/
355	DECL_INLINE_THROW(uint32_t)
356	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
357	{
358	#ifdef RT_ARCH_AMD64
359	/* mov reg32, mem32 */
360	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
361	if (iGpr >= 8)
362	pbCodeBuf[off++] = X86_OP_REX_R;
363	pbCodeBuf[off++] = 0x8b;
364	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
365	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
366
367	#elif defined(RT_ARCH_ARM64)
368	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
369
370	#else
371	# error "port me"
372	#endif
373	return off;
374	}
375
376
377	/**
378	* Emits a 16-bit GPR load of a VCpu value.
379	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
380	*/
381	DECL_INLINE_THROW(uint32_t)
382	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
383	{
384	#ifdef RT_ARCH_AMD64
385	/* movzx reg32, mem16 */
386	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
387	if (iGpr >= 8)
388	pbCodeBuf[off++] = X86_OP_REX_R;
389	pbCodeBuf[off++] = 0x0f;
390	pbCodeBuf[off++] = 0xb7;
391	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
392	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
393
394	#elif defined(RT_ARCH_ARM64)
395	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
396
397	#else
398	# error "port me"
399	#endif
400	return off;
401	}
402
403
404	/**
405	* Emits a 8-bit GPR load of a VCpu value.
406	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
407	*/
408	DECL_INLINE_THROW(uint32_t)
409	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
410	{
411	#ifdef RT_ARCH_AMD64
412	/* movzx reg32, mem8 */
413	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
414	if (iGpr >= 8)
415	pbCodeBuf[off++] = X86_OP_REX_R;
416	pbCodeBuf[off++] = 0x0f;
417	pbCodeBuf[off++] = 0xb6;
418	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
419	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
420
421	#elif defined(RT_ARCH_ARM64)
422	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
423
424	#else
425	# error "port me"
426	#endif
427	return off;
428	}
429
430
431	/**
432	* Emits a store of a GPR value to a 64-bit VCpu field.
433	*/
434	DECL_INLINE_THROW(uint32_t)
435	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
436	{
437	#ifdef RT_ARCH_AMD64
438	/* mov mem64, reg64 */
439	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
440	if (iGpr < 8)
441	pbCodeBuf[off++] = X86_OP_REX_W;
442	else
443	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
444	pbCodeBuf[off++] = 0x89;
445	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
446	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
447
448	#elif defined(RT_ARCH_ARM64)
449	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
450
451	#else
452	# error "port me"
453	#endif
454	return off;
455	}
456
457
458	/**
459	* Emits a store of a GPR value to a 32-bit VCpu field.
460	*/
461	DECL_INLINE_THROW(uint32_t)
462	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
463	{
464	#ifdef RT_ARCH_AMD64
465	/* mov mem32, reg32 */
466	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
467	if (iGpr >= 8)
468	pbCodeBuf[off++] = X86_OP_REX_R;
469	pbCodeBuf[off++] = 0x89;
470	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472
473	#elif defined(RT_ARCH_ARM64)
474	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
475
476	#else
477	# error "port me"
478	#endif
479	return off;
480	}
481
482
483	/**
484	* Emits a store of a GPR value to a 16-bit VCpu field.
485	*/
486	DECL_INLINE_THROW(uint32_t)
487	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
488	{
489	#ifdef RT_ARCH_AMD64
490	/* mov mem16, reg16 */
491	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
492	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
493	if (iGpr >= 8)
494	pbCodeBuf[off++] = X86_OP_REX_R;
495	pbCodeBuf[off++] = 0x89;
496	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
497	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
498
499	#elif defined(RT_ARCH_ARM64)
500	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
501
502	#else
503	# error "port me"
504	#endif
505	return off;
506	}
507
508
509	/**
510	* Emits a store of a GPR value to a 8-bit VCpu field.
511	*/
512	DECL_INLINE_THROW(uint32_t)
513	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
514	{
515	#ifdef RT_ARCH_AMD64
516	/* mov mem8, reg8 */
517	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
518	if (iGpr >= 8)
519	pbCodeBuf[off++] = X86_OP_REX_R;
520	pbCodeBuf[off++] = 0x88;
521	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
522	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
523
524	#elif defined(RT_ARCH_ARM64)
525	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
526
527	#else
528	# error "port me"
529	#endif
530	return off;
531	}
532
533
534	/**
535	* Emits a store of an immediate value to a 8-bit VCpu field.
536	*/
537	DECL_INLINE_THROW(uint32_t)
538	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
539	{
540	#ifdef RT_ARCH_AMD64
541	/* mov mem8, imm8 */
542	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
543	pbCodeBuf[off++] = 0xc6;
544	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
545	pbCodeBuf[off++] = bImm;
546	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
547
548	#elif defined(RT_ARCH_ARM64)
549	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
550	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
551	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
552	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
553
554	#else
555	# error "port me"
556	#endif
557	return off;
558	}
559
560
561	/**
562	* Emits a load effective address to a GRP of a VCpu field.
563	*/
564	DECL_INLINE_THROW(uint32_t)
565	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
566	{
567	#ifdef RT_ARCH_AMD64
568	/* lea gprdst, [rbx + offDisp] */
569	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
570	if (iGprDst < 8)
571	pbCodeBuf[off++] = X86_OP_REX_W;
572	else
573	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
574	pbCodeBuf[off++] = 0x8d;
575	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
576
577	#elif defined(RT_ARCH_ARM64)
578	if (offVCpu < (unsigned)_4K)
579	{
580	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
581	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
582	}
583	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
584	{
585	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
586	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
587	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
588	}
589	else
590	{
591	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
592	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
593	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
594	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
595	}
596
597	#else
598	# error "port me"
599	#endif
600	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
601	return off;
602	}
603
604
605	/**
606	* Emits a gprdst = gprsrc load.
607	*/
608	DECL_INLINE_THROW(uint32_t)
609	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
610	{
611	#ifdef RT_ARCH_AMD64
612	/* mov gprdst, gprsrc */
613	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
614	if ((iGprDst \| iGprSrc) >= 8)
615	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
616	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
617	: X86_OP_REX_W \| X86_OP_REX_R;
618	else
619	pbCodeBuf[off++] = X86_OP_REX_W;
620	pbCodeBuf[off++] = 0x8b;
621	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
622
623	#elif defined(RT_ARCH_ARM64)
624	/* mov dst, src; alias for: orr dst, xzr, src */
625	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
626	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
627
628	#else
629	# error "port me"
630	#endif
631	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
632	return off;
633	}
634
635
636	/**
637	* Emits a gprdst = gprsrc[31:0] load.
638	* @note Bits 63 thru 32 are cleared.
639	*/
640	DECL_INLINE_THROW(uint32_t)
641	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
642	{
643	#ifdef RT_ARCH_AMD64
644	/* mov gprdst, gprsrc */
645	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
646	if ((iGprDst \| iGprSrc) >= 8)
647	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
648	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
649	: X86_OP_REX_R;
650	pbCodeBuf[off++] = 0x8b;
651	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
652
653	#elif defined(RT_ARCH_ARM64)
654	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
655	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
656	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
657
658	#else
659	# error "port me"
660	#endif
661	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
662	return off;
663	}
664
665
666	/**
667	* Emits a gprdst = gprsrc[15:0] load.
668	* @note Bits 63 thru 15 are cleared.
669	*/
670	DECL_INLINE_THROW(uint32_t)
671	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
672	{
673	#ifdef RT_ARCH_AMD64
674	/* movzx Gv,Ew */
675	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
676	if ((iGprDst \| iGprSrc) >= 8)
677	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
678	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
679	: X86_OP_REX_R;
680	pbCodeBuf[off++] = 0x0f;
681	pbCodeBuf[off++] = 0xb7;
682	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
683
684	#elif defined(RT_ARCH_ARM64)
685	/* and gprdst, gprsrc, #0xffff */
686	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
687	# if 1
688	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
689	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
690	# else
691	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
692	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
693	# endif
694
695	#else
696	# error "port me"
697	#endif
698	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
699	return off;
700	}
701
702
703	/**
704	* Emits a gprdst = gprsrc[7:0] load.
705	* @note Bits 63 thru 8 are cleared.
706	*/
707	DECL_INLINE_THROW(uint32_t)
708	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
709	{
710	#ifdef RT_ARCH_AMD64
711	/* movzx Gv,Eb */
712	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
713	if (iGprDst >= 8 \|\| iGprSrc >= 8)
714	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
715	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
716	: X86_OP_REX_R;
717	else if (iGprSrc >= 4)
718	pbCodeBuf[off++] = X86_OP_REX;
719	pbCodeBuf[off++] = 0x0f;
720	pbCodeBuf[off++] = 0xb6;
721	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
722
723	#elif defined(RT_ARCH_ARM64)
724	/* and gprdst, gprsrc, #0xff */
725	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
726	# if 1
727	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
728	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
729	# else
730	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
731	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
732	# endif
733
734	#else
735	# error "port me"
736	#endif
737	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
738	return off;
739	}
740
741
742	/**
743	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
744	* @note Bits 63 thru 8 are cleared.
745	*/
746	DECL_INLINE_THROW(uint32_t)
747	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
748	{
749	#ifdef RT_ARCH_AMD64
750	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
751
752	/* movzx Gv,Ew */
753	if ((iGprDst \| iGprSrc) >= 8)
754	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
755	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
756	: X86_OP_REX_R;
757	pbCodeBuf[off++] = 0x0f;
758	pbCodeBuf[off++] = 0xb7;
759	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
760
761	/* shr Ev,8 */
762	if (iGprDst >= 8)
763	pbCodeBuf[off++] = X86_OP_REX_B;
764	pbCodeBuf[off++] = 0xc1;
765	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
766	pbCodeBuf[off++] = 8;
767
768	#elif defined(RT_ARCH_ARM64)
769	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
770	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
771	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
772
773	#else
774	# error "port me"
775	#endif
776	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
777	return off;
778	}
779
780
781	/**
782	* Emits a gprdst = gprsrc + addend load.
783	*/
784	DECL_INLINE_THROW(uint32_t)
785	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
786	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
787	{
788	Assert(iAddend != 0);
789
790	#ifdef RT_ARCH_AMD64
791	/* lea gprdst, [gprsrc + iAddend] */
792	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
793	if ((iGprDst \| iGprSrc) >= 8)
794	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
795	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
796	: X86_OP_REX_W \| X86_OP_REX_R;
797	else
798	pbCodeBuf[off++] = X86_OP_REX_W;
799	pbCodeBuf[off++] = 0x8d;
800	if (iAddend >= -128 && iAddend < 128)
801	{
802	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprDst & 7, iGprSrc & 7);
803	pbCodeBuf[off++] = (int8_t)iAddend;
804	}
805	else
806	{
807	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprDst & 7, iGprSrc & 7);
808	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
809	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
810	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
811	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
812	}
813
814	#elif defined(RT_ARCH_ARM64)
815	if ((uint32_t)iAddend < 4096)
816	{
817	/* add dst, src, uimm12 */
818	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
819	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
820	}
821	else if ((uint32_t)-iAddend < 4096)
822	{
823	/* sub dst, src, uimm12 */
824	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
825	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
826	}
827	else
828	{
829	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
830	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
831	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
832	}
833
834	#else
835	# error "port me"
836	#endif
837	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
838	return off;
839	}
840
841
842	#ifdef RT_ARCH_AMD64
843	/**
844	* Common bit of iemNativeEmitLoadGprByBp and friends.
845	*/
846	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
847	PIEMRECOMPILERSTATE pReNativeAssert)
848	{
849	if (offDisp < 128 && offDisp >= -128)
850	{
851	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
852	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
853	}
854	else
855	{
856	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
857	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
858	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
859	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
860	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
861	}
862	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
863	return off;
864	}
865	#elif defined(RT_ARCH_ARM64)
866	/**
867	* Common bit of iemNativeEmitLoadGprByBp and friends.
868	*/
869	DECL_FORCE_INLINE_THROW(uint32_t)
870	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
871	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
872	{
873	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
874	{
875	/* str w/ unsigned imm12 (scaled) */
876	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
877	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
878	}
879	else if (offDisp >= -256 && offDisp <= 256)
880	{
881	/* stur w/ signed imm9 (unscaled) */
882	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
883	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
884	}
885	else
886	{
887	/* Use temporary indexing register. */
888	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
889	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
890	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
891	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
892	}
893	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
894	return off;
895	}
896	#endif
897
898
899	/**
900	* Emits a 64-bit GRP load instruction with an BP relative source address.
901	*/
902	DECL_INLINE_THROW(uint32_t)
903	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
904	{
905	#ifdef RT_ARCH_AMD64
906	/* mov gprdst, qword [rbp + offDisp] */
907	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
908	if (iGprDst < 8)
909	pbCodeBuf[off++] = X86_OP_REX_W;
910	else
911	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
912	pbCodeBuf[off++] = 0x8b;
913	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
914
915	#elif defined(RT_ARCH_ARM64)
916	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
917
918	#else
919	# error "port me"
920	#endif
921	}
922
923
924	/**
925	* Emits a 32-bit GRP load instruction with an BP relative source address.
926	* @note Bits 63 thru 32 of the GPR will be cleared.
927	*/
928	DECL_INLINE_THROW(uint32_t)
929	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
930	{
931	#ifdef RT_ARCH_AMD64
932	/* mov gprdst, dword [rbp + offDisp] */
933	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
934	if (iGprDst >= 8)
935	pbCodeBuf[off++] = X86_OP_REX_R;
936	pbCodeBuf[off++] = 0x8b;
937	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
938
939	#elif defined(RT_ARCH_ARM64)
940	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
941
942	#else
943	# error "port me"
944	#endif
945	}
946
947
948	/**
949	* Emits a 16-bit GRP load instruction with an BP relative source address.
950	* @note Bits 63 thru 16 of the GPR will be cleared.
951	*/
952	DECL_INLINE_THROW(uint32_t)
953	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
954	{
955	#ifdef RT_ARCH_AMD64
956	/* movzx gprdst, word [rbp + offDisp] */
957	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
958	if (iGprDst >= 8)
959	pbCodeBuf[off++] = X86_OP_REX_R;
960	pbCodeBuf[off++] = 0xb7;
961	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
962
963	#elif defined(RT_ARCH_ARM64)
964	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
965
966	#else
967	# error "port me"
968	#endif
969	}
970
971
972	/**
973	* Emits a 8-bit GRP load instruction with an BP relative source address.
974	* @note Bits 63 thru 8 of the GPR will be cleared.
975	*/
976	DECL_INLINE_THROW(uint32_t)
977	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
978	{
979	#ifdef RT_ARCH_AMD64
980	/* movzx gprdst, byte [rbp + offDisp] */
981	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
982	if (iGprDst >= 8)
983	pbCodeBuf[off++] = X86_OP_REX_R;
984	pbCodeBuf[off++] = 0xb6;
985	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
986
987	#elif defined(RT_ARCH_ARM64)
988	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
989
990	#else
991	# error "port me"
992	#endif
993	}
994
995
996	/**
997	* Emits a load effective address to a GRP with an BP relative source address.
998	*/
999	DECL_INLINE_THROW(uint32_t)
1000	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1001	{
1002	#ifdef RT_ARCH_AMD64
1003	/* lea gprdst, [rbp + offDisp] */
1004	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1005	if (iGprDst < 8)
1006	pbCodeBuf[off++] = X86_OP_REX_W;
1007	else
1008	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1009	pbCodeBuf[off++] = 0x8d;
1010	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1011
1012	#elif defined(RT_ARCH_ARM64)
1013	if ((uint32_t)offDisp < (unsigned)_4K)
1014	{
1015	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1016	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1017	}
1018	else if ((uint32_t)-offDisp < (unsigned)_4K)
1019	{
1020	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1021	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1022	}
1023	else
1024	{
1025	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1026	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1027	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1028	if (offDisp >= 0)
1029	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1030	else
1031	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1032	}
1033
1034	#else
1035	# error "port me"
1036	#endif
1037
1038	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1039	return off;
1040	}
1041
1042
1043	/**
1044	* Emits a 64-bit GPR store with an BP relative destination address.
1045	*
1046	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1047	*/
1048	DECL_INLINE_THROW(uint32_t)
1049	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1050	{
1051	#ifdef RT_ARCH_AMD64
1052	/* mov qword [rbp + offDisp], gprdst */
1053	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1054	if (iGprSrc < 8)
1055	pbCodeBuf[off++] = X86_OP_REX_W;
1056	else
1057	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1058	pbCodeBuf[off++] = 0x89;
1059	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1060
1061	#elif defined(RT_ARCH_ARM64)
1062	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1063	{
1064	/* str w/ unsigned imm12 (scaled) */
1065	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1066	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1067	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1068	}
1069	else if (offDisp >= -256 && offDisp <= 256)
1070	{
1071	/* stur w/ signed imm9 (unscaled) */
1072	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1073	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1074	}
1075	else if ((uint32_t)-offDisp < (unsigned)_4K)
1076	{
1077	/* Use temporary indexing register w/ sub uimm12. */
1078	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1079	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1080	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1081	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1082	}
1083	else
1084	{
1085	/* Use temporary indexing register. */
1086	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1087	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1088	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1089	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1090	}
1091	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1092	return off;
1093
1094	#else
1095	# error "Port me!"
1096	#endif
1097	}
1098
1099
1100	/**
1101	* Emits a 64-bit immediate store with an BP relative destination address.
1102	*
1103	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1104	*/
1105	DECL_INLINE_THROW(uint32_t)
1106	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1107	{
1108	#ifdef RT_ARCH_AMD64
1109	if ((int64_t)uImm64 == (int32_t)uImm64)
1110	{
1111	/* mov qword [rbp + offDisp], imm32 - sign extended */
1112	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1113	pbCodeBuf[off++] = X86_OP_REX_W;
1114	pbCodeBuf[off++] = 0xc7;
1115	if (offDisp < 128 && offDisp >= -128)
1116	{
1117	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1118	pbCodeBuf[off++] = (uint8_t)offDisp;
1119	}
1120	else
1121	{
1122	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1123	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1124	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1125	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1126	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1127	}
1128	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1129	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1130	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1131	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1132	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1133	return off;
1134	}
1135	#endif
1136
1137	/* Load tmp0, imm64; Store tmp to bp+disp. */
1138	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1139	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1140	}
1141
1142
1143	#ifdef RT_ARCH_AMD64
1144	/**
1145	* Common bit of iemNativeEmitLoadGprByGpr and friends.
1146	*/
1147	DECL_FORCE_INLINE(uint32_t)
1148	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1149	{
1150	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1151	{
1152	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1153	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1154	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1155	}
1156	else if (offDisp == (int8_t)offDisp)
1157	{
1158	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1159	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1160	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1161	pbCodeBuf[off++] = (uint8_t)offDisp;
1162	}
1163	else
1164	{
1165	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1166	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1167	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1168	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1169	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1170	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1171	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1172	}
1173	return off;
1174	}
1175	#elif defined(RT_ARCH_ARM64)
1176	/**
1177	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1178	*/
1179	DECL_FORCE_INLINE_THROW(uint32_t)
1180	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1181	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1182	{
1183	/*
1184	* There are a couple of ldr variants that takes an immediate offset, so
1185	* try use those if we can, otherwise we have to use the temporary register
1186	* help with the addressing.
1187	*/
1188	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1189	{
1190	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1191	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1192	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1193	}
1194	else
1195	{
1196	/* The offset is too large, so we must load it into a register and use
1197	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1198	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1199	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1200
1201	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1202	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1203
1204	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1205	}
1206	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1207	return off;
1208	}
1209	#endif
1210
1211
1212	/**
1213	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1214	*/
1215	DECL_INLINE_THROW(uint32_t)
1216	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1217	{
1218	#ifdef RT_ARCH_AMD64
1219	/* mov reg64, mem64 */
1220	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1221	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1222	pbCodeBuf[off++] = 0x8b;
1223	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1224	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1225
1226	#elif defined(RT_ARCH_ARM64)
1227	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1228
1229	#else
1230	# error "port me"
1231	#endif
1232	return off;
1233	}
1234
1235
1236	/**
1237	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1238	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1239	*/
1240	DECL_INLINE_THROW(uint32_t)
1241	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1242	{
1243	#ifdef RT_ARCH_AMD64
1244	/* mov reg32, mem32 */
1245	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1246	if (iGprDst >= 8 \|\| iGprBase >= 8)
1247	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1248	pbCodeBuf[off++] = 0x8b;
1249	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1250	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1251
1252	#elif defined(RT_ARCH_ARM64)
1253	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1254
1255	#else
1256	# error "port me"
1257	#endif
1258	return off;
1259	}
1260
1261
1262	/*********************************************************************************************************************************
1263	* Subtraction and Additions *
1264	*********************************************************************************************************************************/
1265
1266
1267	#ifdef RT_ARCH_AMD64
1268	/**
1269	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1270	*/
1271	DECL_INLINE_THROW(uint32_t)
1272	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1273	{
1274	/* sub gprdst, imm8/imm32 */
1275	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1276	if (iGprDst < 8)
1277	pbCodeBuf[off++] = X86_OP_REX_W;
1278	else
1279	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1280	if (iSubtrahend < 128 && iSubtrahend >= -128)
1281	{
1282	pbCodeBuf[off++] = 0x83;
1283	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1284	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1285	}
1286	else
1287	{
1288	pbCodeBuf[off++] = 0x81;
1289	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1290	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1291	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1292	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1293	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1294	}
1295	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1296	return off;
1297	}
1298	#endif
1299
1300
1301	/**
1302	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1303	* @note The AMD64 version sets flags.
1304	*/
1305	DECL_INLINE_THROW(uint32_t)
1306	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1307	{
1308	#if defined(RT_ARCH_AMD64)
1309	/* add Gv,Ev */
1310	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1311	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1312	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1313	pbCodeBuf[off++] = 0x04;
1314	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1315
1316	#elif defined(RT_ARCH_ARM64)
1317	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1318	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1319
1320	#else
1321	# error "Port me"
1322	#endif
1323	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1324	return off;
1325	}
1326
1327
1328	/**
1329	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1330	*/
1331	DECL_INLINE_THROW(uint32_t)
1332	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1333	{
1334	#if defined(RT_ARCH_AMD64)
1335	/* add or inc */
1336	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1337	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1338	if (iImm8 != 1)
1339	{
1340	pbCodeBuf[off++] = 0x83;
1341	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1342	pbCodeBuf[off++] = (uint8_t)iImm8;
1343	}
1344	else
1345	{
1346	pbCodeBuf[off++] = 0xff;
1347	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1348	}
1349
1350	#elif defined(RT_ARCH_ARM64)
1351	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1352	if (iImm8 >= 0)
1353	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1354	else
1355	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1356
1357	#else
1358	# error "Port me"
1359	#endif
1360	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1361	return off;
1362	}
1363
1364
1365	/**
1366	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1367	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1368	*/
1369	DECL_INLINE_THROW(uint32_t)
1370	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1371	{
1372	#if defined(RT_ARCH_AMD64)
1373	/* add or inc */
1374	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1375	if (iGprDst >= 8)
1376	pbCodeBuf[off++] = X86_OP_REX_B;
1377	if (iImm8 != 1)
1378	{
1379	pbCodeBuf[off++] = 0x83;
1380	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1381	pbCodeBuf[off++] = (uint8_t)iImm8;
1382	}
1383	else
1384	{
1385	pbCodeBuf[off++] = 0xff;
1386	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1387	}
1388
1389	#elif defined(RT_ARCH_ARM64)
1390	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1391	if (iImm8 >= 0)
1392	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1393	else
1394	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1395
1396	#else
1397	# error "Port me"
1398	#endif
1399	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1400	return off;
1401	}
1402
1403
1404	/**
1405	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1406	*/
1407	DECL_INLINE_THROW(uint32_t)
1408	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1409	{
1410	#if defined(RT_ARCH_AMD64)
1411	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1412	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1413
1414	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1415	{
1416	/* add grp, imm32 */
1417	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1418	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1419	pbCodeBuf[off++] = 0x81;
1420	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1421	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1422	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1423	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1424	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1425	}
1426	else
1427	{
1428	/* Best to use a temporary register to deal with this in the simplest way: */
1429	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1430
1431	/* add dst, tmpreg */
1432	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1433	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1434	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1435	pbCodeBuf[off++] = 0x03;
1436	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1437
1438	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1439	}
1440
1441	#elif defined(RT_ARCH_ARM64)
1442	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1443	{
1444	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1445	if (iAddend >= 0)
1446	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1447	else
1448	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1449	}
1450	else
1451	{
1452	/* Use temporary register for the immediate. */
1453	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1454
1455	/* add gprdst, gprdst, tmpreg */
1456	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1457	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1458
1459	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1460	}
1461
1462	#else
1463	# error "Port me"
1464	#endif
1465	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1466	return off;
1467	}
1468
1469
1470	/**
1471	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1472	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1473	*/
1474	DECL_INLINE_THROW(uint32_t)
1475	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1476	{
1477	#if defined(RT_ARCH_AMD64)
1478	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1479	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1480
1481	/* add grp, imm32 */
1482	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1483	if (iGprDst >= 8)
1484	pbCodeBuf[off++] = X86_OP_REX_B;
1485	pbCodeBuf[off++] = 0x81;
1486	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1487	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1488	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1489	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1490	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1491
1492	#elif defined(RT_ARCH_ARM64)
1493	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1494	{
1495	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1496	if (iAddend >= 0)
1497	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1498	else
1499	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1500	}
1501	else
1502	{
1503	/* Use temporary register for the immediate. */
1504	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1505
1506	/* add gprdst, gprdst, tmpreg */
1507	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1508	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1509
1510	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1511	}
1512
1513	#else
1514	# error "Port me"
1515	#endif
1516	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1517	return off;
1518	}
1519
1520
1521
1522	/*********************************************************************************************************************************
1523	* Bit Operations *
1524	*********************************************************************************************************************************/
1525
1526	/**
1527	* Emits code for clearing bits 16 thru 63 in the GPR.
1528	*/
1529	DECL_INLINE_THROW(uint32_t)
1530	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1531	{
1532	#if defined(RT_ARCH_AMD64)
1533	/* movzx Gv,Ew */
1534	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1535	if (iGprDst >= 8)
1536	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1537	pbCodeBuf[off++] = 0x0f;
1538	pbCodeBuf[off++] = 0xb7;
1539	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1540
1541	#elif defined(RT_ARCH_ARM64)
1542	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1543	# if 1
1544	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1545	# else
1546	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1547	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1548	# endif
1549	#else
1550	# error "Port me"
1551	#endif
1552	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1553	return off;
1554	}
1555
1556
1557	/**
1558	* Emits code for AND'ing two 64-bit GPRs.
1559	*
1560	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1561	* and ARM64 hosts.
1562	*/
1563	DECL_INLINE_THROW(uint32_t)
1564	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1565	{
1566	#if defined(RT_ARCH_AMD64)
1567	/* and Gv, Ev */
1568	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1569	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1570	pbCodeBuf[off++] = 0x23;
1571	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1572	RT_NOREF(fSetFlags);
1573
1574	#elif defined(RT_ARCH_ARM64)
1575	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1576	if (!fSetFlags)
1577	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1578	else
1579	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
1580
1581	#else
1582	# error "Port me"
1583	#endif
1584	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1585	return off;
1586	}
1587
1588
1589	/**
1590	* Emits code for AND'ing two 32-bit GPRs.
1591	*/
1592	DECL_INLINE_THROW(uint32_t)
1593	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1594	{
1595	#if defined(RT_ARCH_AMD64)
1596	/* and Gv, Ev */
1597	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1598	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1599	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1600	pbCodeBuf[off++] = 0x23;
1601	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1602	RT_NOREF(fSetFlags);
1603
1604	#elif defined(RT_ARCH_ARM64)
1605	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1606	if (!fSetFlags)
1607	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1608	else
1609	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1610
1611	#else
1612	# error "Port me"
1613	#endif
1614	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1615	return off;
1616	}
1617
1618
1619	/**
1620	* Emits code for AND'ing a 64-bit GPRs with a constant.
1621	*
1622	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1623	* and ARM64 hosts.
1624	*/
1625	DECL_INLINE_THROW(uint32_t)
1626	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
1627	{
1628	#if defined(RT_ARCH_AMD64)
1629	if ((int64_t)uImm == (int8_t)uImm)
1630	{
1631	/* and Ev, imm8 */
1632	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1633	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1634	pbCodeBuf[off++] = 0x83;
1635	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1636	pbCodeBuf[off++] = (uint8_t)uImm;
1637	}
1638	else if ((int64_t)uImm == (int32_t)uImm)
1639	{
1640	/* and Ev, imm32 */
1641	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1642	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1643	pbCodeBuf[off++] = 0x81;
1644	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1645	pbCodeBuf[off++] = RT_BYTE1(uImm);
1646	pbCodeBuf[off++] = RT_BYTE2(uImm);
1647	pbCodeBuf[off++] = RT_BYTE3(uImm);
1648	pbCodeBuf[off++] = RT_BYTE4(uImm);
1649	}
1650	else
1651	{
1652	/* Use temporary register for the 64-bit immediate. */
1653	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1654	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
1655	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1656	}
1657	RT_NOREF(fSetFlags);
1658
1659	#elif defined(RT_ARCH_ARM64)
1660	uint32_t uImmR = 0;
1661	uint32_t uImmNandS = 0;
1662	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
1663	{
1664	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1665	if (!fSetFlags)
1666	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
1667	else
1668	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
1669	}
1670	else
1671	{
1672	/* Use temporary register for the 64-bit immediate. */
1673	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1674	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1675	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1676	}
1677
1678	#else
1679	# error "Port me"
1680	#endif
1681	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1682	return off;
1683	}
1684
1685
1686	/**
1687	* Emits code for AND'ing an 32-bit GPRs with a constant.
1688	*/
1689	DECL_INLINE_THROW(uint32_t)
1690	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
1691	{
1692	#if defined(RT_ARCH_AMD64)
1693	/* and Ev, imm */
1694	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1695	if (iGprDst >= 8)
1696	pbCodeBuf[off++] = X86_OP_REX_B;
1697	if ((int32_t)uImm == (int8_t)uImm)
1698	{
1699	pbCodeBuf[off++] = 0x83;
1700	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1701	pbCodeBuf[off++] = (uint8_t)uImm;
1702	}
1703	else
1704	{
1705	pbCodeBuf[off++] = 0x81;
1706	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1707	pbCodeBuf[off++] = RT_BYTE1(uImm);
1708	pbCodeBuf[off++] = RT_BYTE2(uImm);
1709	pbCodeBuf[off++] = RT_BYTE3(uImm);
1710	pbCodeBuf[off++] = RT_BYTE4(uImm);
1711	}
1712	RT_NOREF(fSetFlags);
1713
1714	#elif defined(RT_ARCH_ARM64)
1715	uint32_t uImmR = 0;
1716	uint32_t uImmNandS = 0;
1717	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
1718	{
1719	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1720	if (!fSetFlags)
1721	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1722	else
1723	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1724	}
1725	else
1726	{
1727	/* Use temporary register for the 64-bit immediate. */
1728	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1729	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1730	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1731	}
1732
1733	#else
1734	# error "Port me"
1735	#endif
1736	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1737	return off;
1738	}
1739
1740
1741	/**
1742	* Emits code for XOR'ing two 64-bit GPRs.
1743	*/
1744	DECL_INLINE_THROW(uint32_t)
1745	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1746	{
1747	#if defined(RT_ARCH_AMD64)
1748	/* and Gv, Ev */
1749	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1750	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1751	pbCodeBuf[off++] = 0x33;
1752	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1753
1754	#elif defined(RT_ARCH_ARM64)
1755	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1756	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1757
1758	#else
1759	# error "Port me"
1760	#endif
1761	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1762	return off;
1763	}
1764
1765
1766	/**
1767	* Emits code for XOR'ing two 32-bit GPRs.
1768	*/
1769	DECL_INLINE_THROW(uint32_t)
1770	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1771	{
1772	#if defined(RT_ARCH_AMD64)
1773	/* and Gv, Ev */
1774	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1775	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1776	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1777	pbCodeBuf[off++] = 0x33;
1778	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1779
1780	#elif defined(RT_ARCH_ARM64)
1781	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1782	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1783
1784	#else
1785	# error "Port me"
1786	#endif
1787	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1788	return off;
1789	}
1790
1791
1792	/*********************************************************************************************************************************
1793	* Shifting *
1794	*********************************************************************************************************************************/
1795
1796	/**
1797	* Emits code for shifting a GPR a fixed number of bits to the left.
1798	*/
1799	DECL_INLINE_THROW(uint32_t)
1800	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1801	{
1802	Assert(cShift > 0 && cShift < 64);
1803
1804	#if defined(RT_ARCH_AMD64)
1805	/* shl dst, cShift */
1806	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1807	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1808	if (cShift != 1)
1809	{
1810	pbCodeBuf[off++] = 0xc1;
1811	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1812	pbCodeBuf[off++] = cShift;
1813	}
1814	else
1815	{
1816	pbCodeBuf[off++] = 0xd1;
1817	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1818	}
1819
1820	#elif defined(RT_ARCH_ARM64)
1821	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1822	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
1823
1824	#else
1825	# error "Port me"
1826	#endif
1827	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1828	return off;
1829	}
1830
1831
1832	/**
1833	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
1834	*/
1835	DECL_INLINE_THROW(uint32_t)
1836	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1837	{
1838	Assert(cShift > 0 && cShift < 32);
1839
1840	#if defined(RT_ARCH_AMD64)
1841	/* shl dst, cShift */
1842	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1843	if (iGprDst >= 8)
1844	pbCodeBuf[off++] = X86_OP_REX_B;
1845	if (cShift != 1)
1846	{
1847	pbCodeBuf[off++] = 0xc1;
1848	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1849	pbCodeBuf[off++] = cShift;
1850	}
1851	else
1852	{
1853	pbCodeBuf[off++] = 0xd1;
1854	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1855	}
1856
1857	#elif defined(RT_ARCH_ARM64)
1858	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1859	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
1860
1861	#else
1862	# error "Port me"
1863	#endif
1864	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1865	return off;
1866	}
1867
1868
1869	/**
1870	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1871	*/
1872	DECL_INLINE_THROW(uint32_t)
1873	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1874	{
1875	Assert(cShift > 0 && cShift < 64);
1876
1877	#if defined(RT_ARCH_AMD64)
1878	/* shr dst, cShift */
1879	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1880	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1881	if (cShift != 1)
1882	{
1883	pbCodeBuf[off++] = 0xc1;
1884	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1885	pbCodeBuf[off++] = cShift;
1886	}
1887	else
1888	{
1889	pbCodeBuf[off++] = 0xd1;
1890	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1891	}
1892
1893	#elif defined(RT_ARCH_ARM64)
1894	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1895	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1896
1897	#else
1898	# error "Port me"
1899	#endif
1900	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1901	return off;
1902	}
1903
1904
1905	/**
1906	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
1907	* right.
1908	*/
1909	DECL_INLINE_THROW(uint32_t)
1910	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1911	{
1912	Assert(cShift > 0 && cShift < 32);
1913
1914	#if defined(RT_ARCH_AMD64)
1915	/* shr dst, cShift */
1916	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1917	if (iGprDst >= 8)
1918	pbCodeBuf[off++] = X86_OP_REX_B;
1919	if (cShift != 1)
1920	{
1921	pbCodeBuf[off++] = 0xc1;
1922	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1923	pbCodeBuf[off++] = cShift;
1924	}
1925	else
1926	{
1927	pbCodeBuf[off++] = 0xd1;
1928	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1929	}
1930
1931	#elif defined(RT_ARCH_ARM64)
1932	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1933	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
1934
1935	#else
1936	# error "Port me"
1937	#endif
1938	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1939	return off;
1940	}
1941
1942
1943
1944	/*********************************************************************************************************************************
1945	* Compare and Testing *
1946	*********************************************************************************************************************************/
1947
1948
1949	#ifdef RT_ARCH_ARM64
1950	/**
1951	* Emits an ARM64 compare instruction.
1952	*/
1953	DECL_INLINE_THROW(uint32_t)
1954	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1955	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1956	{
1957	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1958	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
1959	f64Bit, true /fSetFlags/, cShift, enmShift);
1960	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1961	return off;
1962	}
1963	#endif
1964
1965
1966	/**
1967	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1968	* with conditional instruction.
1969	*/
1970	DECL_INLINE_THROW(uint32_t)
1971	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1972	{
1973	#ifdef RT_ARCH_AMD64
1974	/* cmp Gv, Ev */
1975	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1976	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1977	pbCodeBuf[off++] = 0x3b;
1978	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1979	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1980
1981	#elif defined(RT_ARCH_ARM64)
1982	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1983
1984	#else
1985	# error "Port me!"
1986	#endif
1987	return off;
1988	}
1989
1990
1991	/**
1992	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1993	* with conditional instruction.
1994	*/
1995	DECL_INLINE_THROW(uint32_t)
1996	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1997	{
1998	#ifdef RT_ARCH_AMD64
1999	/* cmp Gv, Ev */
2000	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2001	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2002	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2003	pbCodeBuf[off++] = 0x3b;
2004	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2005	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2006
2007	#elif defined(RT_ARCH_ARM64)
2008	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2009
2010	#else
2011	# error "Port me!"
2012	#endif
2013	return off;
2014	}
2015
2016
2017	/**
2018	* Emits a compare of a 64-bit GPR with a constant value, settings status
2019	* flags/whatever for use with conditional instruction.
2020	*/
2021	DECL_INLINE_THROW(uint32_t)
2022	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2023	{
2024	#ifdef RT_ARCH_AMD64
2025	if (uImm <= UINT32_C(0xff))
2026	{
2027	/* cmp Ev, Ib */
2028	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2029	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2030	pbCodeBuf[off++] = 0x83;
2031	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2032	pbCodeBuf[off++] = (uint8_t)uImm;
2033	}
2034	else if ((int64_t)uImm == (int32_t)uImm)
2035	{
2036	/* cmp Ev, imm */
2037	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2038	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2039	pbCodeBuf[off++] = 0x81;
2040	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2041	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2042	pbCodeBuf[off++] = RT_BYTE1(uImm);
2043	pbCodeBuf[off++] = RT_BYTE2(uImm);
2044	pbCodeBuf[off++] = RT_BYTE3(uImm);
2045	pbCodeBuf[off++] = RT_BYTE4(uImm);
2046	}
2047	else
2048	{
2049	/* Use temporary register for the immediate. */
2050	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2051	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2052	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2053	}
2054
2055	#elif defined(RT_ARCH_ARM64)
2056	/** @todo guess there are clevere things we can do here... */
2057	if (uImm < _4K)
2058	{
2059	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2060	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2061	true /64Bit/, true /fSetFlags/);
2062	}
2063	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2064	{
2065	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2066	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2067	true /64Bit/, true /fSetFlags/, true /fShift12/);
2068	}
2069	else
2070	{
2071	/* Use temporary register for the immediate. */
2072	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2073	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2074	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2075	}
2076
2077	#else
2078	# error "Port me!"
2079	#endif
2080
2081	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2082	return off;
2083	}
2084
2085
2086	/**
2087	* Emits a compare of a 32-bit GPR with a constant value, settings status
2088	* flags/whatever for use with conditional instruction.
2089	*/
2090	DECL_INLINE_THROW(uint32_t)
2091	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2092	{
2093	#ifdef RT_ARCH_AMD64
2094	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2095	if (iGprLeft >= 8)
2096	pbCodeBuf[off++] = X86_OP_REX_B;
2097	if (uImm <= UINT32_C(0xff))
2098	{
2099	/* cmp Ev, Ib */
2100	pbCodeBuf[off++] = 0x83;
2101	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2102	pbCodeBuf[off++] = (uint8_t)uImm;
2103	}
2104	else
2105	{
2106	/* cmp Ev, imm */
2107	pbCodeBuf[off++] = 0x81;
2108	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2109	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2110	pbCodeBuf[off++] = RT_BYTE1(uImm);
2111	pbCodeBuf[off++] = RT_BYTE2(uImm);
2112	pbCodeBuf[off++] = RT_BYTE3(uImm);
2113	pbCodeBuf[off++] = RT_BYTE4(uImm);
2114	}
2115
2116	#elif defined(RT_ARCH_ARM64)
2117	/** @todo guess there are clevere things we can do here... */
2118	if (uImm < _4K)
2119	{
2120	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2121	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2122	false /64Bit/, true /fSetFlags/);
2123	}
2124	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2125	{
2126	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2127	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2128	false /64Bit/, true /fSetFlags/, true /fShift12/);
2129	}
2130	else
2131	{
2132	/* Use temporary register for the immediate. */
2133	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2134	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2135	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2136	}
2137
2138	#else
2139	# error "Port me!"
2140	#endif
2141
2142	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2143	return off;
2144	}
2145
2146
2147
2148	/*********************************************************************************************************************************
2149	* Branching *
2150	*********************************************************************************************************************************/
2151
2152	/**
2153	* Emits a JMP rel32 / B imm19 to the given label.
2154	*/
2155	DECL_INLINE_THROW(uint32_t)
2156	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2157	{
2158	Assert(idxLabel < pReNative->cLabels);
2159
2160	#ifdef RT_ARCH_AMD64
2161	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2162	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2163	{
2164	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2165	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2166	{
2167	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2168	pbCodeBuf[off++] = (uint8_t)offRel;
2169	}
2170	else
2171	{
2172	offRel -= 3;
2173	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2174	pbCodeBuf[off++] = RT_BYTE1(offRel);
2175	pbCodeBuf[off++] = RT_BYTE2(offRel);
2176	pbCodeBuf[off++] = RT_BYTE3(offRel);
2177	pbCodeBuf[off++] = RT_BYTE4(offRel);
2178	}
2179	}
2180	else
2181	{
2182	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2183	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2184	pbCodeBuf[off++] = 0xfe;
2185	pbCodeBuf[off++] = 0xff;
2186	pbCodeBuf[off++] = 0xff;
2187	pbCodeBuf[off++] = 0xff;
2188	}
2189	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2190
2191	#elif defined(RT_ARCH_ARM64)
2192	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2193	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2194	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2195	else
2196	{
2197	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2198	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2199	}
2200
2201	#else
2202	# error "Port me!"
2203	#endif
2204	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2205	return off;
2206	}
2207
2208
2209	/**
2210	* Emits a JMP rel32 / B imm19 to a new undefined label.
2211	*/
2212	DECL_INLINE_THROW(uint32_t)
2213	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2214	{
2215	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2216	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2217	}
2218
2219	/** Condition type. */
2220	#ifdef RT_ARCH_AMD64
2221	typedef enum IEMNATIVEINSTRCOND : uint8_t
2222	{
2223	kIemNativeInstrCond_o = 0,
2224	kIemNativeInstrCond_no,
2225	kIemNativeInstrCond_c,
2226	kIemNativeInstrCond_nc,
2227	kIemNativeInstrCond_e,
2228	kIemNativeInstrCond_ne,
2229	kIemNativeInstrCond_be,
2230	kIemNativeInstrCond_nbe,
2231	kIemNativeInstrCond_s,
2232	kIemNativeInstrCond_ns,
2233	kIemNativeInstrCond_p,
2234	kIemNativeInstrCond_np,
2235	kIemNativeInstrCond_l,
2236	kIemNativeInstrCond_nl,
2237	kIemNativeInstrCond_le,
2238	kIemNativeInstrCond_nle
2239	} IEMNATIVEINSTRCOND;
2240	#elif defined(RT_ARCH_ARM64)
2241	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2242	#else
2243	# error "Port me!"
2244	#endif
2245
2246
2247	/**
2248	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2249	*/
2250	DECL_INLINE_THROW(uint32_t)
2251	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2252	{
2253	Assert(idxLabel < pReNative->cLabels);
2254
2255	#ifdef RT_ARCH_AMD64
2256	/* jcc rel32 */
2257	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2258	pbCodeBuf[off++] = 0x0f;
2259	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2260	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2261	pbCodeBuf[off++] = 0x00;
2262	pbCodeBuf[off++] = 0x00;
2263	pbCodeBuf[off++] = 0x00;
2264	pbCodeBuf[off++] = 0x00;
2265
2266	#elif defined(RT_ARCH_ARM64)
2267	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2268	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2269	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2270
2271	#else
2272	# error "Port me!"
2273	#endif
2274	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2275	return off;
2276	}
2277
2278
2279	/**
2280	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2281	*/
2282	DECL_INLINE_THROW(uint32_t)
2283	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2284	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2285	{
2286	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2287	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2288	}
2289
2290
2291	/**
2292	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2293	*/
2294	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2295	{
2296	#ifdef RT_ARCH_AMD64
2297	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2298	#elif defined(RT_ARCH_ARM64)
2299	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2300	#else
2301	# error "Port me!"
2302	#endif
2303	}
2304
2305	/**
2306	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2307	*/
2308	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2309	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2310	{
2311	#ifdef RT_ARCH_AMD64
2312	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2313	#elif defined(RT_ARCH_ARM64)
2314	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2315	#else
2316	# error "Port me!"
2317	#endif
2318	}
2319
2320
2321	/**
2322	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2323	*/
2324	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2325	{
2326	#ifdef RT_ARCH_AMD64
2327	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2328	#elif defined(RT_ARCH_ARM64)
2329	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2330	#else
2331	# error "Port me!"
2332	#endif
2333	}
2334
2335	/**
2336	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2337	*/
2338	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2339	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2340	{
2341	#ifdef RT_ARCH_AMD64
2342	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2343	#elif defined(RT_ARCH_ARM64)
2344	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2345	#else
2346	# error "Port me!"
2347	#endif
2348	}
2349
2350
2351	/**
2352	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2353	*/
2354	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2355	{
2356	#ifdef RT_ARCH_AMD64
2357	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2358	#elif defined(RT_ARCH_ARM64)
2359	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2360	#else
2361	# error "Port me!"
2362	#endif
2363	}
2364
2365	/**
2366	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2367	*/
2368	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2369	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2370	{
2371	#ifdef RT_ARCH_AMD64
2372	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2373	#elif defined(RT_ARCH_ARM64)
2374	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2375	#else
2376	# error "Port me!"
2377	#endif
2378	}
2379
2380
2381	/**
2382	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2383	*/
2384	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2385	{
2386	#ifdef RT_ARCH_AMD64
2387	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2388	#elif defined(RT_ARCH_ARM64)
2389	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2390	#else
2391	# error "Port me!"
2392	#endif
2393	}
2394
2395	/**
2396	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2397	*/
2398	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2399	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2400	{
2401	#ifdef RT_ARCH_AMD64
2402	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2403	#elif defined(RT_ARCH_ARM64)
2404	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2405	#else
2406	# error "Port me!"
2407	#endif
2408	}
2409
2410
2411	/**
2412	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2413	* How @a offJmp is applied is are target specific.
2414	*/
2415	DECL_INLINE_THROW(uint32_t)
2416	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2417	{
2418	#ifdef RT_ARCH_AMD64
2419	/* jcc rel32 */
2420	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2421	if (offTarget < 128 && offTarget >= -128)
2422	{
2423	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2424	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2425	}
2426	else
2427	{
2428	pbCodeBuf[off++] = 0x0f;
2429	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2430	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2431	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2432	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2433	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2434	}
2435
2436	#elif defined(RT_ARCH_ARM64)
2437	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2438	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2439
2440	#else
2441	# error "Port me!"
2442	#endif
2443	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2444	return off;
2445	}
2446
2447
2448	/**
2449	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2450	* How @a offJmp is applied is are target specific.
2451	*/
2452	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2453	{
2454	#ifdef RT_ARCH_AMD64
2455	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2456	#elif defined(RT_ARCH_ARM64)
2457	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2458	#else
2459	# error "Port me!"
2460	#endif
2461	}
2462
2463
2464	/**
2465	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2466	* How @a offJmp is applied is are target specific.
2467	*/
2468	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2469	{
2470	#ifdef RT_ARCH_AMD64
2471	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2472	#elif defined(RT_ARCH_ARM64)
2473	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2474	#else
2475	# error "Port me!"
2476	#endif
2477	}
2478
2479
2480	/**
2481	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2482	* How @a offJmp is applied is are target specific.
2483	*/
2484	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2485	{
2486	#ifdef RT_ARCH_AMD64
2487	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2488	#elif defined(RT_ARCH_ARM64)
2489	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2490	#else
2491	# error "Port me!"
2492	#endif
2493	}
2494
2495
2496	/**
2497	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2498	* How @a offJmp is applied is are target specific.
2499	*/
2500	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2501	{
2502	#ifdef RT_ARCH_AMD64
2503	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2504	#elif defined(RT_ARCH_ARM64)
2505	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2506	#else
2507	# error "Port me!"
2508	#endif
2509	}
2510
2511
2512	/**
2513	* Fixes up a conditional jump to a fixed label.
2514	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2515	*/
2516	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2517	{
2518	# if defined(RT_ARCH_AMD64)
2519	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2520	if (pbCodeBuf[offFixup] != 0x0f)
2521	{
2522	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2523	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2524	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2525	}
2526	else
2527	{
2528	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2529	uint32_t const offRel32 = offTarget - (offFixup + 6);
2530	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2531	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2532	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2533	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2534	}
2535
2536	# elif defined(RT_ARCH_ARM64)
2537	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2538
2539	int32_t const offDisp = offTarget - offFixup;
2540	Assert(offDisp >= -262144 && offDisp < 262144);
2541	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
2542
2543	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
2544	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
2545
2546	# endif
2547	}
2548
2549
2550	/**
2551	* Internal helper, don't call directly.
2552	*/
2553	DECL_INLINE_THROW(uint32_t)
2554	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2555	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
2556	{
2557	Assert(iBitNo < 64);
2558	#ifdef RT_ARCH_AMD64
2559	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2560	if (iBitNo < 8)
2561	{
2562	/* test Eb, imm8 */
2563	if (iGprSrc >= 4)
2564	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2565	pbCodeBuf[off++] = 0xf6;
2566	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2567	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2568	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2569	}
2570	else
2571	{
2572	/* bt Ev, imm8 */
2573	if (iBitNo >= 32)
2574	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2575	else if (iGprSrc >= 8)
2576	pbCodeBuf[off++] = X86_OP_REX_B;
2577	pbCodeBuf[off++] = 0x0f;
2578	pbCodeBuf[off++] = 0xba;
2579	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2580	pbCodeBuf[off++] = iBitNo;
2581	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2582	}
2583
2584	#elif defined(RT_ARCH_ARM64)
2585	/* Use the TBNZ instruction here. */
2586	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2587	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
2588	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2589
2590	#else
2591	# error "Port me!"
2592	#endif
2593	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2594	return off;
2595	}
2596
2597
2598	/**
2599	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2600	* @a iGprSrc.
2601	*
2602	* @note On ARM64 the range is only +/-8191 instructions.
2603	*/
2604	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2605	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2606	{
2607	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
2608	}
2609
2610
2611	/**
2612	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2613	* _set_ in @a iGprSrc.
2614	*
2615	* @note On ARM64 the range is only +/-8191 instructions.
2616	*/
2617	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2618	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2619	{
2620	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
2621	}
2622
2623
2624	/**
2625	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2626	* flags accordingly.
2627	*/
2628	DECL_INLINE_THROW(uint32_t)
2629	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2630	{
2631	Assert(fBits != 0);
2632	#ifdef RT_ARCH_AMD64
2633
2634	if (fBits >= UINT32_MAX)
2635	{
2636	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2637
2638	/* test Ev,Gv */
2639	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2640	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2641	pbCodeBuf[off++] = 0x85;
2642	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2643
2644	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2645	}
2646	else if (fBits <= UINT32_MAX)
2647	{
2648	/* test Eb, imm8 or test Ev, imm32 */
2649	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2650	if (fBits <= UINT8_MAX)
2651	{
2652	if (iGprSrc >= 4)
2653	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2654	pbCodeBuf[off++] = 0xf6;
2655	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2656	pbCodeBuf[off++] = (uint8_t)fBits;
2657	}
2658	else
2659	{
2660	if (iGprSrc >= 8)
2661	pbCodeBuf[off++] = X86_OP_REX_B;
2662	pbCodeBuf[off++] = 0xf7;
2663	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2664	pbCodeBuf[off++] = RT_BYTE1(fBits);
2665	pbCodeBuf[off++] = RT_BYTE2(fBits);
2666	pbCodeBuf[off++] = RT_BYTE3(fBits);
2667	pbCodeBuf[off++] = RT_BYTE4(fBits);
2668	}
2669	}
2670	/** @todo implement me. */
2671	else
2672	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
2673
2674	#elif defined(RT_ARCH_ARM64)
2675
2676	if (false)
2677	{
2678	/** @todo figure out how to work the immr / N:imms constants. */
2679	}
2680	else
2681	{
2682	/* ands Zr, iGprSrc, iTmpReg */
2683	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2684	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2685	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2686	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2687	}
2688
2689	#else
2690	# error "Port me!"
2691	#endif
2692	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2693	return off;
2694	}
2695
2696
2697	/**
2698	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
2699	* @a iGprSrc, setting CPU flags accordingly.
2700	*/
2701	DECL_INLINE_THROW(uint32_t)
2702	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
2703	{
2704	Assert(fBits != 0);
2705
2706	#ifdef RT_ARCH_AMD64
2707	/* test Eb, imm8 */
2708	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2709	if (iGprSrc >= 4)
2710	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2711	pbCodeBuf[off++] = 0xf6;
2712	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2713	pbCodeBuf[off++] = fBits;
2714
2715	#elif defined(RT_ARCH_ARM64)
2716
2717	/* ands xzr, src, [tmp\|#imm] */
2718	uint32_t uImmR = 0;
2719	uint32_t uImmNandS = 0;
2720	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2721	{
2722	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2723	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2724	}
2725	else
2726	{
2727	/* Use temporary register for the 64-bit immediate. */
2728	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2729	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2730	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
2731	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2732	}
2733
2734	#else
2735	# error "Port me!"
2736	#endif
2737	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2738	return off;
2739	}
2740
2741
2742	/**
2743	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2744	* are set in @a iGprSrc.
2745	*/
2746	DECL_INLINE_THROW(uint32_t)
2747	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2748	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2749	{
2750	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2751
2752	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2753	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2754
2755	return off;
2756	}
2757
2758
2759	/**
2760	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2761	* are set in @a iGprSrc.
2762	*/
2763	DECL_INLINE_THROW(uint32_t)
2764	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2765	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2766	{
2767	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2768
2769	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2770	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2771
2772	return off;
2773	}
2774
2775
2776	/**
2777	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
2778	*
2779	* The operand size is given by @a f64Bit.
2780	*/
2781	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2782	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
2783	{
2784	Assert(idxLabel < pReNative->cLabels);
2785
2786	#ifdef RT_ARCH_AMD64
2787	/* test reg32,reg32 / test reg64,reg64 */
2788	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2789	if (f64Bit)
2790	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
2791	else if (iGprSrc >= 8)
2792	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
2793	pbCodeBuf[off++] = 0x85;
2794	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
2795	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2796
2797	/* jz idxLabel */
2798	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2799
2800	#elif defined(RT_ARCH_ARM64)
2801	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2802	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2803	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
2804	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2805
2806	#else
2807	# error "Port me!"
2808	#endif
2809	return off;
2810	}
2811
2812
2813	/**
2814	* Emits code that jumps to a new label if @a iGprSrc is zero.
2815	*
2816	* The operand size is given by @a f64Bit.
2817	*/
2818	DECL_INLINE_THROW(uint32_t)
2819	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
2820	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2821	{
2822	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2823	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
2824	}
2825
2826
2827	/**
2828	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
2829	* differs.
2830	*/
2831	DECL_INLINE_THROW(uint32_t)
2832	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2833	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
2834	{
2835	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
2836	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2837	return off;
2838	}
2839
2840
2841	/**
2842	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
2843	*/
2844	DECL_INLINE_THROW(uint32_t)
2845	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2846	uint8_t iGprLeft, uint8_t iGprRight,
2847	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2848	{
2849	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2850	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
2851	}
2852
2853
2854	/**
2855	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
2856	*/
2857	DECL_INLINE_THROW(uint32_t)
2858	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2859	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
2860	{
2861	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
2862	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2863	return off;
2864	}
2865
2866
2867	/**
2868	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
2869	*/
2870	DECL_INLINE_THROW(uint32_t)
2871	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2872	uint8_t iGprSrc, uint64_t uImm,
2873	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2874	{
2875	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2876	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2877	}
2878
2879
2880	/**
2881	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
2882	* @a uImm.
2883	*/
2884	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2885	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
2886	{
2887	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
2888	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2889	return off;
2890	}
2891
2892
2893	/**
2894	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
2895	* @a uImm.
2896	*/
2897	DECL_INLINE_THROW(uint32_t)
2898	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2899	uint8_t iGprSrc, uint32_t uImm,
2900	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2901	{
2902	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2903	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2904	}
2905
2906
2907	/*********************************************************************************************************************************
2908	* Calls. *
2909	*********************************************************************************************************************************/
2910
2911	/**
2912	* Emits a call to a 64-bit address.
2913	*/
2914	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2915	{
2916	#ifdef RT_ARCH_AMD64
2917	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2918
2919	/* call rax */
2920	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2921	pbCodeBuf[off++] = 0xff;
2922	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2923
2924	#elif defined(RT_ARCH_ARM64)
2925	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2926
2927	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2928	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2929
2930	#else
2931	# error "port me"
2932	#endif
2933	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2934	return off;
2935	}
2936
2937
2938	/**
2939	* Emits code to load a stack variable into an argument GPR.
2940	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
2941	*/
2942	DECL_FORCE_INLINE_THROW(uint32_t)
2943	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
2944	int32_t offAddend = 0)
2945	{
2946	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2947	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
2948	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
2949
2950	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
2951	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
2952	{
2953	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK));
2954	if (!offAddend)
2955	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
2956	else
2957	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
2958	}
2959	else
2960	{
2961	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
2962	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
2963	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
2964	if (offAddend)
2965	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
2966	}
2967	return off;
2968	}
2969
2970
2971	/**
2972	* Emits code to load a stack or immediate variable value into an argument GPR,
2973	* optional with a addend.
2974	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
2975	*/
2976	DECL_FORCE_INLINE_THROW(uint32_t)
2977	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
2978	int32_t offAddend = 0)
2979	{
2980	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2981	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
2982	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
2983	else
2984	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend);
2985	return off;
2986	}
2987
2988
2989	/**
2990	* Emits code to load the variable address into an argument GRP.
2991	*
2992	* This only works for uninitialized and stack variables.
2993	*/
2994	DECL_FORCE_INLINE_THROW(uint32_t)
2995	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
2996	bool fFlushShadows)
2997	{
2998	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2999	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
3000	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3001	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3002
3003	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
3004	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
3005
3006	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3007	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3008	{
3009	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
3010	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
3011	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3012	}
3013	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
3014	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3015
3016	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
3017	}
3018
3019
3020	/** @} */
3021
3022	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
3023

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102436

Download in other formats: