IEMN8veRecompilerEmit.h@ 102022

Last change on this file since 102022 was 102022, checked in by vboxsync, 15 months ago
VMM/IEM: Split out the inlined code emitters from IEMN8veRecompiler.h and into IEMN8veRecompilerEmit.h. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 92.6 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102022 2023-11-09 11:38:47Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif RT_ARCH_ARM64
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/*********************************************************************************************************************************
82	* Loads, Stores and Related Stuff. *
83	*********************************************************************************************************************************/
84
85	/**
86	* Emits setting a GPR to zero.
87	*/
88	DECL_INLINE_THROW(uint32_t)
89	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
90	{
91	#ifdef RT_ARCH_AMD64
92	/* xor gpr32, gpr32 */
93	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
94	if (iGpr >= 8)
95	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
96	pbCodeBuf[off++] = 0x33;
97	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
98
99	#elif RT_ARCH_ARM64
100	/* mov gpr, #0x0 */
101	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
102	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
103
104	#else
105	# error "port me"
106	#endif
107	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
108	return off;
109	}
110
111
112	/**
113	* Emits loading a constant into a 64-bit GPR
114	*/
115	DECL_INLINE_THROW(uint32_t)
116	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
117	{
118	if (!uImm64)
119	return iemNativeEmitGprZero(pReNative, off, iGpr);
120
121	#ifdef RT_ARCH_AMD64
122	if (uImm64 <= UINT32_MAX)
123	{
124	/* mov gpr, imm32 */
125	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
126	if (iGpr >= 8)
127	pbCodeBuf[off++] = X86_OP_REX_B;
128	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
129	pbCodeBuf[off++] = RT_BYTE1(uImm64);
130	pbCodeBuf[off++] = RT_BYTE2(uImm64);
131	pbCodeBuf[off++] = RT_BYTE3(uImm64);
132	pbCodeBuf[off++] = RT_BYTE4(uImm64);
133	}
134	else
135	{
136	/* mov gpr, imm64 */
137	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
138	if (iGpr < 8)
139	pbCodeBuf[off++] = X86_OP_REX_W;
140	else
141	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
142	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
143	pbCodeBuf[off++] = RT_BYTE1(uImm64);
144	pbCodeBuf[off++] = RT_BYTE2(uImm64);
145	pbCodeBuf[off++] = RT_BYTE3(uImm64);
146	pbCodeBuf[off++] = RT_BYTE4(uImm64);
147	pbCodeBuf[off++] = RT_BYTE5(uImm64);
148	pbCodeBuf[off++] = RT_BYTE6(uImm64);
149	pbCodeBuf[off++] = RT_BYTE7(uImm64);
150	pbCodeBuf[off++] = RT_BYTE8(uImm64);
151	}
152
153	#elif RT_ARCH_ARM64
154	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
155
156	/*
157	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
158	* supply remaining bits using 'movk grp, imm16, lsl #x'.
159	*
160	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
161	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
162	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
163	* after the first non-zero immediate component so we switch to movk for
164	* the remainder.
165	*/
166	uint32_t fMovK = 0;
167	/* mov gpr, imm16 */
168	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
169	if (uImmPart)
170	{
171	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
172	fMovK \|= RT_BIT_32(29);
173	}
174	/* mov[k] gpr, imm16, lsl #16 */
175	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
176	if (uImmPart)
177	{
178	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
179	fMovK \|= RT_BIT_32(29);
180	}
181	/* mov[k] gpr, imm16, lsl #32 */
182	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
183	if (uImmPart)
184	{
185	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
186	fMovK \|= RT_BIT_32(29);
187	}
188	/* mov[k] gpr, imm16, lsl #48 */
189	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
190	if (uImmPart)
191	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
192
193	/** @todo there is an inverted mask variant we might want to explore if it
194	* reduces the number of instructions... */
195	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
196	* clang 12.x does that, only to use the 'x' version for the
197	* addressing in the following ldr). */
198
199	#else
200	# error "port me"
201	#endif
202	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
203	return off;
204	}
205
206
207	/**
208	* Emits loading a constant into a 8-bit GPR
209	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
210	* only the ARM64 version does that.
211	*/
212	DECL_INLINE_THROW(uint32_t)
213	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
214	{
215	#ifdef RT_ARCH_AMD64
216	/* mov gpr, imm8 */
217	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
218	if (iGpr >= 8)
219	pbCodeBuf[off++] = X86_OP_REX_B;
220	else if (iGpr >= 4)
221	pbCodeBuf[off++] = X86_OP_REX;
222	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
223	pbCodeBuf[off++] = RT_BYTE1(uImm8);
224
225	#elif RT_ARCH_ARM64
226	/* movz gpr, imm16, lsl #0 */
227	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
228	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
229
230	#else
231	# error "port me"
232	#endif
233	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
234	return off;
235	}
236
237
238	#ifdef RT_ARCH_AMD64
239	/**
240	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
241	*/
242	DECL_FORCE_INLINE(uint32_t)
243	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
244	{
245	if (offVCpu < 128)
246	{
247	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
248	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
249	}
250	else
251	{
252	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
253	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
254	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
255	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
256	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
257	}
258	return off;
259	}
260	#elif RT_ARCH_ARM64
261	/**
262	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
263	*/
264	DECL_FORCE_INLINE_THROW(uint32_t)
265	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
266	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
267	{
268	/*
269	* There are a couple of ldr variants that takes an immediate offset, so
270	* try use those if we can, otherwise we have to use the temporary register
271	* help with the addressing.
272	*/
273	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
274	{
275	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
276	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
277	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
278	}
279	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
280	{
281	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
282	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
283	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
284	}
285	else
286	{
287	/* The offset is too large, so we must load it into a register and use
288	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
289	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
290	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
291
292	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
293	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
294	IEMNATIVE_REG_FIXED_TMP0);
295	}
296	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
297	return off;
298	}
299	#endif
300
301
302	/**
303	* Emits a 64-bit GPR load of a VCpu value.
304	*/
305	DECL_INLINE_THROW(uint32_t)
306	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
307	{
308	#ifdef RT_ARCH_AMD64
309	/* mov reg64, mem64 */
310	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
311	if (iGpr < 8)
312	pbCodeBuf[off++] = X86_OP_REX_W;
313	else
314	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
315	pbCodeBuf[off++] = 0x8b;
316	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
317	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
318
319	#elif RT_ARCH_ARM64
320	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
321
322	#else
323	# error "port me"
324	#endif
325	return off;
326	}
327
328
329	/**
330	* Emits a 32-bit GPR load of a VCpu value.
331	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
332	*/
333	DECL_INLINE_THROW(uint32_t)
334	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
335	{
336	#ifdef RT_ARCH_AMD64
337	/* mov reg32, mem32 */
338	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
339	if (iGpr >= 8)
340	pbCodeBuf[off++] = X86_OP_REX_R;
341	pbCodeBuf[off++] = 0x8b;
342	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
343	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
344
345	#elif RT_ARCH_ARM64
346	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
347
348	#else
349	# error "port me"
350	#endif
351	return off;
352	}
353
354
355	/**
356	* Emits a 16-bit GPR load of a VCpu value.
357	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
358	*/
359	DECL_INLINE_THROW(uint32_t)
360	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
361	{
362	#ifdef RT_ARCH_AMD64
363	/* movzx reg32, mem16 */
364	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
365	if (iGpr >= 8)
366	pbCodeBuf[off++] = X86_OP_REX_R;
367	pbCodeBuf[off++] = 0x0f;
368	pbCodeBuf[off++] = 0xb7;
369	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
370	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
371
372	#elif RT_ARCH_ARM64
373	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
374
375	#else
376	# error "port me"
377	#endif
378	return off;
379	}
380
381
382	/**
383	* Emits a 8-bit GPR load of a VCpu value.
384	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
385	*/
386	DECL_INLINE_THROW(uint32_t)
387	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
388	{
389	#ifdef RT_ARCH_AMD64
390	/* movzx reg32, mem8 */
391	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
392	if (iGpr >= 8)
393	pbCodeBuf[off++] = X86_OP_REX_R;
394	pbCodeBuf[off++] = 0x0f;
395	pbCodeBuf[off++] = 0xb6;
396	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
397	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
398
399	#elif RT_ARCH_ARM64
400	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
401
402	#else
403	# error "port me"
404	#endif
405	return off;
406	}
407
408
409	/**
410	* Emits a store of a GPR value to a 64-bit VCpu field.
411	*/
412	DECL_INLINE_THROW(uint32_t)
413	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
414	{
415	#ifdef RT_ARCH_AMD64
416	/* mov mem64, reg64 */
417	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
418	if (iGpr < 8)
419	pbCodeBuf[off++] = X86_OP_REX_W;
420	else
421	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
422	pbCodeBuf[off++] = 0x89;
423	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
424	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
425
426	#elif RT_ARCH_ARM64
427	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
428
429	#else
430	# error "port me"
431	#endif
432	return off;
433	}
434
435
436	/**
437	* Emits a store of a GPR value to a 32-bit VCpu field.
438	*/
439	DECL_INLINE_THROW(uint32_t)
440	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
441	{
442	#ifdef RT_ARCH_AMD64
443	/* mov mem32, reg32 */
444	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
445	if (iGpr >= 8)
446	pbCodeBuf[off++] = X86_OP_REX_R;
447	pbCodeBuf[off++] = 0x89;
448	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
449	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
450
451	#elif RT_ARCH_ARM64
452	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
453
454	#else
455	# error "port me"
456	#endif
457	return off;
458	}
459
460
461	/**
462	* Emits a store of a GPR value to a 16-bit VCpu field.
463	*/
464	DECL_INLINE_THROW(uint32_t)
465	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
466	{
467	#ifdef RT_ARCH_AMD64
468	/* mov mem16, reg16 */
469	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
470	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
471	if (iGpr >= 8)
472	pbCodeBuf[off++] = X86_OP_REX_R;
473	pbCodeBuf[off++] = 0x89;
474	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
475	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
476
477	#elif RT_ARCH_ARM64
478	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
479
480	#else
481	# error "port me"
482	#endif
483	return off;
484	}
485
486
487	/**
488	* Emits a store of a GPR value to a 8-bit VCpu field.
489	*/
490	DECL_INLINE_THROW(uint32_t)
491	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
492	{
493	#ifdef RT_ARCH_AMD64
494	/* mov mem8, reg8 */
495	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
496	if (iGpr >= 8)
497	pbCodeBuf[off++] = X86_OP_REX_R;
498	pbCodeBuf[off++] = 0x88;
499	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
500	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
501
502	#elif RT_ARCH_ARM64
503	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
504
505	#else
506	# error "port me"
507	#endif
508	return off;
509	}
510
511
512	/**
513	* Emits a load effective address to a GRP of a VCpu field.
514	*/
515	DECL_INLINE_THROW(uint32_t)
516	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
517	{
518	#ifdef RT_ARCH_AMD64
519	/* lea gprdst, [rbx + offDisp] */
520	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
521	if (iGprDst < 8)
522	pbCodeBuf[off++] = X86_OP_REX_W;
523	else
524	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
525	pbCodeBuf[off++] = 0x8d;
526	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
527
528	#elif defined(RT_ARCH_ARM64)
529	if (offVCpu < (unsigned)_4K)
530	{
531	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
532	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
533	}
534	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
535	{
536	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
537	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
538	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
539	}
540	else
541	{
542	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
543	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
544	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
545	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
546	}
547
548	#else
549	# error "port me"
550	#endif
551	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
552	return off;
553	}
554
555
556	/**
557	* Emits a gprdst = gprsrc load.
558	*/
559	DECL_INLINE_THROW(uint32_t)
560	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
561	{
562	#ifdef RT_ARCH_AMD64
563	/* mov gprdst, gprsrc */
564	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
565	if ((iGprDst \| iGprSrc) >= 8)
566	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
567	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
568	: X86_OP_REX_W \| X86_OP_REX_R;
569	else
570	pbCodeBuf[off++] = X86_OP_REX_W;
571	pbCodeBuf[off++] = 0x8b;
572	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
573
574	#elif RT_ARCH_ARM64
575	/* mov dst, src; alias for: orr dst, xzr, src */
576	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
577	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
578
579	#else
580	# error "port me"
581	#endif
582	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
583	return off;
584	}
585
586
587	/**
588	* Emits a gprdst = gprsrc[31:0] load.
589	* @note Bits 63 thru 32 are cleared.
590	*/
591	DECL_INLINE_THROW(uint32_t)
592	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
593	{
594	#ifdef RT_ARCH_AMD64
595	/* mov gprdst, gprsrc */
596	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
597	if ((iGprDst \| iGprSrc) >= 8)
598	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
599	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
600	: X86_OP_REX_R;
601	pbCodeBuf[off++] = 0x8b;
602	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
603
604	#elif RT_ARCH_ARM64
605	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
606	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
607	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
608
609	#else
610	# error "port me"
611	#endif
612	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
613	return off;
614	}
615
616
617	/**
618	* Emits a gprdst = gprsrc[15:0] load.
619	* @note Bits 63 thru 15 are cleared.
620	*/
621	DECL_INLINE_THROW(uint32_t)
622	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
623	{
624	#ifdef RT_ARCH_AMD64
625	/* movzx Gv,Ew */
626	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
627	if ((iGprDst \| iGprSrc) >= 8)
628	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
629	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
630	: X86_OP_REX_R;
631	pbCodeBuf[off++] = 0x0f;
632	pbCodeBuf[off++] = 0xb7;
633	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
634
635	#elif RT_ARCH_ARM64
636	/* and gprdst, gprsrc, #0xffff */
637	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
638	# if 1
639	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
640	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
641	# else
642	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
643	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
644	# endif
645
646	#else
647	# error "port me"
648	#endif
649	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
650	return off;
651	}
652
653
654	/**
655	* Emits a gprdst = gprsrc[7:0] load.
656	* @note Bits 63 thru 8 are cleared.
657	*/
658	DECL_INLINE_THROW(uint32_t)
659	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
660	{
661	#ifdef RT_ARCH_AMD64
662	/* movzx Gv,Eb */
663	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
664	if ((iGprDst \| iGprSrc) >= 8)
665	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
666	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
667	: X86_OP_REX_R;
668	pbCodeBuf[off++] = 0x0f;
669	pbCodeBuf[off++] = 0xb6;
670	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
671
672	#elif RT_ARCH_ARM64
673	/* and gprdst, gprsrc, #0xff */
674	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
675	# if 1
676	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
677	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
678	# else
679	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
680	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
681	# endif
682
683	#else
684	# error "port me"
685	#endif
686	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
687	return off;
688	}
689
690	#if 0 /** @todo */
691	/**
692	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
693	* @note Bits 63 thru 8 are cleared.
694	*/
695	DECL_INLINE_THROW(uint32_t)
696	iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
697	{
698	#ifdef RT_ARCH_AMD64
699	/* movzx Gv,Eb */
700	/** @todo */
701	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
702	if ((iGprDst \| iGprSrc) >= 8)
703	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
704	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
705	: X86_OP_REX_R;
706	pbCodeBuf[off++] = 0x0f;
707	pbCodeBuf[off++] = 0xb6;
708	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
709
710	#elif RT_ARCH_ARM64
711	/* ubfx gprdst, gprsrc, #8, #8 */
712	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
713	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
714	pu32CodeBuf[off++] = /** @todo ubfx */;
715
716	#else
717	# error "port me"
718	#endif
719	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
720	return off;
721	}
722	#endif
723
724	#ifdef RT_ARCH_AMD64
725	/**
726	* Common bit of iemNativeEmitLoadGprByBp and friends.
727	*/
728	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
729	PIEMRECOMPILERSTATE pReNativeAssert)
730	{
731	if (offDisp < 128 && offDisp >= -128)
732	{
733	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
734	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
735	}
736	else
737	{
738	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
739	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
740	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
741	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
742	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
743	}
744	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
745	return off;
746	}
747	#elif defined(RT_ARCH_ARM64)
748	/**
749	* Common bit of iemNativeEmitLoadGprByBp and friends.
750	*/
751	DECL_FORCE_INLINE_THROW(uint32_t)
752	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
753	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
754	{
755	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
756	{
757	/* str w/ unsigned imm12 (scaled) */
758	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
759	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
760	}
761	else if (offDisp >= -256 && offDisp <= 256)
762	{
763	/* stur w/ signed imm9 (unscaled) */
764	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
765	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
766	}
767	else
768	{
769	/* Use temporary indexing register. */
770	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
771	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
772	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
773	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
774	}
775	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
776	return off;
777	}
778	#endif
779
780
781	/**
782	* Emits a 64-bit GRP load instruction with an BP relative source address.
783	*/
784	DECL_INLINE_THROW(uint32_t)
785	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
786	{
787	#ifdef RT_ARCH_AMD64
788	/* mov gprdst, qword [rbp + offDisp] */
789	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
790	if (iGprDst < 8)
791	pbCodeBuf[off++] = X86_OP_REX_W;
792	else
793	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
794	pbCodeBuf[off++] = 0x8b;
795	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
796
797	#elif defined(RT_ARCH_ARM64)
798	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
799
800	#else
801	# error "port me"
802	#endif
803	}
804
805
806	/**
807	* Emits a 32-bit GRP load instruction with an BP relative source address.
808	*/
809	DECL_INLINE_THROW(uint32_t)
810	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
811	{
812	#ifdef RT_ARCH_AMD64
813	/* mov gprdst, dword [rbp + offDisp] */
814	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
815	if (iGprDst >= 8)
816	pbCodeBuf[off++] = X86_OP_REX_R;
817	pbCodeBuf[off++] = 0x8b;
818	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
819
820	#elif defined(RT_ARCH_ARM64)
821	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
822
823	#else
824	# error "port me"
825	#endif
826	}
827
828
829	/**
830	* Emits a load effective address to a GRP with an BP relative source address.
831	*/
832	DECL_INLINE_THROW(uint32_t)
833	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
834	{
835	#ifdef RT_ARCH_AMD64
836	/* lea gprdst, [rbp + offDisp] */
837	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
838	if (iGprDst < 8)
839	pbCodeBuf[off++] = X86_OP_REX_W;
840	else
841	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
842	pbCodeBuf[off++] = 0x8d;
843	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
844
845	#elif defined(RT_ARCH_ARM64)
846	if ((uint32_t)offDisp < (unsigned)_4K)
847	{
848	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
849	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);
850	}
851	else if ((uint32_t)-offDisp < (unsigned)_4K)
852	{
853	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
854	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);
855	}
856	else
857	{
858	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
859	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
860	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
861	if (offDisp >= 0)
862	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
863	else
864	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
865	}
866
867	#else
868	# error "port me"
869	#endif
870
871	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
872	return off;
873	}
874
875
876	/**
877	* Emits a 64-bit GPR store with an BP relative destination address.
878	*
879	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
880	*/
881	DECL_INLINE_THROW(uint32_t)
882	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
883	{
884	#ifdef RT_ARCH_AMD64
885	/* mov qword [rbp + offDisp], gprdst */
886	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
887	if (iGprSrc < 8)
888	pbCodeBuf[off++] = X86_OP_REX_W;
889	else
890	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
891	pbCodeBuf[off++] = 0x89;
892	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
893
894	#elif defined(RT_ARCH_ARM64)
895	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
896	{
897	/* str w/ unsigned imm12 (scaled) */
898	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
899	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
900	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
901	}
902	else if (offDisp >= -256 && offDisp <= 256)
903	{
904	/* stur w/ signed imm9 (unscaled) */
905	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
906	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
907	}
908	else
909	{
910	/* Use temporary indexing register. */
911	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
912	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
913	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
914	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
915	}
916	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
917	return off;
918
919	#else
920	# error "Port me!"
921	#endif
922	}
923
924
925	/**
926	* Emits a 64-bit immediate store with an BP relative destination address.
927	*
928	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
929	*/
930	DECL_INLINE_THROW(uint32_t)
931	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
932	{
933	#ifdef RT_ARCH_AMD64
934	if ((int64_t)uImm64 == (int32_t)uImm64)
935	{
936	/* mov qword [rbp + offDisp], imm32 - sign extended */
937	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
938	pbCodeBuf[off++] = X86_OP_REX_W;
939	pbCodeBuf[off++] = 0xc7;
940	if (offDisp < 128 && offDisp >= -128)
941	{
942	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
943	pbCodeBuf[off++] = (uint8_t)offDisp;
944	}
945	else
946	{
947	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
948	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
949	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
950	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
951	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
952	}
953	pbCodeBuf[off++] = RT_BYTE1(uImm64);
954	pbCodeBuf[off++] = RT_BYTE2(uImm64);
955	pbCodeBuf[off++] = RT_BYTE3(uImm64);
956	pbCodeBuf[off++] = RT_BYTE4(uImm64);
957	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
958	return off;
959	}
960	#endif
961
962	/* Load tmp0, imm64; Store tmp to bp+disp. */
963	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
964	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
965	}
966
967
968	#ifdef RT_ARCH_AMD64
969	/**
970	* Common bit of iemNativeEmitLoadGprByGpr and friends.
971	*/
972	DECL_FORCE_INLINE(uint32_t)
973	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
974	{
975	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
976	{
977	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
978	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
979	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
980	}
981	else if (offDisp == (int8_t)offDisp)
982	{
983	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
984	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
985	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
986	pbCodeBuf[off++] = (uint8_t)offDisp;
987	}
988	else
989	{
990	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
991	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
992	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
993	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
994	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
995	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
996	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
997	}
998	return off;
999	}
1000	#elif RT_ARCH_ARM64
1001	/**
1002	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1003	*/
1004	DECL_FORCE_INLINE_THROW(uint32_t)
1005	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1006	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1007	{
1008	/*
1009	* There are a couple of ldr variants that takes an immediate offset, so
1010	* try use those if we can, otherwise we have to use the temporary register
1011	* help with the addressing.
1012	*/
1013	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1014	{
1015	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1016	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1017	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1018	}
1019	else
1020	{
1021	/* The offset is too large, so we must load it into a register and use
1022	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1023	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1024	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1025
1026	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1027	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1028
1029	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1030	}
1031	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1032	return off;
1033	}
1034	#endif
1035
1036
1037	/**
1038	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1039	*/
1040	DECL_INLINE_THROW(uint32_t)
1041	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1042	{
1043	#ifdef RT_ARCH_AMD64
1044	/* mov reg64, mem64 */
1045	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1046	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1047	pbCodeBuf[off++] = 0x8b;
1048	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1049	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1050
1051	#elif RT_ARCH_ARM64
1052	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1053
1054	#else
1055	# error "port me"
1056	#endif
1057	return off;
1058	}
1059
1060
1061	/**
1062	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1063	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1064	*/
1065	DECL_INLINE_THROW(uint32_t)
1066	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1067	{
1068	#ifdef RT_ARCH_AMD64
1069	/* mov reg32, mem32 */
1070	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1071	if (iGprDst >= 8 \|\| iGprBase >= 8)
1072	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1073	pbCodeBuf[off++] = 0x8b;
1074	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1075	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1076
1077	#elif RT_ARCH_ARM64
1078	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1079
1080	#else
1081	# error "port me"
1082	#endif
1083	return off;
1084	}
1085
1086
1087	/*********************************************************************************************************************************
1088	* Subtraction and Additions *
1089	*********************************************************************************************************************************/
1090
1091
1092	#ifdef RT_ARCH_AMD64
1093	/**
1094	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1095	*/
1096	DECL_INLINE_THROW(uint32_t)
1097	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1098	{
1099	/* sub gprdst, imm8/imm32 */
1100	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1101	if (iGprDst < 8)
1102	pbCodeBuf[off++] = X86_OP_REX_W;
1103	else
1104	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1105	if (iSubtrahend < 128 && iSubtrahend >= -128)
1106	{
1107	pbCodeBuf[off++] = 0x83;
1108	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1109	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1110	}
1111	else
1112	{
1113	pbCodeBuf[off++] = 0x81;
1114	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1115	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1116	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1117	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1118	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1119	}
1120	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1121	return off;
1122	}
1123	#endif
1124
1125
1126	/**
1127	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1128	* @note The AMD64 version sets flags.
1129	*/
1130	DECL_INLINE_THROW(uint32_t)
1131	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1132	{
1133	#if defined(RT_ARCH_AMD64)
1134	/* add Gv,Ev */
1135	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1136	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1137	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1138	pbCodeBuf[off++] = 0x04;
1139	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1140
1141	#elif defined(RT_ARCH_ARM64)
1142	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1143	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1144
1145	#else
1146	# error "Port me"
1147	#endif
1148	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1149	return off;
1150	}
1151
1152
1153	/**
1154	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1155	*/
1156	DECL_INLINE_THROW(uint32_t)
1157	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1158	{
1159	#if defined(RT_ARCH_AMD64)
1160	/* add or inc */
1161	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1162	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1163	if (iImm8 != 1)
1164	{
1165	pbCodeBuf[off++] = 0x83;
1166	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1167	pbCodeBuf[off++] = (uint8_t)iImm8;
1168	}
1169	else
1170	{
1171	pbCodeBuf[off++] = 0xff;
1172	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1173	}
1174
1175	#elif defined(RT_ARCH_ARM64)
1176	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1177	if (iImm8 >= 0)
1178	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1179	else
1180	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1181
1182	#else
1183	# error "Port me"
1184	#endif
1185	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1186	return off;
1187	}
1188
1189
1190	/**
1191	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1192	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1193	*/
1194	DECL_INLINE_THROW(uint32_t)
1195	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1196	{
1197	#if defined(RT_ARCH_AMD64)
1198	/* add or inc */
1199	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1200	if (iGprDst >= 8)
1201	pbCodeBuf[off++] = X86_OP_REX_B;
1202	if (iImm8 != 1)
1203	{
1204	pbCodeBuf[off++] = 0x83;
1205	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1206	pbCodeBuf[off++] = (uint8_t)iImm8;
1207	}
1208	else
1209	{
1210	pbCodeBuf[off++] = 0xff;
1211	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1212	}
1213
1214	#elif defined(RT_ARCH_ARM64)
1215	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1216	if (iImm8 >= 0)
1217	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1218	else
1219	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1220
1221	#else
1222	# error "Port me"
1223	#endif
1224	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1225	return off;
1226	}
1227
1228
1229	/**
1230	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1231	*/
1232	DECL_INLINE_THROW(uint32_t)
1233	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1234	{
1235	#if defined(RT_ARCH_AMD64)
1236	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1237	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1238
1239	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1240	{
1241	/* add grp, imm32 */
1242	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1243	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1244	pbCodeBuf[off++] = 0x81;
1245	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1246	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1247	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1248	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1249	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1250	}
1251	else
1252	{
1253	/* Best to use a temporary register to deal with this in the simplest way: */
1254	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1255
1256	/* add dst, tmpreg */
1257	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1258	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1259	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1260	pbCodeBuf[off++] = 0x03;
1261	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1262
1263	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1264	}
1265
1266	#elif defined(RT_ARCH_ARM64)
1267	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1268	{
1269	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1270	if (iAddend >= 0)
1271	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1272	else
1273	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1274	}
1275	else
1276	{
1277	/* Use temporary register for the immediate. */
1278	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1279
1280	/* add gprdst, gprdst, tmpreg */
1281	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1282	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1283
1284	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1285	}
1286
1287	#else
1288	# error "Port me"
1289	#endif
1290	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1291	return off;
1292	}
1293
1294
1295	/**
1296	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1297	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1298	*/
1299	DECL_INLINE_THROW(uint32_t)
1300	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1301	{
1302	#if defined(RT_ARCH_AMD64)
1303	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1304	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1305
1306	/* add grp, imm32 */
1307	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1308	if (iGprDst >= 8)
1309	pbCodeBuf[off++] = X86_OP_REX_B;
1310	pbCodeBuf[off++] = 0x81;
1311	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1312	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1313	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1314	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1315	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1316
1317	#elif defined(RT_ARCH_ARM64)
1318	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1319	{
1320	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1321	if (iAddend >= 0)
1322	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1323	else
1324	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1325	}
1326	else
1327	{
1328	/* Use temporary register for the immediate. */
1329	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1330
1331	/* add gprdst, gprdst, tmpreg */
1332	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1333	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1334
1335	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1336	}
1337
1338	#else
1339	# error "Port me"
1340	#endif
1341	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1342	return off;
1343	}
1344
1345
1346
1347	/*********************************************************************************************************************************
1348	* Bit Operations *
1349	*********************************************************************************************************************************/
1350
1351	/**
1352	* Emits code for clearing bits 16 thru 63 in the GPR.
1353	*/
1354	DECL_INLINE_THROW(uint32_t)
1355	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1356	{
1357	#if defined(RT_ARCH_AMD64)
1358	/* movzx Gv,Ew */
1359	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1360	if (iGprDst >= 8)
1361	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1362	pbCodeBuf[off++] = 0x0f;
1363	pbCodeBuf[off++] = 0xb7;
1364	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1365
1366	#elif defined(RT_ARCH_ARM64)
1367	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1368	# if 1
1369	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1370	# else
1371	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1372	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1373	# endif
1374	#else
1375	# error "Port me"
1376	#endif
1377	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1378	return off;
1379	}
1380
1381
1382	/**
1383	* Emits code for AND'ing two 64-bit GPRs.
1384	*
1385	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1386	* and ARM64 hosts.
1387	*/
1388	DECL_INLINE_THROW(uint32_t)
1389	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1390	{
1391	#if defined(RT_ARCH_AMD64)
1392	/* and Gv, Ev */
1393	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1394	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1395	pbCodeBuf[off++] = 0x23;
1396	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1397	RT_NOREF(fSetFlags);
1398
1399	#elif defined(RT_ARCH_ARM64)
1400	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1401	if (!fSetFlags)
1402	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1403	else
1404	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
1405
1406	#else
1407	# error "Port me"
1408	#endif
1409	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1410	return off;
1411	}
1412
1413
1414	/**
1415	* Emits code for AND'ing two 32-bit GPRs.
1416	*/
1417	DECL_INLINE_THROW(uint32_t)
1418	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1419	{
1420	#if defined(RT_ARCH_AMD64)
1421	/* and Gv, Ev */
1422	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1423	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1424	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1425	pbCodeBuf[off++] = 0x23;
1426	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1427	RT_NOREF(fSetFlags);
1428
1429	#elif defined(RT_ARCH_ARM64)
1430	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1431	if (fSetFlags)
1432	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1433	else
1434	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1435
1436	#else
1437	# error "Port me"
1438	#endif
1439	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1440	return off;
1441	}
1442
1443
1444	/**
1445	* Emits code for AND'ing a 64-bit GPRs with a constant.
1446	*
1447	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1448	* and ARM64 hosts.
1449	*/
1450	DECL_INLINE_THROW(uint32_t)
1451	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
1452	{
1453	#if defined(RT_ARCH_AMD64)
1454	if ((int64_t)uImm == (int8_t)uImm)
1455	{
1456	/* and Ev, imm8 */
1457	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1458	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1459	pbCodeBuf[off++] = 0x83;
1460	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1461	pbCodeBuf[off++] = (uint8_t)uImm;
1462	}
1463	else if ((int64_t)uImm == (int32_t)uImm)
1464	{
1465	/* and Ev, imm32 */
1466	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1467	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1468	pbCodeBuf[off++] = 0x81;
1469	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1470	pbCodeBuf[off++] = RT_BYTE1(uImm);
1471	pbCodeBuf[off++] = RT_BYTE2(uImm);
1472	pbCodeBuf[off++] = RT_BYTE3(uImm);
1473	pbCodeBuf[off++] = RT_BYTE4(uImm);
1474	}
1475	else
1476	{
1477	/* Use temporary register for the 64-bit immediate. */
1478	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1479	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
1480	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1481	}
1482	RT_NOREF(fSetFlags);
1483
1484	#elif defined(RT_ARCH_ARM64)
1485	uint32_t uImmR = 0;
1486	uint32_t uImmNandS = 0;
1487	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
1488	{
1489	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1490	if (!fSetFlags)
1491	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
1492	else
1493	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
1494	}
1495	else
1496	{
1497	/* Use temporary register for the 64-bit immediate. */
1498	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1499	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1500	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1501	}
1502
1503	#else
1504	# error "Port me"
1505	#endif
1506	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1507	return off;
1508	}
1509
1510
1511	/**
1512	* Emits code for AND'ing an 32-bit GPRs with a constant.
1513	*/
1514	DECL_INLINE_THROW(uint32_t)
1515	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
1516	{
1517	#if defined(RT_ARCH_AMD64)
1518	/* and Ev, imm */
1519	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1520	if (iGprDst >= 8)
1521	pbCodeBuf[off++] = X86_OP_REX_B;
1522	if ((int32_t)uImm == (int8_t)uImm)
1523	{
1524	pbCodeBuf[off++] = 0x83;
1525	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1526	pbCodeBuf[off++] = (uint8_t)uImm;
1527	}
1528	else
1529	{
1530	pbCodeBuf[off++] = 0x81;
1531	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1532	pbCodeBuf[off++] = RT_BYTE1(uImm);
1533	pbCodeBuf[off++] = RT_BYTE2(uImm);
1534	pbCodeBuf[off++] = RT_BYTE3(uImm);
1535	pbCodeBuf[off++] = RT_BYTE4(uImm);
1536	}
1537	RT_NOREF(fSetFlags);
1538
1539	#elif defined(RT_ARCH_ARM64)
1540	uint32_t uImmR = 0;
1541	uint32_t uImmNandS = 0;
1542	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
1543	{
1544	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1545	if (!fSetFlags)
1546	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1547	else
1548	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1549	}
1550	else
1551	{
1552	/* Use temporary register for the 64-bit immediate. */
1553	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1554	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1555	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1556	}
1557
1558	#else
1559	# error "Port me"
1560	#endif
1561	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1562	return off;
1563	}
1564
1565
1566	/**
1567	* Emits code for XOR'ing two 64-bit GPRs.
1568	*/
1569	DECL_INLINE_THROW(uint32_t)
1570	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1571	{
1572	#if defined(RT_ARCH_AMD64)
1573	/* and Gv, Ev */
1574	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1575	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1576	pbCodeBuf[off++] = 0x33;
1577	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1578
1579	#elif defined(RT_ARCH_ARM64)
1580	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1581	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1582
1583	#else
1584	# error "Port me"
1585	#endif
1586	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1587	return off;
1588	}
1589
1590
1591	/**
1592	* Emits code for XOR'ing two 32-bit GPRs.
1593	*/
1594	DECL_INLINE_THROW(uint32_t)
1595	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1596	{
1597	#if defined(RT_ARCH_AMD64)
1598	/* and Gv, Ev */
1599	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1600	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1601	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1602	pbCodeBuf[off++] = 0x33;
1603	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1604
1605	#elif defined(RT_ARCH_ARM64)
1606	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1607	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1608
1609	#else
1610	# error "Port me"
1611	#endif
1612	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1613	return off;
1614	}
1615
1616
1617	/*********************************************************************************************************************************
1618	* Shifting *
1619	*********************************************************************************************************************************/
1620
1621	/**
1622	* Emits code for shifting a GPR a fixed number of bits to the left.
1623	*/
1624	DECL_INLINE_THROW(uint32_t)
1625	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1626	{
1627	Assert(cShift > 0 && cShift < 64);
1628
1629	#if defined(RT_ARCH_AMD64)
1630	/* shl dst, cShift */
1631	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1632	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1633	if (cShift != 1)
1634	{
1635	pbCodeBuf[off++] = 0xc1;
1636	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1637	pbCodeBuf[off++] = cShift;
1638	}
1639	else
1640	{
1641	pbCodeBuf[off++] = 0xd1;
1642	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1643	}
1644
1645	#elif defined(RT_ARCH_ARM64)
1646	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1647	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
1648
1649	#else
1650	# error "Port me"
1651	#endif
1652	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1653	return off;
1654	}
1655
1656
1657	/**
1658	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
1659	*/
1660	DECL_INLINE_THROW(uint32_t)
1661	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1662	{
1663	Assert(cShift > 0 && cShift < 32);
1664
1665	#if defined(RT_ARCH_AMD64)
1666	/* shl dst, cShift */
1667	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1668	if (iGprDst >= 8)
1669	pbCodeBuf[off++] = X86_OP_REX_B;
1670	if (cShift != 1)
1671	{
1672	pbCodeBuf[off++] = 0xc1;
1673	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1674	pbCodeBuf[off++] = cShift;
1675	}
1676	else
1677	{
1678	pbCodeBuf[off++] = 0xd1;
1679	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1680	}
1681
1682	#elif defined(RT_ARCH_ARM64)
1683	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1684	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
1685
1686	#else
1687	# error "Port me"
1688	#endif
1689	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1690	return off;
1691	}
1692
1693
1694	/**
1695	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1696	*/
1697	DECL_INLINE_THROW(uint32_t)
1698	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1699	{
1700	Assert(cShift > 0 && cShift < 64);
1701
1702	#if defined(RT_ARCH_AMD64)
1703	/* shr dst, cShift */
1704	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1705	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1706	if (cShift != 1)
1707	{
1708	pbCodeBuf[off++] = 0xc1;
1709	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1710	pbCodeBuf[off++] = cShift;
1711	}
1712	else
1713	{
1714	pbCodeBuf[off++] = 0xd1;
1715	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1716	}
1717
1718	#elif defined(RT_ARCH_ARM64)
1719	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1720	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1721
1722	#else
1723	# error "Port me"
1724	#endif
1725	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1726	return off;
1727	}
1728
1729
1730	/**
1731	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
1732	* right.
1733	*/
1734	DECL_INLINE_THROW(uint32_t)
1735	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1736	{
1737	Assert(cShift > 0 && cShift < 32);
1738
1739	#if defined(RT_ARCH_AMD64)
1740	/* shr dst, cShift */
1741	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1742	if (iGprDst >= 8)
1743	pbCodeBuf[off++] = X86_OP_REX_B;
1744	if (cShift != 1)
1745	{
1746	pbCodeBuf[off++] = 0xc1;
1747	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1748	pbCodeBuf[off++] = cShift;
1749	}
1750	else
1751	{
1752	pbCodeBuf[off++] = 0xd1;
1753	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1754	}
1755
1756	#elif defined(RT_ARCH_ARM64)
1757	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1758	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
1759
1760	#else
1761	# error "Port me"
1762	#endif
1763	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1764	return off;
1765	}
1766
1767
1768
1769	/*********************************************************************************************************************************
1770	* Compare and Testing *
1771	*********************************************************************************************************************************/
1772
1773
1774	#ifdef RT_ARCH_ARM64
1775	/**
1776	* Emits an ARM64 compare instruction.
1777	*/
1778	DECL_INLINE_THROW(uint32_t)
1779	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1780	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1781	{
1782	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1783	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
1784	f64Bit, true /fSetFlags/, cShift, enmShift);
1785	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1786	return off;
1787	}
1788	#endif
1789
1790
1791	/**
1792	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1793	* with conditional instruction.
1794	*/
1795	DECL_INLINE_THROW(uint32_t)
1796	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1797	{
1798	#ifdef RT_ARCH_AMD64
1799	/* cmp Gv, Ev */
1800	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1801	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1802	pbCodeBuf[off++] = 0x3b;
1803	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1804	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1805
1806	#elif defined(RT_ARCH_ARM64)
1807	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1808
1809	#else
1810	# error "Port me!"
1811	#endif
1812	return off;
1813	}
1814
1815
1816	/**
1817	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1818	* with conditional instruction.
1819	*/
1820	DECL_INLINE_THROW(uint32_t)
1821	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1822	{
1823	#ifdef RT_ARCH_AMD64
1824	/* cmp Gv, Ev */
1825	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1826	if (iGprLeft >= 8 \|\| iGprRight >= 8)
1827	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1828	pbCodeBuf[off++] = 0x3b;
1829	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1830	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1831
1832	#elif defined(RT_ARCH_ARM64)
1833	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1834
1835	#else
1836	# error "Port me!"
1837	#endif
1838	return off;
1839	}
1840
1841
1842	/**
1843	* Emits a compare of a 64-bit GPR with a constant value, settings status
1844	* flags/whatever for use with conditional instruction.
1845	*/
1846	DECL_INLINE_THROW(uint32_t)
1847	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
1848	{
1849	#ifdef RT_ARCH_AMD64
1850	if (uImm <= UINT32_C(0xff))
1851	{
1852	/* cmp Ev, Ib */
1853	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1854	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
1855	pbCodeBuf[off++] = 0x83;
1856	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
1857	pbCodeBuf[off++] = (uint8_t)uImm;
1858	}
1859	else if ((int64_t)uImm == (int32_t)uImm)
1860	{
1861	/* cmp Ev, imm */
1862	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1863	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
1864	pbCodeBuf[off++] = 0x81;
1865	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
1866	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1867	pbCodeBuf[off++] = RT_BYTE1(uImm);
1868	pbCodeBuf[off++] = RT_BYTE2(uImm);
1869	pbCodeBuf[off++] = RT_BYTE3(uImm);
1870	pbCodeBuf[off++] = RT_BYTE4(uImm);
1871	}
1872	else
1873	{
1874	/* Use temporary register for the immediate. */
1875	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1876	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
1877	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1878	}
1879
1880	#elif defined(RT_ARCH_ARM64)
1881	/** @todo guess there are clevere things we can do here... */
1882	if (uImm < _4K)
1883	{
1884	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1885	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
1886	true /64Bit/, true /fSetFlags/);
1887	}
1888	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
1889	{
1890	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1891	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
1892	true /64Bit/, true /fSetFlags/, true /fShift12/);
1893	}
1894	else
1895	{
1896	/* Use temporary register for the immediate. */
1897	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1898	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
1899	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1900	}
1901
1902	#else
1903	# error "Port me!"
1904	#endif
1905
1906	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1907	return off;
1908	}
1909
1910
1911	/**
1912	* Emits a compare of a 32-bit GPR with a constant value, settings status
1913	* flags/whatever for use with conditional instruction.
1914	*/
1915	DECL_INLINE_THROW(uint32_t)
1916	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
1917	{
1918	#ifdef RT_ARCH_AMD64
1919	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1920	if (iGprLeft >= 8)
1921	pbCodeBuf[off++] = X86_OP_REX_B;
1922	if (uImm <= UINT32_C(0xff))
1923	{
1924	/* cmp Ev, Ib */
1925	pbCodeBuf[off++] = 0x83;
1926	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
1927	pbCodeBuf[off++] = (uint8_t)uImm;
1928	}
1929	else
1930	{
1931	/* cmp Ev, imm */
1932	pbCodeBuf[off++] = 0x81;
1933	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
1934	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1935	pbCodeBuf[off++] = RT_BYTE1(uImm);
1936	pbCodeBuf[off++] = RT_BYTE2(uImm);
1937	pbCodeBuf[off++] = RT_BYTE3(uImm);
1938	pbCodeBuf[off++] = RT_BYTE4(uImm);
1939	}
1940
1941	#elif defined(RT_ARCH_ARM64)
1942	/** @todo guess there are clevere things we can do here... */
1943	if (uImm < _4K)
1944	{
1945	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1946	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
1947	false /64Bit/, true /fSetFlags/);
1948	}
1949	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
1950	{
1951	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1952	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
1953	false /64Bit/, true /fSetFlags/, true /fShift12/);
1954	}
1955	else
1956	{
1957	/* Use temporary register for the immediate. */
1958	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1959	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
1960	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1961	}
1962
1963	#else
1964	# error "Port me!"
1965	#endif
1966
1967	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1968	return off;
1969	}
1970
1971
1972
1973	/*********************************************************************************************************************************
1974	* Branching *
1975	*********************************************************************************************************************************/
1976
1977	/**
1978	* Emits a JMP rel32 / B imm19 to the given label.
1979	*/
1980	DECL_INLINE_THROW(uint32_t)
1981	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1982	{
1983	Assert(idxLabel < pReNative->cLabels);
1984
1985	#ifdef RT_ARCH_AMD64
1986	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1987	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1988	{
1989	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
1990	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
1991	{
1992	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
1993	pbCodeBuf[off++] = (uint8_t)offRel;
1994	}
1995	else
1996	{
1997	offRel -= 3;
1998	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1999	pbCodeBuf[off++] = RT_BYTE1(offRel);
2000	pbCodeBuf[off++] = RT_BYTE2(offRel);
2001	pbCodeBuf[off++] = RT_BYTE3(offRel);
2002	pbCodeBuf[off++] = RT_BYTE4(offRel);
2003	}
2004	}
2005	else
2006	{
2007	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2008	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2009	pbCodeBuf[off++] = 0xfe;
2010	pbCodeBuf[off++] = 0xff;
2011	pbCodeBuf[off++] = 0xff;
2012	pbCodeBuf[off++] = 0xff;
2013	}
2014	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2015
2016	#elif defined(RT_ARCH_ARM64)
2017	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2018	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2019	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2020	else
2021	{
2022	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2023	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2024	}
2025
2026	#else
2027	# error "Port me!"
2028	#endif
2029	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2030	return off;
2031	}
2032
2033
2034	/**
2035	* Emits a JMP rel32 / B imm19 to a new undefined label.
2036	*/
2037	DECL_INLINE_THROW(uint32_t)
2038	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2039	{
2040	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2041	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2042	}
2043
2044	/** Condition type. */
2045	#ifdef RT_ARCH_AMD64
2046	typedef enum IEMNATIVEINSTRCOND : uint8_t
2047	{
2048	kIemNativeInstrCond_o = 0,
2049	kIemNativeInstrCond_no,
2050	kIemNativeInstrCond_c,
2051	kIemNativeInstrCond_nc,
2052	kIemNativeInstrCond_e,
2053	kIemNativeInstrCond_ne,
2054	kIemNativeInstrCond_be,
2055	kIemNativeInstrCond_nbe,
2056	kIemNativeInstrCond_s,
2057	kIemNativeInstrCond_ns,
2058	kIemNativeInstrCond_p,
2059	kIemNativeInstrCond_np,
2060	kIemNativeInstrCond_l,
2061	kIemNativeInstrCond_nl,
2062	kIemNativeInstrCond_le,
2063	kIemNativeInstrCond_nle
2064	} IEMNATIVEINSTRCOND;
2065	#elif defined(RT_ARCH_ARM64)
2066	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2067	#else
2068	# error "Port me!"
2069	#endif
2070
2071
2072	/**
2073	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2074	*/
2075	DECL_INLINE_THROW(uint32_t)
2076	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2077	{
2078	Assert(idxLabel < pReNative->cLabels);
2079
2080	#ifdef RT_ARCH_AMD64
2081	/* jcc rel32 */
2082	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2083	pbCodeBuf[off++] = 0x0f;
2084	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2085	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2086	pbCodeBuf[off++] = 0x00;
2087	pbCodeBuf[off++] = 0x00;
2088	pbCodeBuf[off++] = 0x00;
2089	pbCodeBuf[off++] = 0x00;
2090
2091	#elif defined(RT_ARCH_ARM64)
2092	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2093	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2094	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2095
2096	#else
2097	# error "Port me!"
2098	#endif
2099	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2100	return off;
2101	}
2102
2103
2104	/**
2105	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2106	*/
2107	DECL_INLINE_THROW(uint32_t)
2108	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2109	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2110	{
2111	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2112	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2113	}
2114
2115
2116	/**
2117	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2118	*/
2119	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2120	{
2121	#ifdef RT_ARCH_AMD64
2122	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2123	#elif defined(RT_ARCH_ARM64)
2124	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2125	#else
2126	# error "Port me!"
2127	#endif
2128	}
2129
2130	/**
2131	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2132	*/
2133	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2134	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2135	{
2136	#ifdef RT_ARCH_AMD64
2137	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2138	#elif defined(RT_ARCH_ARM64)
2139	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2140	#else
2141	# error "Port me!"
2142	#endif
2143	}
2144
2145
2146	/**
2147	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2148	*/
2149	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2150	{
2151	#ifdef RT_ARCH_AMD64
2152	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2153	#elif defined(RT_ARCH_ARM64)
2154	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2155	#else
2156	# error "Port me!"
2157	#endif
2158	}
2159
2160	/**
2161	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2162	*/
2163	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2164	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2165	{
2166	#ifdef RT_ARCH_AMD64
2167	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2168	#elif defined(RT_ARCH_ARM64)
2169	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2170	#else
2171	# error "Port me!"
2172	#endif
2173	}
2174
2175
2176	/**
2177	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2178	*/
2179	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2180	{
2181	#ifdef RT_ARCH_AMD64
2182	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2183	#elif defined(RT_ARCH_ARM64)
2184	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2185	#else
2186	# error "Port me!"
2187	#endif
2188	}
2189
2190	/**
2191	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2192	*/
2193	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2194	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2195	{
2196	#ifdef RT_ARCH_AMD64
2197	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2198	#elif defined(RT_ARCH_ARM64)
2199	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2200	#else
2201	# error "Port me!"
2202	#endif
2203	}
2204
2205
2206	/**
2207	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2208	*/
2209	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2210	{
2211	#ifdef RT_ARCH_AMD64
2212	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2213	#elif defined(RT_ARCH_ARM64)
2214	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2215	#else
2216	# error "Port me!"
2217	#endif
2218	}
2219
2220	/**
2221	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2222	*/
2223	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2224	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2225	{
2226	#ifdef RT_ARCH_AMD64
2227	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2228	#elif defined(RT_ARCH_ARM64)
2229	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2230	#else
2231	# error "Port me!"
2232	#endif
2233	}
2234
2235
2236	/**
2237	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2238	* How @a offJmp is applied is are target specific.
2239	*/
2240	DECL_INLINE_THROW(uint32_t)
2241	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2242	{
2243	#ifdef RT_ARCH_AMD64
2244	/* jcc rel32 */
2245	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2246	if (offTarget < 128 && offTarget >= -128)
2247	{
2248	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2249	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2250	}
2251	else
2252	{
2253	pbCodeBuf[off++] = 0x0f;
2254	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2255	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2256	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2257	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2258	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2259	}
2260
2261	#elif defined(RT_ARCH_ARM64)
2262	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2263	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2264
2265	#else
2266	# error "Port me!"
2267	#endif
2268	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2269	return off;
2270	}
2271
2272
2273	/**
2274	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2275	* How @a offJmp is applied is are target specific.
2276	*/
2277	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2278	{
2279	#ifdef RT_ARCH_AMD64
2280	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2281	#elif defined(RT_ARCH_ARM64)
2282	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2283	#else
2284	# error "Port me!"
2285	#endif
2286	}
2287
2288
2289	/**
2290	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2291	* How @a offJmp is applied is are target specific.
2292	*/
2293	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2294	{
2295	#ifdef RT_ARCH_AMD64
2296	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2297	#elif defined(RT_ARCH_ARM64)
2298	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2299	#else
2300	# error "Port me!"
2301	#endif
2302	}
2303
2304
2305	/**
2306	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2307	* How @a offJmp is applied is are target specific.
2308	*/
2309	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2310	{
2311	#ifdef RT_ARCH_AMD64
2312	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2313	#elif defined(RT_ARCH_ARM64)
2314	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2315	#else
2316	# error "Port me!"
2317	#endif
2318	}
2319
2320
2321	/**
2322	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2323	* How @a offJmp is applied is are target specific.
2324	*/
2325	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2326	{
2327	#ifdef RT_ARCH_AMD64
2328	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2329	#elif defined(RT_ARCH_ARM64)
2330	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2331	#else
2332	# error "Port me!"
2333	#endif
2334	}
2335
2336
2337	/**
2338	* Fixes up a conditional jump to a fixed label.
2339	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2340	*/
2341	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2342	{
2343	# if defined(RT_ARCH_AMD64)
2344	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2345	if (pbCodeBuf[offFixup] != 0x0f)
2346	{
2347	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2348	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2349	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2350	}
2351	else
2352	{
2353	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2354	uint32_t const offRel32 = offTarget - (offFixup + 6);
2355	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2356	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2357	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2358	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2359	}
2360
2361	# elif defined(RT_ARCH_ARM64)
2362	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2363
2364	int32_t const offDisp = offTarget - offFixup;
2365	Assert(offDisp >= -262144 && offDisp < 262144);
2366	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
2367
2368	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
2369	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
2370
2371	# endif
2372	}
2373
2374
2375	/**
2376	* Internal helper, don't call directly.
2377	*/
2378	DECL_INLINE_THROW(uint32_t)
2379	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2380	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
2381	{
2382	Assert(iBitNo < 64);
2383	#ifdef RT_ARCH_AMD64
2384	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2385	if (iBitNo < 8)
2386	{
2387	/* test Eb, imm8 */
2388	if (iGprSrc >= 4)
2389	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2390	pbCodeBuf[off++] = 0xf6;
2391	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2392	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2393	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2394	}
2395	else
2396	{
2397	/* bt Ev, imm8 */
2398	if (iBitNo >= 32)
2399	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2400	else if (iGprSrc >= 8)
2401	pbCodeBuf[off++] = X86_OP_REX_B;
2402	pbCodeBuf[off++] = 0x0f;
2403	pbCodeBuf[off++] = 0xba;
2404	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2405	pbCodeBuf[off++] = iBitNo;
2406	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2407	}
2408
2409	#elif defined(RT_ARCH_ARM64)
2410	/* Use the TBNZ instruction here. */
2411	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2412	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
2413	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2414
2415	#else
2416	# error "Port me!"
2417	#endif
2418	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2419	return off;
2420	}
2421
2422
2423	/**
2424	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2425	* @a iGprSrc.
2426	*
2427	* @note On ARM64 the range is only +/-8191 instructions.
2428	*/
2429	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2430	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2431	{
2432	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
2433	}
2434
2435
2436	/**
2437	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2438	* _set_ in @a iGprSrc.
2439	*
2440	* @note On ARM64 the range is only +/-8191 instructions.
2441	*/
2442	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2443	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2444	{
2445	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
2446	}
2447
2448
2449	/**
2450	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2451	* flags accordingly.
2452	*/
2453	DECL_INLINE_THROW(uint32_t)
2454	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2455	{
2456	Assert(fBits != 0);
2457	#ifdef RT_ARCH_AMD64
2458
2459	if (fBits >= UINT32_MAX)
2460	{
2461	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2462
2463	/* test Ev,Gv */
2464	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2465	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2466	pbCodeBuf[off++] = 0x85;
2467	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2468
2469	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2470	}
2471	else if (fBits <= UINT32_MAX)
2472	{
2473	/* test Eb, imm8 or test Ev, imm32 */
2474	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2475	if (fBits <= UINT8_MAX)
2476	{
2477	if (iGprSrc >= 4)
2478	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2479	pbCodeBuf[off++] = 0xf6;
2480	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2481	pbCodeBuf[off++] = (uint8_t)fBits;
2482	}
2483	else
2484	{
2485	if (iGprSrc >= 8)
2486	pbCodeBuf[off++] = X86_OP_REX_B;
2487	pbCodeBuf[off++] = 0xf7;
2488	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2489	pbCodeBuf[off++] = RT_BYTE1(fBits);
2490	pbCodeBuf[off++] = RT_BYTE2(fBits);
2491	pbCodeBuf[off++] = RT_BYTE3(fBits);
2492	pbCodeBuf[off++] = RT_BYTE4(fBits);
2493	}
2494	}
2495	/** @todo implement me. */
2496	else
2497	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
2498
2499	#elif defined(RT_ARCH_ARM64)
2500
2501	if (false)
2502	{
2503	/** @todo figure out how to work the immr / N:imms constants. */
2504	}
2505	else
2506	{
2507	/* ands Zr, iGprSrc, iTmpReg */
2508	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2509	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2510	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2511	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2512	}
2513
2514	#else
2515	# error "Port me!"
2516	#endif
2517	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2518	return off;
2519	}
2520
2521
2522	/**
2523	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2524	* are set in @a iGprSrc.
2525	*/
2526	DECL_INLINE_THROW(uint32_t)
2527	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2528	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2529	{
2530	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2531
2532	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2533	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2534
2535	return off;
2536	}
2537
2538
2539	/**
2540	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2541	* are set in @a iGprSrc.
2542	*/
2543	DECL_INLINE_THROW(uint32_t)
2544	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2545	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2546	{
2547	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2548
2549	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2550	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2551
2552	return off;
2553	}
2554
2555
2556	/**
2557	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
2558	*
2559	* The operand size is given by @a f64Bit.
2560	*/
2561	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2562	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
2563	{
2564	Assert(idxLabel < pReNative->cLabels);
2565
2566	#ifdef RT_ARCH_AMD64
2567	/* test reg32,reg32 / test reg64,reg64 */
2568	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2569	if (f64Bit)
2570	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
2571	else if (iGprSrc >= 8)
2572	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
2573	pbCodeBuf[off++] = 0x85;
2574	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
2575	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2576
2577	/* jz idxLabel */
2578	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2579
2580	#elif defined(RT_ARCH_ARM64)
2581	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2582	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2583	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
2584	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2585
2586	#else
2587	# error "Port me!"
2588	#endif
2589	return off;
2590	}
2591
2592
2593	/**
2594	* Emits code that jumps to a new label if @a iGprSrc is zero.
2595	*
2596	* The operand size is given by @a f64Bit.
2597	*/
2598	DECL_INLINE_THROW(uint32_t)
2599	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
2600	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2601	{
2602	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2603	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
2604	}
2605
2606
2607	/**
2608	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
2609	* differs.
2610	*/
2611	DECL_INLINE_THROW(uint32_t)
2612	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2613	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
2614	{
2615	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
2616	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2617	return off;
2618	}
2619
2620
2621	/**
2622	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
2623	*/
2624	DECL_INLINE_THROW(uint32_t)
2625	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2626	uint8_t iGprLeft, uint8_t iGprRight,
2627	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2628	{
2629	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2630	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
2631	}
2632
2633
2634	/**
2635	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
2636	*/
2637	DECL_INLINE_THROW(uint32_t)
2638	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2639	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
2640	{
2641	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
2642	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2643	return off;
2644	}
2645
2646
2647	/**
2648	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
2649	*/
2650	DECL_INLINE_THROW(uint32_t)
2651	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2652	uint8_t iGprSrc, uint64_t uImm,
2653	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2654	{
2655	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2656	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2657	}
2658
2659
2660	/**
2661	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
2662	* @a uImm.
2663	*/
2664	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2665	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
2666	{
2667	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
2668	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2669	return off;
2670	}
2671
2672
2673	/**
2674	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
2675	* @a uImm.
2676	*/
2677	DECL_INLINE_THROW(uint32_t)
2678	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2679	uint8_t iGprSrc, uint32_t uImm,
2680	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2681	{
2682	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2683	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2684	}
2685
2686
2687
2688	/**
2689	* Emits a call to a 64-bit address.
2690	*/
2691	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2692	{
2693	#ifdef RT_ARCH_AMD64
2694	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2695
2696	/* call rax */
2697	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2698	pbCodeBuf[off++] = 0xff;
2699	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2700
2701	#elif defined(RT_ARCH_ARM64)
2702	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2703
2704	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2705	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2706
2707	#else
2708	# error "port me"
2709	#endif
2710	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2711	return off;
2712	}
2713
2714
2715	/** @} */
2716
2717	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
2718

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102022

Download in other formats: