IEMN8veRecompilerEmit.h@ 102424

Last change on this file since 102424 was 102394, checked in by vboxsync, 17 months ago
VMM/IEM: Refactored the native IEM_MC_FETCH_MEM_Uxx code so it can be shared with the store MCs and is a little bit more efficient. Found 2 instructions that wasn't getting FLAT variants because they were using hardcoded addresses (Ov/Ob). Implemented a simple solution for the instruction counting over longjmp/throw. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 98.0 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102394 2023-11-30 13:28:53Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif RT_ARCH_ARM64
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/*********************************************************************************************************************************
82	* Loads, Stores and Related Stuff. *
83	*********************************************************************************************************************************/
84
85	/**
86	* Emits setting a GPR to zero.
87	*/
88	DECL_INLINE_THROW(uint32_t)
89	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
90	{
91	#ifdef RT_ARCH_AMD64
92	/* xor gpr32, gpr32 */
93	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
94	if (iGpr >= 8)
95	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
96	pbCodeBuf[off++] = 0x33;
97	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
98
99	#elif RT_ARCH_ARM64
100	/* mov gpr, #0x0 */
101	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
102	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
103
104	#else
105	# error "port me"
106	#endif
107	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
108	return off;
109	}
110
111
112	/**
113	* Emits loading a constant into a 64-bit GPR
114	*/
115	DECL_INLINE_THROW(uint32_t)
116	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
117	{
118	if (!uImm64)
119	return iemNativeEmitGprZero(pReNative, off, iGpr);
120
121	#ifdef RT_ARCH_AMD64
122	if (uImm64 <= UINT32_MAX)
123	{
124	/* mov gpr, imm32 */
125	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
126	if (iGpr >= 8)
127	pbCodeBuf[off++] = X86_OP_REX_B;
128	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
129	pbCodeBuf[off++] = RT_BYTE1(uImm64);
130	pbCodeBuf[off++] = RT_BYTE2(uImm64);
131	pbCodeBuf[off++] = RT_BYTE3(uImm64);
132	pbCodeBuf[off++] = RT_BYTE4(uImm64);
133	}
134	else
135	{
136	/* mov gpr, imm64 */
137	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
138	if (iGpr < 8)
139	pbCodeBuf[off++] = X86_OP_REX_W;
140	else
141	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
142	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
143	pbCodeBuf[off++] = RT_BYTE1(uImm64);
144	pbCodeBuf[off++] = RT_BYTE2(uImm64);
145	pbCodeBuf[off++] = RT_BYTE3(uImm64);
146	pbCodeBuf[off++] = RT_BYTE4(uImm64);
147	pbCodeBuf[off++] = RT_BYTE5(uImm64);
148	pbCodeBuf[off++] = RT_BYTE6(uImm64);
149	pbCodeBuf[off++] = RT_BYTE7(uImm64);
150	pbCodeBuf[off++] = RT_BYTE8(uImm64);
151	}
152
153	#elif RT_ARCH_ARM64
154	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
155
156	/*
157	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
158	* supply remaining bits using 'movk grp, imm16, lsl #x'.
159	*
160	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
161	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
162	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
163	* after the first non-zero immediate component so we switch to movk for
164	* the remainder.
165	*/
166	uint32_t fMovK = 0;
167	/* mov gpr, imm16 */
168	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
169	if (uImmPart)
170	{
171	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
172	fMovK \|= RT_BIT_32(29);
173	}
174	/* mov[k] gpr, imm16, lsl #16 */
175	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
176	if (uImmPart)
177	{
178	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
179	fMovK \|= RT_BIT_32(29);
180	}
181	/* mov[k] gpr, imm16, lsl #32 */
182	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
183	if (uImmPart)
184	{
185	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
186	fMovK \|= RT_BIT_32(29);
187	}
188	/* mov[k] gpr, imm16, lsl #48 */
189	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
190	if (uImmPart)
191	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
192
193	/** @todo there is an inverted mask variant we might want to explore if it
194	* reduces the number of instructions... */
195	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
196	* clang 12.x does that, only to use the 'x' version for the
197	* addressing in the following ldr). */
198
199	#else
200	# error "port me"
201	#endif
202	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
203	return off;
204	}
205
206
207	/**
208	* Emits loading a constant into a 8-bit GPR
209	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
210	* only the ARM64 version does that.
211	*/
212	DECL_INLINE_THROW(uint32_t)
213	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
214	{
215	#ifdef RT_ARCH_AMD64
216	/* mov gpr, imm8 */
217	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
218	if (iGpr >= 8)
219	pbCodeBuf[off++] = X86_OP_REX_B;
220	else if (iGpr >= 4)
221	pbCodeBuf[off++] = X86_OP_REX;
222	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
223	pbCodeBuf[off++] = RT_BYTE1(uImm8);
224
225	#elif RT_ARCH_ARM64
226	/* movz gpr, imm16, lsl #0 */
227	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
228	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
229
230	#else
231	# error "port me"
232	#endif
233	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
234	return off;
235	}
236
237
238	#ifdef RT_ARCH_AMD64
239	/**
240	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
241	*/
242	DECL_FORCE_INLINE(uint32_t)
243	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
244	{
245	if (offVCpu < 128)
246	{
247	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
248	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
249	}
250	else
251	{
252	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
253	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
254	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
255	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
256	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
257	}
258	return off;
259	}
260	#elif RT_ARCH_ARM64
261	/**
262	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
263	*/
264	DECL_FORCE_INLINE_THROW(uint32_t)
265	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
266	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
267	{
268	/*
269	* There are a couple of ldr variants that takes an immediate offset, so
270	* try use those if we can, otherwise we have to use the temporary register
271	* help with the addressing.
272	*/
273	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
274	{
275	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
276	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
277	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
278	}
279	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
280	{
281	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
282	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
283	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
284	}
285	else
286	{
287	/* The offset is too large, so we must load it into a register and use
288	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
289	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
290	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
291
292	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
293	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
294	IEMNATIVE_REG_FIXED_TMP0);
295	}
296	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
297	return off;
298	}
299	#endif
300
301
302	/**
303	* Emits a 64-bit GPR load of a VCpu value.
304	*/
305	DECL_INLINE_THROW(uint32_t)
306	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
307	{
308	#ifdef RT_ARCH_AMD64
309	/* mov reg64, mem64 */
310	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
311	if (iGpr < 8)
312	pbCodeBuf[off++] = X86_OP_REX_W;
313	else
314	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
315	pbCodeBuf[off++] = 0x8b;
316	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
317	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
318
319	#elif RT_ARCH_ARM64
320	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
321
322	#else
323	# error "port me"
324	#endif
325	return off;
326	}
327
328
329	/**
330	* Emits a 32-bit GPR load of a VCpu value.
331	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
332	*/
333	DECL_INLINE_THROW(uint32_t)
334	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
335	{
336	#ifdef RT_ARCH_AMD64
337	/* mov reg32, mem32 */
338	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
339	if (iGpr >= 8)
340	pbCodeBuf[off++] = X86_OP_REX_R;
341	pbCodeBuf[off++] = 0x8b;
342	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
343	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
344
345	#elif RT_ARCH_ARM64
346	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
347
348	#else
349	# error "port me"
350	#endif
351	return off;
352	}
353
354
355	/**
356	* Emits a 16-bit GPR load of a VCpu value.
357	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
358	*/
359	DECL_INLINE_THROW(uint32_t)
360	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
361	{
362	#ifdef RT_ARCH_AMD64
363	/* movzx reg32, mem16 */
364	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
365	if (iGpr >= 8)
366	pbCodeBuf[off++] = X86_OP_REX_R;
367	pbCodeBuf[off++] = 0x0f;
368	pbCodeBuf[off++] = 0xb7;
369	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
370	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
371
372	#elif RT_ARCH_ARM64
373	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
374
375	#else
376	# error "port me"
377	#endif
378	return off;
379	}
380
381
382	/**
383	* Emits a 8-bit GPR load of a VCpu value.
384	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
385	*/
386	DECL_INLINE_THROW(uint32_t)
387	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
388	{
389	#ifdef RT_ARCH_AMD64
390	/* movzx reg32, mem8 */
391	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
392	if (iGpr >= 8)
393	pbCodeBuf[off++] = X86_OP_REX_R;
394	pbCodeBuf[off++] = 0x0f;
395	pbCodeBuf[off++] = 0xb6;
396	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
397	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
398
399	#elif RT_ARCH_ARM64
400	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
401
402	#else
403	# error "port me"
404	#endif
405	return off;
406	}
407
408
409	/**
410	* Emits a store of a GPR value to a 64-bit VCpu field.
411	*/
412	DECL_INLINE_THROW(uint32_t)
413	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
414	{
415	#ifdef RT_ARCH_AMD64
416	/* mov mem64, reg64 */
417	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
418	if (iGpr < 8)
419	pbCodeBuf[off++] = X86_OP_REX_W;
420	else
421	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
422	pbCodeBuf[off++] = 0x89;
423	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
424	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
425
426	#elif RT_ARCH_ARM64
427	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
428
429	#else
430	# error "port me"
431	#endif
432	return off;
433	}
434
435
436	/**
437	* Emits a store of a GPR value to a 32-bit VCpu field.
438	*/
439	DECL_INLINE_THROW(uint32_t)
440	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
441	{
442	#ifdef RT_ARCH_AMD64
443	/* mov mem32, reg32 */
444	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
445	if (iGpr >= 8)
446	pbCodeBuf[off++] = X86_OP_REX_R;
447	pbCodeBuf[off++] = 0x89;
448	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
449	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
450
451	#elif RT_ARCH_ARM64
452	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
453
454	#else
455	# error "port me"
456	#endif
457	return off;
458	}
459
460
461	/**
462	* Emits a store of a GPR value to a 16-bit VCpu field.
463	*/
464	DECL_INLINE_THROW(uint32_t)
465	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
466	{
467	#ifdef RT_ARCH_AMD64
468	/* mov mem16, reg16 */
469	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
470	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
471	if (iGpr >= 8)
472	pbCodeBuf[off++] = X86_OP_REX_R;
473	pbCodeBuf[off++] = 0x89;
474	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
475	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
476
477	#elif RT_ARCH_ARM64
478	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
479
480	#else
481	# error "port me"
482	#endif
483	return off;
484	}
485
486
487	/**
488	* Emits a store of a GPR value to a 8-bit VCpu field.
489	*/
490	DECL_INLINE_THROW(uint32_t)
491	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
492	{
493	#ifdef RT_ARCH_AMD64
494	/* mov mem8, reg8 */
495	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
496	if (iGpr >= 8)
497	pbCodeBuf[off++] = X86_OP_REX_R;
498	pbCodeBuf[off++] = 0x88;
499	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
500	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
501
502	#elif RT_ARCH_ARM64
503	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
504
505	#else
506	# error "port me"
507	#endif
508	return off;
509	}
510
511
512	/**
513	* Emits a store of an immediate value to a 8-bit VCpu field.
514	*/
515	DECL_INLINE_THROW(uint32_t)
516	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
517	{
518	#ifdef RT_ARCH_AMD64
519	/* mov mem8, imm8 */
520	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
521	pbCodeBuf[off++] = 0xc6;
522	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
523	pbCodeBuf[off++] = bImm;
524	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
525
526	#elif RT_ARCH_ARM64
527	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
528	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
529	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
530	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
531
532	#else
533	# error "port me"
534	#endif
535	return off;
536	}
537
538
539	/**
540	* Emits a load effective address to a GRP of a VCpu field.
541	*/
542	DECL_INLINE_THROW(uint32_t)
543	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
544	{
545	#ifdef RT_ARCH_AMD64
546	/* lea gprdst, [rbx + offDisp] */
547	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
548	if (iGprDst < 8)
549	pbCodeBuf[off++] = X86_OP_REX_W;
550	else
551	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
552	pbCodeBuf[off++] = 0x8d;
553	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
554
555	#elif defined(RT_ARCH_ARM64)
556	if (offVCpu < (unsigned)_4K)
557	{
558	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
559	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
560	}
561	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
562	{
563	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
564	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
565	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
566	}
567	else
568	{
569	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
570	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
571	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
572	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
573	}
574
575	#else
576	# error "port me"
577	#endif
578	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
579	return off;
580	}
581
582
583	/**
584	* Emits a gprdst = gprsrc load.
585	*/
586	DECL_INLINE_THROW(uint32_t)
587	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
588	{
589	#ifdef RT_ARCH_AMD64
590	/* mov gprdst, gprsrc */
591	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
592	if ((iGprDst \| iGprSrc) >= 8)
593	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
594	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
595	: X86_OP_REX_W \| X86_OP_REX_R;
596	else
597	pbCodeBuf[off++] = X86_OP_REX_W;
598	pbCodeBuf[off++] = 0x8b;
599	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
600
601	#elif RT_ARCH_ARM64
602	/* mov dst, src; alias for: orr dst, xzr, src */
603	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
604	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
605
606	#else
607	# error "port me"
608	#endif
609	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
610	return off;
611	}
612
613
614	/**
615	* Emits a gprdst = gprsrc[31:0] load.
616	* @note Bits 63 thru 32 are cleared.
617	*/
618	DECL_INLINE_THROW(uint32_t)
619	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
620	{
621	#ifdef RT_ARCH_AMD64
622	/* mov gprdst, gprsrc */
623	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
624	if ((iGprDst \| iGprSrc) >= 8)
625	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
626	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
627	: X86_OP_REX_R;
628	pbCodeBuf[off++] = 0x8b;
629	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
630
631	#elif RT_ARCH_ARM64
632	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
633	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
634	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
635
636	#else
637	# error "port me"
638	#endif
639	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
640	return off;
641	}
642
643
644	/**
645	* Emits a gprdst = gprsrc[15:0] load.
646	* @note Bits 63 thru 15 are cleared.
647	*/
648	DECL_INLINE_THROW(uint32_t)
649	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
650	{
651	#ifdef RT_ARCH_AMD64
652	/* movzx Gv,Ew */
653	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
654	if ((iGprDst \| iGprSrc) >= 8)
655	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
656	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
657	: X86_OP_REX_R;
658	pbCodeBuf[off++] = 0x0f;
659	pbCodeBuf[off++] = 0xb7;
660	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
661
662	#elif RT_ARCH_ARM64
663	/* and gprdst, gprsrc, #0xffff */
664	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
665	# if 1
666	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
667	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
668	# else
669	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
670	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
671	# endif
672
673	#else
674	# error "port me"
675	#endif
676	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
677	return off;
678	}
679
680
681	/**
682	* Emits a gprdst = gprsrc[7:0] load.
683	* @note Bits 63 thru 8 are cleared.
684	*/
685	DECL_INLINE_THROW(uint32_t)
686	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
687	{
688	#ifdef RT_ARCH_AMD64
689	/* movzx Gv,Eb */
690	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
691	if (iGprDst >= 8 \|\| iGprSrc >= 8)
692	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
693	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
694	: X86_OP_REX_R;
695	else if (iGprSrc >= 4)
696	pbCodeBuf[off++] = X86_OP_REX;
697	pbCodeBuf[off++] = 0x0f;
698	pbCodeBuf[off++] = 0xb6;
699	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
700
701	#elif RT_ARCH_ARM64
702	/* and gprdst, gprsrc, #0xff */
703	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
704	# if 1
705	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
706	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
707	# else
708	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
709	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
710	# endif
711
712	#else
713	# error "port me"
714	#endif
715	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
716	return off;
717	}
718
719
720	/**
721	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
722	* @note Bits 63 thru 8 are cleared.
723	*/
724	DECL_INLINE_THROW(uint32_t)
725	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
726	{
727	#ifdef RT_ARCH_AMD64
728	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
729
730	/* movzx Gv,Ew */
731	if ((iGprDst \| iGprSrc) >= 8)
732	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
733	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
734	: X86_OP_REX_R;
735	pbCodeBuf[off++] = 0x0f;
736	pbCodeBuf[off++] = 0xb7;
737	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
738
739	/* shr Ev,8 */
740	if (iGprDst >= 8)
741	pbCodeBuf[off++] = X86_OP_REX_B;
742	pbCodeBuf[off++] = 0xc1;
743	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
744	pbCodeBuf[off++] = 8;
745
746	#elif RT_ARCH_ARM64
747	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
748	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
749	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
750
751	#else
752	# error "port me"
753	#endif
754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
755	return off;
756	}
757
758
759	/**
760	* Emits a gprdst = gprsrc + addend load.
761	*/
762	DECL_INLINE_THROW(uint32_t)
763	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
764	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
765	{
766	Assert(iAddend != 0);
767
768	#ifdef RT_ARCH_AMD64
769	/* lea gprdst, [gprsrc + iAddend] */
770	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
771	if ((iGprDst \| iGprSrc) >= 8)
772	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
773	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
774	: X86_OP_REX_W \| X86_OP_REX_R;
775	else
776	pbCodeBuf[off++] = X86_OP_REX_W;
777	pbCodeBuf[off++] = 0x8d;
778	if (iAddend >= -128 && iAddend < 128)
779	{
780	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprDst & 7, iGprSrc & 7);
781	pbCodeBuf[off++] = (int8_t)iAddend;
782	}
783	else
784	{
785	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprDst & 7, iGprSrc & 7);
786	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
787	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
788	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
789	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
790	}
791
792	#elif RT_ARCH_ARM64
793	if ((uint32_t)iAddend < 4096)
794	{
795	/* add dst, src, uimm12 */
796	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
797	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
798	}
799	else if ((uint32_t)-iAddend < 4096)
800	{
801	/* sub dst, src, uimm12 */
802	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
803	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
804	}
805	else
806	{
807	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
808	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
809	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
810	}
811
812	#else
813	# error "port me"
814	#endif
815	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
816	return off;
817	}
818
819
820	#ifdef RT_ARCH_AMD64
821	/**
822	* Common bit of iemNativeEmitLoadGprByBp and friends.
823	*/
824	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
825	PIEMRECOMPILERSTATE pReNativeAssert)
826	{
827	if (offDisp < 128 && offDisp >= -128)
828	{
829	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
830	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
831	}
832	else
833	{
834	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
835	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
836	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
837	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
838	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
839	}
840	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
841	return off;
842	}
843	#elif defined(RT_ARCH_ARM64)
844	/**
845	* Common bit of iemNativeEmitLoadGprByBp and friends.
846	*/
847	DECL_FORCE_INLINE_THROW(uint32_t)
848	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
849	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
850	{
851	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
852	{
853	/* str w/ unsigned imm12 (scaled) */
854	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
855	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
856	}
857	else if (offDisp >= -256 && offDisp <= 256)
858	{
859	/* stur w/ signed imm9 (unscaled) */
860	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
861	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
862	}
863	else
864	{
865	/* Use temporary indexing register. */
866	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
867	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
868	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
869	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
870	}
871	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
872	return off;
873	}
874	#endif
875
876
877	/**
878	* Emits a 64-bit GRP load instruction with an BP relative source address.
879	*/
880	DECL_INLINE_THROW(uint32_t)
881	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
882	{
883	#ifdef RT_ARCH_AMD64
884	/* mov gprdst, qword [rbp + offDisp] */
885	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
886	if (iGprDst < 8)
887	pbCodeBuf[off++] = X86_OP_REX_W;
888	else
889	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
890	pbCodeBuf[off++] = 0x8b;
891	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
892
893	#elif defined(RT_ARCH_ARM64)
894	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
895
896	#else
897	# error "port me"
898	#endif
899	}
900
901
902	/**
903	* Emits a 32-bit GRP load instruction with an BP relative source address.
904	* @note Bits 63 thru 32 of the GPR will be cleared.
905	*/
906	DECL_INLINE_THROW(uint32_t)
907	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
908	{
909	#ifdef RT_ARCH_AMD64
910	/* mov gprdst, dword [rbp + offDisp] */
911	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
912	if (iGprDst >= 8)
913	pbCodeBuf[off++] = X86_OP_REX_R;
914	pbCodeBuf[off++] = 0x8b;
915	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
916
917	#elif defined(RT_ARCH_ARM64)
918	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
919
920	#else
921	# error "port me"
922	#endif
923	}
924
925
926	/**
927	* Emits a 16-bit GRP load instruction with an BP relative source address.
928	* @note Bits 63 thru 16 of the GPR will be cleared.
929	*/
930	DECL_INLINE_THROW(uint32_t)
931	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
932	{
933	#ifdef RT_ARCH_AMD64
934	/* movzx gprdst, word [rbp + offDisp] */
935	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
936	if (iGprDst >= 8)
937	pbCodeBuf[off++] = X86_OP_REX_R;
938	pbCodeBuf[off++] = 0xb7;
939	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
940
941	#elif defined(RT_ARCH_ARM64)
942	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
943
944	#else
945	# error "port me"
946	#endif
947	}
948
949
950	/**
951	* Emits a 8-bit GRP load instruction with an BP relative source address.
952	* @note Bits 63 thru 8 of the GPR will be cleared.
953	*/
954	DECL_INLINE_THROW(uint32_t)
955	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
956	{
957	#ifdef RT_ARCH_AMD64
958	/* movzx gprdst, byte [rbp + offDisp] */
959	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
960	if (iGprDst >= 8)
961	pbCodeBuf[off++] = X86_OP_REX_R;
962	pbCodeBuf[off++] = 0xb6;
963	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
964
965	#elif defined(RT_ARCH_ARM64)
966	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
967
968	#else
969	# error "port me"
970	#endif
971	}
972
973
974	/**
975	* Emits a load effective address to a GRP with an BP relative source address.
976	*/
977	DECL_INLINE_THROW(uint32_t)
978	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
979	{
980	#ifdef RT_ARCH_AMD64
981	/* lea gprdst, [rbp + offDisp] */
982	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
983	if (iGprDst < 8)
984	pbCodeBuf[off++] = X86_OP_REX_W;
985	else
986	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
987	pbCodeBuf[off++] = 0x8d;
988	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
989
990	#elif defined(RT_ARCH_ARM64)
991	if ((uint32_t)offDisp < (unsigned)_4K)
992	{
993	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
994	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
995	}
996	else if ((uint32_t)-offDisp < (unsigned)_4K)
997	{
998	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
999	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1000	}
1001	else
1002	{
1003	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1004	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1005	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1006	if (offDisp >= 0)
1007	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1008	else
1009	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1010	}
1011
1012	#else
1013	# error "port me"
1014	#endif
1015
1016	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1017	return off;
1018	}
1019
1020
1021	/**
1022	* Emits a 64-bit GPR store with an BP relative destination address.
1023	*
1024	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1025	*/
1026	DECL_INLINE_THROW(uint32_t)
1027	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1028	{
1029	#ifdef RT_ARCH_AMD64
1030	/* mov qword [rbp + offDisp], gprdst */
1031	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1032	if (iGprSrc < 8)
1033	pbCodeBuf[off++] = X86_OP_REX_W;
1034	else
1035	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1036	pbCodeBuf[off++] = 0x89;
1037	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1038
1039	#elif defined(RT_ARCH_ARM64)
1040	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1041	{
1042	/* str w/ unsigned imm12 (scaled) */
1043	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1044	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1045	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1046	}
1047	else if (offDisp >= -256 && offDisp <= 256)
1048	{
1049	/* stur w/ signed imm9 (unscaled) */
1050	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1051	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1052	}
1053	else if ((uint32_t)-offDisp < (unsigned)_4K)
1054	{
1055	/* Use temporary indexing register w/ sub uimm12. */
1056	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1057	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1058	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1059	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1060	}
1061	else
1062	{
1063	/* Use temporary indexing register. */
1064	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1065	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1066	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1067	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1068	}
1069	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1070	return off;
1071
1072	#else
1073	# error "Port me!"
1074	#endif
1075	}
1076
1077
1078	/**
1079	* Emits a 64-bit immediate store with an BP relative destination address.
1080	*
1081	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1082	*/
1083	DECL_INLINE_THROW(uint32_t)
1084	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1085	{
1086	#ifdef RT_ARCH_AMD64
1087	if ((int64_t)uImm64 == (int32_t)uImm64)
1088	{
1089	/* mov qword [rbp + offDisp], imm32 - sign extended */
1090	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1091	pbCodeBuf[off++] = X86_OP_REX_W;
1092	pbCodeBuf[off++] = 0xc7;
1093	if (offDisp < 128 && offDisp >= -128)
1094	{
1095	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1096	pbCodeBuf[off++] = (uint8_t)offDisp;
1097	}
1098	else
1099	{
1100	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1101	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1102	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1103	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1104	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1105	}
1106	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1107	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1108	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1109	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1110	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1111	return off;
1112	}
1113	#endif
1114
1115	/* Load tmp0, imm64; Store tmp to bp+disp. */
1116	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1117	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1118	}
1119
1120
1121	#ifdef RT_ARCH_AMD64
1122	/**
1123	* Common bit of iemNativeEmitLoadGprByGpr and friends.
1124	*/
1125	DECL_FORCE_INLINE(uint32_t)
1126	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1127	{
1128	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1129	{
1130	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1131	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1132	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1133	}
1134	else if (offDisp == (int8_t)offDisp)
1135	{
1136	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1137	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1138	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1139	pbCodeBuf[off++] = (uint8_t)offDisp;
1140	}
1141	else
1142	{
1143	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1144	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1145	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1146	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1147	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1148	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1149	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1150	}
1151	return off;
1152	}
1153	#elif RT_ARCH_ARM64
1154	/**
1155	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1156	*/
1157	DECL_FORCE_INLINE_THROW(uint32_t)
1158	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1159	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1160	{
1161	/*
1162	* There are a couple of ldr variants that takes an immediate offset, so
1163	* try use those if we can, otherwise we have to use the temporary register
1164	* help with the addressing.
1165	*/
1166	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1167	{
1168	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1169	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1170	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1171	}
1172	else
1173	{
1174	/* The offset is too large, so we must load it into a register and use
1175	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1176	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1177	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1178
1179	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1180	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1181
1182	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1183	}
1184	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1185	return off;
1186	}
1187	#endif
1188
1189
1190	/**
1191	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1192	*/
1193	DECL_INLINE_THROW(uint32_t)
1194	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1195	{
1196	#ifdef RT_ARCH_AMD64
1197	/* mov reg64, mem64 */
1198	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1199	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1200	pbCodeBuf[off++] = 0x8b;
1201	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1202	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1203
1204	#elif RT_ARCH_ARM64
1205	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1206
1207	#else
1208	# error "port me"
1209	#endif
1210	return off;
1211	}
1212
1213
1214	/**
1215	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1216	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1217	*/
1218	DECL_INLINE_THROW(uint32_t)
1219	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1220	{
1221	#ifdef RT_ARCH_AMD64
1222	/* mov reg32, mem32 */
1223	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1224	if (iGprDst >= 8 \|\| iGprBase >= 8)
1225	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1226	pbCodeBuf[off++] = 0x8b;
1227	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1228	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1229
1230	#elif RT_ARCH_ARM64
1231	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1232
1233	#else
1234	# error "port me"
1235	#endif
1236	return off;
1237	}
1238
1239
1240	/*********************************************************************************************************************************
1241	* Subtraction and Additions *
1242	*********************************************************************************************************************************/
1243
1244
1245	#ifdef RT_ARCH_AMD64
1246	/**
1247	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1248	*/
1249	DECL_INLINE_THROW(uint32_t)
1250	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1251	{
1252	/* sub gprdst, imm8/imm32 */
1253	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1254	if (iGprDst < 8)
1255	pbCodeBuf[off++] = X86_OP_REX_W;
1256	else
1257	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1258	if (iSubtrahend < 128 && iSubtrahend >= -128)
1259	{
1260	pbCodeBuf[off++] = 0x83;
1261	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1262	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1263	}
1264	else
1265	{
1266	pbCodeBuf[off++] = 0x81;
1267	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1268	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1269	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1270	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1271	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1272	}
1273	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1274	return off;
1275	}
1276	#endif
1277
1278
1279	/**
1280	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1281	* @note The AMD64 version sets flags.
1282	*/
1283	DECL_INLINE_THROW(uint32_t)
1284	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1285	{
1286	#if defined(RT_ARCH_AMD64)
1287	/* add Gv,Ev */
1288	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1289	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1290	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1291	pbCodeBuf[off++] = 0x04;
1292	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1293
1294	#elif defined(RT_ARCH_ARM64)
1295	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1296	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1297
1298	#else
1299	# error "Port me"
1300	#endif
1301	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1302	return off;
1303	}
1304
1305
1306	/**
1307	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1308	*/
1309	DECL_INLINE_THROW(uint32_t)
1310	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1311	{
1312	#if defined(RT_ARCH_AMD64)
1313	/* add or inc */
1314	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1315	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1316	if (iImm8 != 1)
1317	{
1318	pbCodeBuf[off++] = 0x83;
1319	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1320	pbCodeBuf[off++] = (uint8_t)iImm8;
1321	}
1322	else
1323	{
1324	pbCodeBuf[off++] = 0xff;
1325	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1326	}
1327
1328	#elif defined(RT_ARCH_ARM64)
1329	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1330	if (iImm8 >= 0)
1331	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1332	else
1333	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1334
1335	#else
1336	# error "Port me"
1337	#endif
1338	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1339	return off;
1340	}
1341
1342
1343	/**
1344	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1345	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1346	*/
1347	DECL_INLINE_THROW(uint32_t)
1348	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1349	{
1350	#if defined(RT_ARCH_AMD64)
1351	/* add or inc */
1352	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1353	if (iGprDst >= 8)
1354	pbCodeBuf[off++] = X86_OP_REX_B;
1355	if (iImm8 != 1)
1356	{
1357	pbCodeBuf[off++] = 0x83;
1358	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1359	pbCodeBuf[off++] = (uint8_t)iImm8;
1360	}
1361	else
1362	{
1363	pbCodeBuf[off++] = 0xff;
1364	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1365	}
1366
1367	#elif defined(RT_ARCH_ARM64)
1368	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1369	if (iImm8 >= 0)
1370	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1371	else
1372	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1373
1374	#else
1375	# error "Port me"
1376	#endif
1377	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1378	return off;
1379	}
1380
1381
1382	/**
1383	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1384	*/
1385	DECL_INLINE_THROW(uint32_t)
1386	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1387	{
1388	#if defined(RT_ARCH_AMD64)
1389	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1390	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1391
1392	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1393	{
1394	/* add grp, imm32 */
1395	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1396	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1397	pbCodeBuf[off++] = 0x81;
1398	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1399	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1400	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1401	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1402	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1403	}
1404	else
1405	{
1406	/* Best to use a temporary register to deal with this in the simplest way: */
1407	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1408
1409	/* add dst, tmpreg */
1410	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1411	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1412	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1413	pbCodeBuf[off++] = 0x03;
1414	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1415
1416	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1417	}
1418
1419	#elif defined(RT_ARCH_ARM64)
1420	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1421	{
1422	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1423	if (iAddend >= 0)
1424	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1425	else
1426	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1427	}
1428	else
1429	{
1430	/* Use temporary register for the immediate. */
1431	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1432
1433	/* add gprdst, gprdst, tmpreg */
1434	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1435	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1436
1437	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1438	}
1439
1440	#else
1441	# error "Port me"
1442	#endif
1443	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1444	return off;
1445	}
1446
1447
1448	/**
1449	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1450	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1451	*/
1452	DECL_INLINE_THROW(uint32_t)
1453	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1454	{
1455	#if defined(RT_ARCH_AMD64)
1456	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1457	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1458
1459	/* add grp, imm32 */
1460	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1461	if (iGprDst >= 8)
1462	pbCodeBuf[off++] = X86_OP_REX_B;
1463	pbCodeBuf[off++] = 0x81;
1464	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1465	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1466	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1467	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1468	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1469
1470	#elif defined(RT_ARCH_ARM64)
1471	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1472	{
1473	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1474	if (iAddend >= 0)
1475	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1476	else
1477	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1478	}
1479	else
1480	{
1481	/* Use temporary register for the immediate. */
1482	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1483
1484	/* add gprdst, gprdst, tmpreg */
1485	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1486	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1487
1488	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1489	}
1490
1491	#else
1492	# error "Port me"
1493	#endif
1494	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1495	return off;
1496	}
1497
1498
1499
1500	/*********************************************************************************************************************************
1501	* Bit Operations *
1502	*********************************************************************************************************************************/
1503
1504	/**
1505	* Emits code for clearing bits 16 thru 63 in the GPR.
1506	*/
1507	DECL_INLINE_THROW(uint32_t)
1508	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1509	{
1510	#if defined(RT_ARCH_AMD64)
1511	/* movzx Gv,Ew */
1512	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1513	if (iGprDst >= 8)
1514	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1515	pbCodeBuf[off++] = 0x0f;
1516	pbCodeBuf[off++] = 0xb7;
1517	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1518
1519	#elif defined(RT_ARCH_ARM64)
1520	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1521	# if 1
1522	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1523	# else
1524	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1525	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1526	# endif
1527	#else
1528	# error "Port me"
1529	#endif
1530	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1531	return off;
1532	}
1533
1534
1535	/**
1536	* Emits code for AND'ing two 64-bit GPRs.
1537	*
1538	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1539	* and ARM64 hosts.
1540	*/
1541	DECL_INLINE_THROW(uint32_t)
1542	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1543	{
1544	#if defined(RT_ARCH_AMD64)
1545	/* and Gv, Ev */
1546	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1547	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1548	pbCodeBuf[off++] = 0x23;
1549	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1550	RT_NOREF(fSetFlags);
1551
1552	#elif defined(RT_ARCH_ARM64)
1553	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1554	if (!fSetFlags)
1555	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1556	else
1557	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
1558
1559	#else
1560	# error "Port me"
1561	#endif
1562	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1563	return off;
1564	}
1565
1566
1567	/**
1568	* Emits code for AND'ing two 32-bit GPRs.
1569	*/
1570	DECL_INLINE_THROW(uint32_t)
1571	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1572	{
1573	#if defined(RT_ARCH_AMD64)
1574	/* and Gv, Ev */
1575	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1576	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1577	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1578	pbCodeBuf[off++] = 0x23;
1579	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1580	RT_NOREF(fSetFlags);
1581
1582	#elif defined(RT_ARCH_ARM64)
1583	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1584	if (fSetFlags)
1585	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1586	else
1587	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1588
1589	#else
1590	# error "Port me"
1591	#endif
1592	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1593	return off;
1594	}
1595
1596
1597	/**
1598	* Emits code for AND'ing a 64-bit GPRs with a constant.
1599	*
1600	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1601	* and ARM64 hosts.
1602	*/
1603	DECL_INLINE_THROW(uint32_t)
1604	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
1605	{
1606	#if defined(RT_ARCH_AMD64)
1607	if ((int64_t)uImm == (int8_t)uImm)
1608	{
1609	/* and Ev, imm8 */
1610	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1611	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1612	pbCodeBuf[off++] = 0x83;
1613	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1614	pbCodeBuf[off++] = (uint8_t)uImm;
1615	}
1616	else if ((int64_t)uImm == (int32_t)uImm)
1617	{
1618	/* and Ev, imm32 */
1619	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1620	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1621	pbCodeBuf[off++] = 0x81;
1622	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1623	pbCodeBuf[off++] = RT_BYTE1(uImm);
1624	pbCodeBuf[off++] = RT_BYTE2(uImm);
1625	pbCodeBuf[off++] = RT_BYTE3(uImm);
1626	pbCodeBuf[off++] = RT_BYTE4(uImm);
1627	}
1628	else
1629	{
1630	/* Use temporary register for the 64-bit immediate. */
1631	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1632	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
1633	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1634	}
1635	RT_NOREF(fSetFlags);
1636
1637	#elif defined(RT_ARCH_ARM64)
1638	uint32_t uImmR = 0;
1639	uint32_t uImmNandS = 0;
1640	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
1641	{
1642	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1643	if (!fSetFlags)
1644	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
1645	else
1646	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
1647	}
1648	else
1649	{
1650	/* Use temporary register for the 64-bit immediate. */
1651	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1652	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1653	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1654	}
1655
1656	#else
1657	# error "Port me"
1658	#endif
1659	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1660	return off;
1661	}
1662
1663
1664	/**
1665	* Emits code for AND'ing an 32-bit GPRs with a constant.
1666	*/
1667	DECL_INLINE_THROW(uint32_t)
1668	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
1669	{
1670	#if defined(RT_ARCH_AMD64)
1671	/* and Ev, imm */
1672	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1673	if (iGprDst >= 8)
1674	pbCodeBuf[off++] = X86_OP_REX_B;
1675	if ((int32_t)uImm == (int8_t)uImm)
1676	{
1677	pbCodeBuf[off++] = 0x83;
1678	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1679	pbCodeBuf[off++] = (uint8_t)uImm;
1680	}
1681	else
1682	{
1683	pbCodeBuf[off++] = 0x81;
1684	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1685	pbCodeBuf[off++] = RT_BYTE1(uImm);
1686	pbCodeBuf[off++] = RT_BYTE2(uImm);
1687	pbCodeBuf[off++] = RT_BYTE3(uImm);
1688	pbCodeBuf[off++] = RT_BYTE4(uImm);
1689	}
1690	RT_NOREF(fSetFlags);
1691
1692	#elif defined(RT_ARCH_ARM64)
1693	uint32_t uImmR = 0;
1694	uint32_t uImmNandS = 0;
1695	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
1696	{
1697	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1698	if (!fSetFlags)
1699	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1700	else
1701	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1702	}
1703	else
1704	{
1705	/* Use temporary register for the 64-bit immediate. */
1706	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1707	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1708	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1709	}
1710
1711	#else
1712	# error "Port me"
1713	#endif
1714	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1715	return off;
1716	}
1717
1718
1719	/**
1720	* Emits code for XOR'ing two 64-bit GPRs.
1721	*/
1722	DECL_INLINE_THROW(uint32_t)
1723	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1724	{
1725	#if defined(RT_ARCH_AMD64)
1726	/* and Gv, Ev */
1727	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1728	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1729	pbCodeBuf[off++] = 0x33;
1730	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1731
1732	#elif defined(RT_ARCH_ARM64)
1733	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1734	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1735
1736	#else
1737	# error "Port me"
1738	#endif
1739	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1740	return off;
1741	}
1742
1743
1744	/**
1745	* Emits code for XOR'ing two 32-bit GPRs.
1746	*/
1747	DECL_INLINE_THROW(uint32_t)
1748	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1749	{
1750	#if defined(RT_ARCH_AMD64)
1751	/* and Gv, Ev */
1752	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1753	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1754	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1755	pbCodeBuf[off++] = 0x33;
1756	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1757
1758	#elif defined(RT_ARCH_ARM64)
1759	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1760	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1761
1762	#else
1763	# error "Port me"
1764	#endif
1765	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1766	return off;
1767	}
1768
1769
1770	/*********************************************************************************************************************************
1771	* Shifting *
1772	*********************************************************************************************************************************/
1773
1774	/**
1775	* Emits code for shifting a GPR a fixed number of bits to the left.
1776	*/
1777	DECL_INLINE_THROW(uint32_t)
1778	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1779	{
1780	Assert(cShift > 0 && cShift < 64);
1781
1782	#if defined(RT_ARCH_AMD64)
1783	/* shl dst, cShift */
1784	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1785	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1786	if (cShift != 1)
1787	{
1788	pbCodeBuf[off++] = 0xc1;
1789	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1790	pbCodeBuf[off++] = cShift;
1791	}
1792	else
1793	{
1794	pbCodeBuf[off++] = 0xd1;
1795	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1796	}
1797
1798	#elif defined(RT_ARCH_ARM64)
1799	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1800	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
1801
1802	#else
1803	# error "Port me"
1804	#endif
1805	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1806	return off;
1807	}
1808
1809
1810	/**
1811	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
1812	*/
1813	DECL_INLINE_THROW(uint32_t)
1814	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1815	{
1816	Assert(cShift > 0 && cShift < 32);
1817
1818	#if defined(RT_ARCH_AMD64)
1819	/* shl dst, cShift */
1820	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1821	if (iGprDst >= 8)
1822	pbCodeBuf[off++] = X86_OP_REX_B;
1823	if (cShift != 1)
1824	{
1825	pbCodeBuf[off++] = 0xc1;
1826	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1827	pbCodeBuf[off++] = cShift;
1828	}
1829	else
1830	{
1831	pbCodeBuf[off++] = 0xd1;
1832	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1833	}
1834
1835	#elif defined(RT_ARCH_ARM64)
1836	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1837	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
1838
1839	#else
1840	# error "Port me"
1841	#endif
1842	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1843	return off;
1844	}
1845
1846
1847	/**
1848	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1849	*/
1850	DECL_INLINE_THROW(uint32_t)
1851	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1852	{
1853	Assert(cShift > 0 && cShift < 64);
1854
1855	#if defined(RT_ARCH_AMD64)
1856	/* shr dst, cShift */
1857	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1858	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1859	if (cShift != 1)
1860	{
1861	pbCodeBuf[off++] = 0xc1;
1862	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1863	pbCodeBuf[off++] = cShift;
1864	}
1865	else
1866	{
1867	pbCodeBuf[off++] = 0xd1;
1868	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1869	}
1870
1871	#elif defined(RT_ARCH_ARM64)
1872	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1873	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1874
1875	#else
1876	# error "Port me"
1877	#endif
1878	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1879	return off;
1880	}
1881
1882
1883	/**
1884	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
1885	* right.
1886	*/
1887	DECL_INLINE_THROW(uint32_t)
1888	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1889	{
1890	Assert(cShift > 0 && cShift < 32);
1891
1892	#if defined(RT_ARCH_AMD64)
1893	/* shr dst, cShift */
1894	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1895	if (iGprDst >= 8)
1896	pbCodeBuf[off++] = X86_OP_REX_B;
1897	if (cShift != 1)
1898	{
1899	pbCodeBuf[off++] = 0xc1;
1900	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1901	pbCodeBuf[off++] = cShift;
1902	}
1903	else
1904	{
1905	pbCodeBuf[off++] = 0xd1;
1906	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1907	}
1908
1909	#elif defined(RT_ARCH_ARM64)
1910	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1911	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
1912
1913	#else
1914	# error "Port me"
1915	#endif
1916	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1917	return off;
1918	}
1919
1920
1921
1922	/*********************************************************************************************************************************
1923	* Compare and Testing *
1924	*********************************************************************************************************************************/
1925
1926
1927	#ifdef RT_ARCH_ARM64
1928	/**
1929	* Emits an ARM64 compare instruction.
1930	*/
1931	DECL_INLINE_THROW(uint32_t)
1932	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1933	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1934	{
1935	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1936	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
1937	f64Bit, true /fSetFlags/, cShift, enmShift);
1938	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1939	return off;
1940	}
1941	#endif
1942
1943
1944	/**
1945	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1946	* with conditional instruction.
1947	*/
1948	DECL_INLINE_THROW(uint32_t)
1949	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1950	{
1951	#ifdef RT_ARCH_AMD64
1952	/* cmp Gv, Ev */
1953	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1954	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1955	pbCodeBuf[off++] = 0x3b;
1956	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1957	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1958
1959	#elif defined(RT_ARCH_ARM64)
1960	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1961
1962	#else
1963	# error "Port me!"
1964	#endif
1965	return off;
1966	}
1967
1968
1969	/**
1970	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1971	* with conditional instruction.
1972	*/
1973	DECL_INLINE_THROW(uint32_t)
1974	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1975	{
1976	#ifdef RT_ARCH_AMD64
1977	/* cmp Gv, Ev */
1978	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1979	if (iGprLeft >= 8 \|\| iGprRight >= 8)
1980	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1981	pbCodeBuf[off++] = 0x3b;
1982	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1983	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1984
1985	#elif defined(RT_ARCH_ARM64)
1986	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1987
1988	#else
1989	# error "Port me!"
1990	#endif
1991	return off;
1992	}
1993
1994
1995	/**
1996	* Emits a compare of a 64-bit GPR with a constant value, settings status
1997	* flags/whatever for use with conditional instruction.
1998	*/
1999	DECL_INLINE_THROW(uint32_t)
2000	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2001	{
2002	#ifdef RT_ARCH_AMD64
2003	if (uImm <= UINT32_C(0xff))
2004	{
2005	/* cmp Ev, Ib */
2006	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2007	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2008	pbCodeBuf[off++] = 0x83;
2009	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2010	pbCodeBuf[off++] = (uint8_t)uImm;
2011	}
2012	else if ((int64_t)uImm == (int32_t)uImm)
2013	{
2014	/* cmp Ev, imm */
2015	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2016	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2017	pbCodeBuf[off++] = 0x81;
2018	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2019	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2020	pbCodeBuf[off++] = RT_BYTE1(uImm);
2021	pbCodeBuf[off++] = RT_BYTE2(uImm);
2022	pbCodeBuf[off++] = RT_BYTE3(uImm);
2023	pbCodeBuf[off++] = RT_BYTE4(uImm);
2024	}
2025	else
2026	{
2027	/* Use temporary register for the immediate. */
2028	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2029	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2030	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2031	}
2032
2033	#elif defined(RT_ARCH_ARM64)
2034	/** @todo guess there are clevere things we can do here... */
2035	if (uImm < _4K)
2036	{
2037	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2038	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2039	true /64Bit/, true /fSetFlags/);
2040	}
2041	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2042	{
2043	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2044	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2045	true /64Bit/, true /fSetFlags/, true /fShift12/);
2046	}
2047	else
2048	{
2049	/* Use temporary register for the immediate. */
2050	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2051	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2052	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2053	}
2054
2055	#else
2056	# error "Port me!"
2057	#endif
2058
2059	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2060	return off;
2061	}
2062
2063
2064	/**
2065	* Emits a compare of a 32-bit GPR with a constant value, settings status
2066	* flags/whatever for use with conditional instruction.
2067	*/
2068	DECL_INLINE_THROW(uint32_t)
2069	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2070	{
2071	#ifdef RT_ARCH_AMD64
2072	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2073	if (iGprLeft >= 8)
2074	pbCodeBuf[off++] = X86_OP_REX_B;
2075	if (uImm <= UINT32_C(0xff))
2076	{
2077	/* cmp Ev, Ib */
2078	pbCodeBuf[off++] = 0x83;
2079	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2080	pbCodeBuf[off++] = (uint8_t)uImm;
2081	}
2082	else
2083	{
2084	/* cmp Ev, imm */
2085	pbCodeBuf[off++] = 0x81;
2086	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2087	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2088	pbCodeBuf[off++] = RT_BYTE1(uImm);
2089	pbCodeBuf[off++] = RT_BYTE2(uImm);
2090	pbCodeBuf[off++] = RT_BYTE3(uImm);
2091	pbCodeBuf[off++] = RT_BYTE4(uImm);
2092	}
2093
2094	#elif defined(RT_ARCH_ARM64)
2095	/** @todo guess there are clevere things we can do here... */
2096	if (uImm < _4K)
2097	{
2098	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2099	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2100	false /64Bit/, true /fSetFlags/);
2101	}
2102	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2103	{
2104	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2105	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2106	false /64Bit/, true /fSetFlags/, true /fShift12/);
2107	}
2108	else
2109	{
2110	/* Use temporary register for the immediate. */
2111	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2112	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2113	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2114	}
2115
2116	#else
2117	# error "Port me!"
2118	#endif
2119
2120	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2121	return off;
2122	}
2123
2124
2125
2126	/*********************************************************************************************************************************
2127	* Branching *
2128	*********************************************************************************************************************************/
2129
2130	/**
2131	* Emits a JMP rel32 / B imm19 to the given label.
2132	*/
2133	DECL_INLINE_THROW(uint32_t)
2134	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2135	{
2136	Assert(idxLabel < pReNative->cLabels);
2137
2138	#ifdef RT_ARCH_AMD64
2139	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2140	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2141	{
2142	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2143	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2144	{
2145	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2146	pbCodeBuf[off++] = (uint8_t)offRel;
2147	}
2148	else
2149	{
2150	offRel -= 3;
2151	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2152	pbCodeBuf[off++] = RT_BYTE1(offRel);
2153	pbCodeBuf[off++] = RT_BYTE2(offRel);
2154	pbCodeBuf[off++] = RT_BYTE3(offRel);
2155	pbCodeBuf[off++] = RT_BYTE4(offRel);
2156	}
2157	}
2158	else
2159	{
2160	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2161	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2162	pbCodeBuf[off++] = 0xfe;
2163	pbCodeBuf[off++] = 0xff;
2164	pbCodeBuf[off++] = 0xff;
2165	pbCodeBuf[off++] = 0xff;
2166	}
2167	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2168
2169	#elif defined(RT_ARCH_ARM64)
2170	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2171	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2172	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2173	else
2174	{
2175	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2176	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2177	}
2178
2179	#else
2180	# error "Port me!"
2181	#endif
2182	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2183	return off;
2184	}
2185
2186
2187	/**
2188	* Emits a JMP rel32 / B imm19 to a new undefined label.
2189	*/
2190	DECL_INLINE_THROW(uint32_t)
2191	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2192	{
2193	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2194	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2195	}
2196
2197	/** Condition type. */
2198	#ifdef RT_ARCH_AMD64
2199	typedef enum IEMNATIVEINSTRCOND : uint8_t
2200	{
2201	kIemNativeInstrCond_o = 0,
2202	kIemNativeInstrCond_no,
2203	kIemNativeInstrCond_c,
2204	kIemNativeInstrCond_nc,
2205	kIemNativeInstrCond_e,
2206	kIemNativeInstrCond_ne,
2207	kIemNativeInstrCond_be,
2208	kIemNativeInstrCond_nbe,
2209	kIemNativeInstrCond_s,
2210	kIemNativeInstrCond_ns,
2211	kIemNativeInstrCond_p,
2212	kIemNativeInstrCond_np,
2213	kIemNativeInstrCond_l,
2214	kIemNativeInstrCond_nl,
2215	kIemNativeInstrCond_le,
2216	kIemNativeInstrCond_nle
2217	} IEMNATIVEINSTRCOND;
2218	#elif defined(RT_ARCH_ARM64)
2219	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2220	#else
2221	# error "Port me!"
2222	#endif
2223
2224
2225	/**
2226	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2227	*/
2228	DECL_INLINE_THROW(uint32_t)
2229	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2230	{
2231	Assert(idxLabel < pReNative->cLabels);
2232
2233	#ifdef RT_ARCH_AMD64
2234	/* jcc rel32 */
2235	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2236	pbCodeBuf[off++] = 0x0f;
2237	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2238	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2239	pbCodeBuf[off++] = 0x00;
2240	pbCodeBuf[off++] = 0x00;
2241	pbCodeBuf[off++] = 0x00;
2242	pbCodeBuf[off++] = 0x00;
2243
2244	#elif defined(RT_ARCH_ARM64)
2245	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2246	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2247	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2248
2249	#else
2250	# error "Port me!"
2251	#endif
2252	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2253	return off;
2254	}
2255
2256
2257	/**
2258	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2259	*/
2260	DECL_INLINE_THROW(uint32_t)
2261	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2262	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2263	{
2264	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2265	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2266	}
2267
2268
2269	/**
2270	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2271	*/
2272	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2273	{
2274	#ifdef RT_ARCH_AMD64
2275	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2276	#elif defined(RT_ARCH_ARM64)
2277	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2278	#else
2279	# error "Port me!"
2280	#endif
2281	}
2282
2283	/**
2284	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2285	*/
2286	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2287	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2288	{
2289	#ifdef RT_ARCH_AMD64
2290	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2291	#elif defined(RT_ARCH_ARM64)
2292	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2293	#else
2294	# error "Port me!"
2295	#endif
2296	}
2297
2298
2299	/**
2300	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2301	*/
2302	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2303	{
2304	#ifdef RT_ARCH_AMD64
2305	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2306	#elif defined(RT_ARCH_ARM64)
2307	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2308	#else
2309	# error "Port me!"
2310	#endif
2311	}
2312
2313	/**
2314	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2315	*/
2316	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2317	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2318	{
2319	#ifdef RT_ARCH_AMD64
2320	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2321	#elif defined(RT_ARCH_ARM64)
2322	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2323	#else
2324	# error "Port me!"
2325	#endif
2326	}
2327
2328
2329	/**
2330	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2331	*/
2332	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2333	{
2334	#ifdef RT_ARCH_AMD64
2335	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2336	#elif defined(RT_ARCH_ARM64)
2337	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2338	#else
2339	# error "Port me!"
2340	#endif
2341	}
2342
2343	/**
2344	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2345	*/
2346	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2347	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2348	{
2349	#ifdef RT_ARCH_AMD64
2350	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2351	#elif defined(RT_ARCH_ARM64)
2352	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2353	#else
2354	# error "Port me!"
2355	#endif
2356	}
2357
2358
2359	/**
2360	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2361	*/
2362	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2363	{
2364	#ifdef RT_ARCH_AMD64
2365	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2366	#elif defined(RT_ARCH_ARM64)
2367	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2368	#else
2369	# error "Port me!"
2370	#endif
2371	}
2372
2373	/**
2374	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2375	*/
2376	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2377	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2378	{
2379	#ifdef RT_ARCH_AMD64
2380	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2381	#elif defined(RT_ARCH_ARM64)
2382	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2383	#else
2384	# error "Port me!"
2385	#endif
2386	}
2387
2388
2389	/**
2390	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2391	* How @a offJmp is applied is are target specific.
2392	*/
2393	DECL_INLINE_THROW(uint32_t)
2394	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2395	{
2396	#ifdef RT_ARCH_AMD64
2397	/* jcc rel32 */
2398	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2399	if (offTarget < 128 && offTarget >= -128)
2400	{
2401	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2402	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2403	}
2404	else
2405	{
2406	pbCodeBuf[off++] = 0x0f;
2407	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2408	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2409	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2410	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2411	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2412	}
2413
2414	#elif defined(RT_ARCH_ARM64)
2415	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2416	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2417
2418	#else
2419	# error "Port me!"
2420	#endif
2421	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2422	return off;
2423	}
2424
2425
2426	/**
2427	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2428	* How @a offJmp is applied is are target specific.
2429	*/
2430	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2431	{
2432	#ifdef RT_ARCH_AMD64
2433	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2434	#elif defined(RT_ARCH_ARM64)
2435	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2436	#else
2437	# error "Port me!"
2438	#endif
2439	}
2440
2441
2442	/**
2443	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2444	* How @a offJmp is applied is are target specific.
2445	*/
2446	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2447	{
2448	#ifdef RT_ARCH_AMD64
2449	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2450	#elif defined(RT_ARCH_ARM64)
2451	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2452	#else
2453	# error "Port me!"
2454	#endif
2455	}
2456
2457
2458	/**
2459	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2460	* How @a offJmp is applied is are target specific.
2461	*/
2462	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2463	{
2464	#ifdef RT_ARCH_AMD64
2465	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2466	#elif defined(RT_ARCH_ARM64)
2467	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2468	#else
2469	# error "Port me!"
2470	#endif
2471	}
2472
2473
2474	/**
2475	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2476	* How @a offJmp is applied is are target specific.
2477	*/
2478	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2479	{
2480	#ifdef RT_ARCH_AMD64
2481	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2482	#elif defined(RT_ARCH_ARM64)
2483	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2484	#else
2485	# error "Port me!"
2486	#endif
2487	}
2488
2489
2490	/**
2491	* Fixes up a conditional jump to a fixed label.
2492	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2493	*/
2494	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2495	{
2496	# if defined(RT_ARCH_AMD64)
2497	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2498	if (pbCodeBuf[offFixup] != 0x0f)
2499	{
2500	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2501	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2502	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2503	}
2504	else
2505	{
2506	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2507	uint32_t const offRel32 = offTarget - (offFixup + 6);
2508	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2509	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2510	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2511	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2512	}
2513
2514	# elif defined(RT_ARCH_ARM64)
2515	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2516
2517	int32_t const offDisp = offTarget - offFixup;
2518	Assert(offDisp >= -262144 && offDisp < 262144);
2519	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
2520
2521	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
2522	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
2523
2524	# endif
2525	}
2526
2527
2528	/**
2529	* Internal helper, don't call directly.
2530	*/
2531	DECL_INLINE_THROW(uint32_t)
2532	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2533	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
2534	{
2535	Assert(iBitNo < 64);
2536	#ifdef RT_ARCH_AMD64
2537	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2538	if (iBitNo < 8)
2539	{
2540	/* test Eb, imm8 */
2541	if (iGprSrc >= 4)
2542	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2543	pbCodeBuf[off++] = 0xf6;
2544	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2545	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2546	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2547	}
2548	else
2549	{
2550	/* bt Ev, imm8 */
2551	if (iBitNo >= 32)
2552	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2553	else if (iGprSrc >= 8)
2554	pbCodeBuf[off++] = X86_OP_REX_B;
2555	pbCodeBuf[off++] = 0x0f;
2556	pbCodeBuf[off++] = 0xba;
2557	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2558	pbCodeBuf[off++] = iBitNo;
2559	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2560	}
2561
2562	#elif defined(RT_ARCH_ARM64)
2563	/* Use the TBNZ instruction here. */
2564	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2565	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
2566	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2567
2568	#else
2569	# error "Port me!"
2570	#endif
2571	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2572	return off;
2573	}
2574
2575
2576	/**
2577	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2578	* @a iGprSrc.
2579	*
2580	* @note On ARM64 the range is only +/-8191 instructions.
2581	*/
2582	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2583	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2584	{
2585	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
2586	}
2587
2588
2589	/**
2590	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2591	* _set_ in @a iGprSrc.
2592	*
2593	* @note On ARM64 the range is only +/-8191 instructions.
2594	*/
2595	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2596	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2597	{
2598	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
2599	}
2600
2601
2602	/**
2603	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2604	* flags accordingly.
2605	*/
2606	DECL_INLINE_THROW(uint32_t)
2607	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2608	{
2609	Assert(fBits != 0);
2610	#ifdef RT_ARCH_AMD64
2611
2612	if (fBits >= UINT32_MAX)
2613	{
2614	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2615
2616	/* test Ev,Gv */
2617	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2618	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2619	pbCodeBuf[off++] = 0x85;
2620	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2621
2622	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2623	}
2624	else if (fBits <= UINT32_MAX)
2625	{
2626	/* test Eb, imm8 or test Ev, imm32 */
2627	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2628	if (fBits <= UINT8_MAX)
2629	{
2630	if (iGprSrc >= 4)
2631	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2632	pbCodeBuf[off++] = 0xf6;
2633	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2634	pbCodeBuf[off++] = (uint8_t)fBits;
2635	}
2636	else
2637	{
2638	if (iGprSrc >= 8)
2639	pbCodeBuf[off++] = X86_OP_REX_B;
2640	pbCodeBuf[off++] = 0xf7;
2641	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2642	pbCodeBuf[off++] = RT_BYTE1(fBits);
2643	pbCodeBuf[off++] = RT_BYTE2(fBits);
2644	pbCodeBuf[off++] = RT_BYTE3(fBits);
2645	pbCodeBuf[off++] = RT_BYTE4(fBits);
2646	}
2647	}
2648	/** @todo implement me. */
2649	else
2650	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
2651
2652	#elif defined(RT_ARCH_ARM64)
2653
2654	if (false)
2655	{
2656	/** @todo figure out how to work the immr / N:imms constants. */
2657	}
2658	else
2659	{
2660	/* ands Zr, iGprSrc, iTmpReg */
2661	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2662	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2663	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2664	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2665	}
2666
2667	#else
2668	# error "Port me!"
2669	#endif
2670	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2671	return off;
2672	}
2673
2674
2675	/**
2676	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2677	* are set in @a iGprSrc.
2678	*/
2679	DECL_INLINE_THROW(uint32_t)
2680	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2681	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2682	{
2683	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2684
2685	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2686	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2687
2688	return off;
2689	}
2690
2691
2692	/**
2693	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2694	* are set in @a iGprSrc.
2695	*/
2696	DECL_INLINE_THROW(uint32_t)
2697	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2698	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2699	{
2700	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2701
2702	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2703	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2704
2705	return off;
2706	}
2707
2708
2709	/**
2710	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
2711	*
2712	* The operand size is given by @a f64Bit.
2713	*/
2714	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2715	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
2716	{
2717	Assert(idxLabel < pReNative->cLabels);
2718
2719	#ifdef RT_ARCH_AMD64
2720	/* test reg32,reg32 / test reg64,reg64 */
2721	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2722	if (f64Bit)
2723	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
2724	else if (iGprSrc >= 8)
2725	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
2726	pbCodeBuf[off++] = 0x85;
2727	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
2728	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2729
2730	/* jz idxLabel */
2731	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2732
2733	#elif defined(RT_ARCH_ARM64)
2734	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2735	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2736	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
2737	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2738
2739	#else
2740	# error "Port me!"
2741	#endif
2742	return off;
2743	}
2744
2745
2746	/**
2747	* Emits code that jumps to a new label if @a iGprSrc is zero.
2748	*
2749	* The operand size is given by @a f64Bit.
2750	*/
2751	DECL_INLINE_THROW(uint32_t)
2752	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
2753	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2754	{
2755	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2756	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
2757	}
2758
2759
2760	/**
2761	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
2762	* differs.
2763	*/
2764	DECL_INLINE_THROW(uint32_t)
2765	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2766	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
2767	{
2768	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
2769	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2770	return off;
2771	}
2772
2773
2774	/**
2775	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
2776	*/
2777	DECL_INLINE_THROW(uint32_t)
2778	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2779	uint8_t iGprLeft, uint8_t iGprRight,
2780	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2781	{
2782	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2783	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
2784	}
2785
2786
2787	/**
2788	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
2789	*/
2790	DECL_INLINE_THROW(uint32_t)
2791	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2792	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
2793	{
2794	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
2795	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2796	return off;
2797	}
2798
2799
2800	/**
2801	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
2802	*/
2803	DECL_INLINE_THROW(uint32_t)
2804	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2805	uint8_t iGprSrc, uint64_t uImm,
2806	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2807	{
2808	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2809	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2810	}
2811
2812
2813	/**
2814	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
2815	* @a uImm.
2816	*/
2817	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2818	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
2819	{
2820	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
2821	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2822	return off;
2823	}
2824
2825
2826	/**
2827	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
2828	* @a uImm.
2829	*/
2830	DECL_INLINE_THROW(uint32_t)
2831	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2832	uint8_t iGprSrc, uint32_t uImm,
2833	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2834	{
2835	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2836	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
2837	}
2838
2839
2840
2841	/**
2842	* Emits a call to a 64-bit address.
2843	*/
2844	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2845	{
2846	#ifdef RT_ARCH_AMD64
2847	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2848
2849	/* call rax */
2850	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2851	pbCodeBuf[off++] = 0xff;
2852	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2853
2854	#elif defined(RT_ARCH_ARM64)
2855	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2856
2857	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2858	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2859
2860	#else
2861	# error "port me"
2862	#endif
2863	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2864	return off;
2865	}
2866
2867
2868	/** @} */
2869
2870	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
2871

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102424

Download in other formats: