IEMN8veRecompilerEmit.h@ 102479

Last change on this file since 102479 was 102447, checked in by vboxsync, 15 months ago
VMM/IEM: movsx & movzx from registers. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 110.2 KB

Line
1	/* $Id: IEMN8veRecompilerEmit.h 102447 2023-12-04 13:05:53Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35
36
37	/** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters
38	* @ingroup grp_iem_n8ve_re
39	* @{
40	*/
41
42	/**
43	* Emit a simple marker instruction to more easily tell where something starts
44	* in the disassembly.
45	*/
46	DECL_INLINE_THROW(uint32_t)
47	iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
48	{
49	#ifdef RT_ARCH_AMD64
50	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
51	if (uInfo == 0)
52	{
53	/* nop */
54	pbCodeBuf[off++] = 0x90;
55	}
56	else
57	{
58	/* nop [disp32] */
59	pbCodeBuf[off++] = 0x0f;
60	pbCodeBuf[off++] = 0x1f;
61	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
62	pbCodeBuf[off++] = RT_BYTE1(uInfo);
63	pbCodeBuf[off++] = RT_BYTE2(uInfo);
64	pbCodeBuf[off++] = RT_BYTE3(uInfo);
65	pbCodeBuf[off++] = RT_BYTE4(uInfo);
66	}
67	#elif defined(RT_ARCH_ARM64)
68	/* nop */
69	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
70	pu32CodeBuf[off++] = 0xd503201f;
71
72	RT_NOREF(uInfo);
73	#else
74	# error "port me"
75	#endif
76	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
77	return off;
78	}
79
80
81	/**
82	* Emit a breakpoint instruction.
83	*/
84	DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
85	{
86	#ifdef RT_ARCH_AMD64
87	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
88	pbCodeBuf[off++] = 0xcc;
89	RT_NOREF(uInfo);
90
91	#elif defined(RT_ARCH_ARM64)
92	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
93	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
94
95	#else
96	# error "error"
97	#endif
98	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
99	return off;
100	}
101
102
103	/*********************************************************************************************************************************
104	* Loads, Stores and Related Stuff. *
105	*********************************************************************************************************************************/
106
107	/**
108	* Emits setting a GPR to zero.
109	*/
110	DECL_INLINE_THROW(uint32_t)
111	iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
112	{
113	#ifdef RT_ARCH_AMD64
114	/* xor gpr32, gpr32 */
115	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
116	if (iGpr >= 8)
117	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
118	pbCodeBuf[off++] = 0x33;
119	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
120
121	#elif defined(RT_ARCH_ARM64)
122	/* mov gpr, #0x0 */
123	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
124	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
125
126	#else
127	# error "port me"
128	#endif
129	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
130	return off;
131	}
132
133
134	/**
135	* Emits loading a constant into a 64-bit GPR
136	*/
137	DECL_INLINE_THROW(uint32_t)
138	iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
139	{
140	if (!uImm64)
141	return iemNativeEmitGprZero(pReNative, off, iGpr);
142
143	#ifdef RT_ARCH_AMD64
144	if (uImm64 <= UINT32_MAX)
145	{
146	/* mov gpr, imm32 */
147	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
148	if (iGpr >= 8)
149	pbCodeBuf[off++] = X86_OP_REX_B;
150	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
151	pbCodeBuf[off++] = RT_BYTE1(uImm64);
152	pbCodeBuf[off++] = RT_BYTE2(uImm64);
153	pbCodeBuf[off++] = RT_BYTE3(uImm64);
154	pbCodeBuf[off++] = RT_BYTE4(uImm64);
155	}
156	else
157	{
158	/* mov gpr, imm64 */
159	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
160	if (iGpr < 8)
161	pbCodeBuf[off++] = X86_OP_REX_W;
162	else
163	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
164	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
165	pbCodeBuf[off++] = RT_BYTE1(uImm64);
166	pbCodeBuf[off++] = RT_BYTE2(uImm64);
167	pbCodeBuf[off++] = RT_BYTE3(uImm64);
168	pbCodeBuf[off++] = RT_BYTE4(uImm64);
169	pbCodeBuf[off++] = RT_BYTE5(uImm64);
170	pbCodeBuf[off++] = RT_BYTE6(uImm64);
171	pbCodeBuf[off++] = RT_BYTE7(uImm64);
172	pbCodeBuf[off++] = RT_BYTE8(uImm64);
173	}
174
175	#elif defined(RT_ARCH_ARM64)
176	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
177
178	/*
179	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
180	* supply remaining bits using 'movk grp, imm16, lsl #x'.
181	*
182	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
183	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
184	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
185	* after the first non-zero immediate component so we switch to movk for
186	* the remainder.
187	*/
188	uint32_t fMovK = 0;
189	/* mov gpr, imm16 */
190	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
191	if (uImmPart)
192	{
193	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
194	fMovK \|= RT_BIT_32(29);
195	}
196	/* mov[k] gpr, imm16, lsl #16 */
197	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
198	if (uImmPart)
199	{
200	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
201	fMovK \|= RT_BIT_32(29);
202	}
203	/* mov[k] gpr, imm16, lsl #32 */
204	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
205	if (uImmPart)
206	{
207	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
208	fMovK \|= RT_BIT_32(29);
209	}
210	/* mov[k] gpr, imm16, lsl #48 */
211	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
212	if (uImmPart)
213	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
214
215	/** @todo there is an inverted mask variant we might want to explore if it
216	* reduces the number of instructions... */
217	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
218	* clang 12.x does that, only to use the 'x' version for the
219	* addressing in the following ldr). */
220
221	#else
222	# error "port me"
223	#endif
224	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
225	return off;
226	}
227
228
229	/**
230	* Emits loading a constant into a 8-bit GPR
231	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
232	* only the ARM64 version does that.
233	*/
234	DECL_INLINE_THROW(uint32_t)
235	iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
236	{
237	#ifdef RT_ARCH_AMD64
238	/* mov gpr, imm8 */
239	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
240	if (iGpr >= 8)
241	pbCodeBuf[off++] = X86_OP_REX_B;
242	else if (iGpr >= 4)
243	pbCodeBuf[off++] = X86_OP_REX;
244	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
245	pbCodeBuf[off++] = RT_BYTE1(uImm8);
246
247	#elif defined(RT_ARCH_ARM64)
248	/* movz gpr, imm16, lsl #0 */
249	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
250	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
251
252	#else
253	# error "port me"
254	#endif
255	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
256	return off;
257	}
258
259
260	#ifdef RT_ARCH_AMD64
261	/**
262	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
263	*/
264	DECL_FORCE_INLINE(uint32_t)
265	iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
266	{
267	if (offVCpu < 128)
268	{
269	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
270	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
271	}
272	else
273	{
274	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
275	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
276	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
277	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
278	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
279	}
280	return off;
281	}
282	#elif defined(RT_ARCH_ARM64)
283	/**
284	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
285	*/
286	DECL_FORCE_INLINE_THROW(uint32_t)
287	iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
288	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
289	{
290	/*
291	* There are a couple of ldr variants that takes an immediate offset, so
292	* try use those if we can, otherwise we have to use the temporary register
293	* help with the addressing.
294	*/
295	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
296	{
297	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
298	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
299	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
300	}
301	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
302	{
303	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
304	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
305	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
306	}
307	else
308	{
309	/* The offset is too large, so we must load it into a register and use
310	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
311	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
312	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
313
314	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
315	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
316	IEMNATIVE_REG_FIXED_TMP0);
317	}
318	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
319	return off;
320	}
321	#endif
322
323
324	/**
325	* Emits a 64-bit GPR load of a VCpu value.
326	*/
327	DECL_INLINE_THROW(uint32_t)
328	iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
329	{
330	#ifdef RT_ARCH_AMD64
331	/* mov reg64, mem64 */
332	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
333	if (iGpr < 8)
334	pbCodeBuf[off++] = X86_OP_REX_W;
335	else
336	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
337	pbCodeBuf[off++] = 0x8b;
338	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
339	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
340
341	#elif defined(RT_ARCH_ARM64)
342	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
343
344	#else
345	# error "port me"
346	#endif
347	return off;
348	}
349
350
351	/**
352	* Emits a 32-bit GPR load of a VCpu value.
353	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
354	*/
355	DECL_INLINE_THROW(uint32_t)
356	iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
357	{
358	#ifdef RT_ARCH_AMD64
359	/* mov reg32, mem32 */
360	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
361	if (iGpr >= 8)
362	pbCodeBuf[off++] = X86_OP_REX_R;
363	pbCodeBuf[off++] = 0x8b;
364	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
365	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
366
367	#elif defined(RT_ARCH_ARM64)
368	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
369
370	#else
371	# error "port me"
372	#endif
373	return off;
374	}
375
376
377	/**
378	* Emits a 16-bit GPR load of a VCpu value.
379	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
380	*/
381	DECL_INLINE_THROW(uint32_t)
382	iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
383	{
384	#ifdef RT_ARCH_AMD64
385	/* movzx reg32, mem16 */
386	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
387	if (iGpr >= 8)
388	pbCodeBuf[off++] = X86_OP_REX_R;
389	pbCodeBuf[off++] = 0x0f;
390	pbCodeBuf[off++] = 0xb7;
391	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
392	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
393
394	#elif defined(RT_ARCH_ARM64)
395	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
396
397	#else
398	# error "port me"
399	#endif
400	return off;
401	}
402
403
404	/**
405	* Emits a 8-bit GPR load of a VCpu value.
406	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
407	*/
408	DECL_INLINE_THROW(uint32_t)
409	iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
410	{
411	#ifdef RT_ARCH_AMD64
412	/* movzx reg32, mem8 */
413	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
414	if (iGpr >= 8)
415	pbCodeBuf[off++] = X86_OP_REX_R;
416	pbCodeBuf[off++] = 0x0f;
417	pbCodeBuf[off++] = 0xb6;
418	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
419	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
420
421	#elif defined(RT_ARCH_ARM64)
422	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
423
424	#else
425	# error "port me"
426	#endif
427	return off;
428	}
429
430
431	/**
432	* Emits a store of a GPR value to a 64-bit VCpu field.
433	*/
434	DECL_INLINE_THROW(uint32_t)
435	iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
436	{
437	#ifdef RT_ARCH_AMD64
438	/* mov mem64, reg64 */
439	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
440	if (iGpr < 8)
441	pbCodeBuf[off++] = X86_OP_REX_W;
442	else
443	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
444	pbCodeBuf[off++] = 0x89;
445	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
446	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
447
448	#elif defined(RT_ARCH_ARM64)
449	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
450
451	#else
452	# error "port me"
453	#endif
454	return off;
455	}
456
457
458	/**
459	* Emits a store of a GPR value to a 32-bit VCpu field.
460	*/
461	DECL_INLINE_THROW(uint32_t)
462	iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
463	{
464	#ifdef RT_ARCH_AMD64
465	/* mov mem32, reg32 */
466	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
467	if (iGpr >= 8)
468	pbCodeBuf[off++] = X86_OP_REX_R;
469	pbCodeBuf[off++] = 0x89;
470	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
471	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
472
473	#elif defined(RT_ARCH_ARM64)
474	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
475
476	#else
477	# error "port me"
478	#endif
479	return off;
480	}
481
482
483	/**
484	* Emits a store of a GPR value to a 16-bit VCpu field.
485	*/
486	DECL_INLINE_THROW(uint32_t)
487	iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
488	{
489	#ifdef RT_ARCH_AMD64
490	/* mov mem16, reg16 */
491	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
492	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
493	if (iGpr >= 8)
494	pbCodeBuf[off++] = X86_OP_REX_R;
495	pbCodeBuf[off++] = 0x89;
496	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
497	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
498
499	#elif defined(RT_ARCH_ARM64)
500	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
501
502	#else
503	# error "port me"
504	#endif
505	return off;
506	}
507
508
509	/**
510	* Emits a store of a GPR value to a 8-bit VCpu field.
511	*/
512	DECL_INLINE_THROW(uint32_t)
513	iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
514	{
515	#ifdef RT_ARCH_AMD64
516	/* mov mem8, reg8 */
517	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
518	if (iGpr >= 8)
519	pbCodeBuf[off++] = X86_OP_REX_R;
520	pbCodeBuf[off++] = 0x88;
521	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
522	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
523
524	#elif defined(RT_ARCH_ARM64)
525	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
526
527	#else
528	# error "port me"
529	#endif
530	return off;
531	}
532
533
534	/**
535	* Emits a store of an immediate value to a 8-bit VCpu field.
536	*/
537	DECL_INLINE_THROW(uint32_t)
538	iemNativeEmitStoreImmToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t bImm, uint32_t offVCpu)
539	{
540	#ifdef RT_ARCH_AMD64
541	/* mov mem8, imm8 */
542	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
543	pbCodeBuf[off++] = 0xc6;
544	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, offVCpu);
545	pbCodeBuf[off++] = bImm;
546	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
547
548	#elif defined(RT_ARCH_ARM64)
549	/* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
550	uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
551	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, idxRegImm, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
552	iemNativeRegFreeTmpImm(pReNative, idxRegImm);
553
554	#else
555	# error "port me"
556	#endif
557	return off;
558	}
559
560
561	/**
562	* Emits a load effective address to a GRP of a VCpu field.
563	*/
564	DECL_INLINE_THROW(uint32_t)
565	iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
566	{
567	#ifdef RT_ARCH_AMD64
568	/* lea gprdst, [rbx + offDisp] */
569	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
570	if (iGprDst < 8)
571	pbCodeBuf[off++] = X86_OP_REX_W;
572	else
573	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
574	pbCodeBuf[off++] = 0x8d;
575	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
576
577	#elif defined(RT_ARCH_ARM64)
578	if (offVCpu < (unsigned)_4K)
579	{
580	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
581	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
582	}
583	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
584	{
585	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
586	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
587	offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
588	}
589	else
590	{
591	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
592	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
593	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
594	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
595	}
596
597	#else
598	# error "port me"
599	#endif
600	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
601	return off;
602	}
603
604
605	/**
606	* Emits a gprdst = gprsrc load.
607	*/
608	DECL_INLINE_THROW(uint32_t)
609	iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
610	{
611	#ifdef RT_ARCH_AMD64
612	/* mov gprdst, gprsrc */
613	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
614	if ((iGprDst \| iGprSrc) >= 8)
615	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
616	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
617	: X86_OP_REX_W \| X86_OP_REX_R;
618	else
619	pbCodeBuf[off++] = X86_OP_REX_W;
620	pbCodeBuf[off++] = 0x8b;
621	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
622
623	#elif defined(RT_ARCH_ARM64)
624	/* mov dst, src; alias for: orr dst, xzr, src */
625	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
626	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
627
628	#else
629	# error "port me"
630	#endif
631	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
632	return off;
633	}
634
635
636	/**
637	* Emits a gprdst = gprsrc[31:0] load.
638	* @note Bits 63 thru 32 are cleared.
639	*/
640	DECL_INLINE_THROW(uint32_t)
641	iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
642	{
643	#ifdef RT_ARCH_AMD64
644	/* mov gprdst, gprsrc */
645	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
646	if ((iGprDst \| iGprSrc) >= 8)
647	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
648	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
649	: X86_OP_REX_R;
650	pbCodeBuf[off++] = 0x8b;
651	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
652
653	#elif defined(RT_ARCH_ARM64)
654	/* mov dst32, src32; alias for: orr dst32, wzr, src32 */
655	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
656	pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /f64bit/);
657
658	#else
659	# error "port me"
660	#endif
661	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
662	return off;
663	}
664
665
666	/**
667	* Emits a gprdst = gprsrc[15:0] load.
668	* @note Bits 63 thru 15 are cleared.
669	*/
670	DECL_INLINE_THROW(uint32_t)
671	iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
672	{
673	#ifdef RT_ARCH_AMD64
674	/* movzx Gv,Ew */
675	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
676	if ((iGprDst \| iGprSrc) >= 8)
677	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
678	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
679	: X86_OP_REX_R;
680	pbCodeBuf[off++] = 0x0f;
681	pbCodeBuf[off++] = 0xb7;
682	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
683
684	#elif defined(RT_ARCH_ARM64)
685	/* and gprdst, gprsrc, #0xffff */
686	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
687	# if 1
688	Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
689	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /f64Bit/);
690	# else
691	Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
692	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
693	# endif
694
695	#else
696	# error "port me"
697	#endif
698	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
699	return off;
700	}
701
702
703	/**
704	* Emits a gprdst = gprsrc[7:0] load.
705	* @note Bits 63 thru 8 are cleared.
706	*/
707	DECL_INLINE_THROW(uint32_t)
708	iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
709	{
710	#ifdef RT_ARCH_AMD64
711	/* movzx Gv,Eb */
712	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
713	if (iGprDst >= 8 \|\| iGprSrc >= 8)
714	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
715	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
716	: X86_OP_REX_R;
717	else if (iGprSrc >= 4)
718	pbCodeBuf[off++] = X86_OP_REX;
719	pbCodeBuf[off++] = 0x0f;
720	pbCodeBuf[off++] = 0xb6;
721	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
722
723	#elif defined(RT_ARCH_ARM64)
724	/* and gprdst, gprsrc, #0xff */
725	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
726	# if 1
727	Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
728	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /f64Bit/);
729	# else
730	Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
731	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
732	# endif
733
734	#else
735	# error "port me"
736	#endif
737	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
738	return off;
739	}
740
741
742	/**
743	* Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
744	* @note Bits 63 thru 8 are cleared.
745	*/
746	DECL_INLINE_THROW(uint32_t)
747	iemNativeEmitLoadGprFromGpr8Hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
748	{
749	#ifdef RT_ARCH_AMD64
750	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
751
752	/* movzx Gv,Ew */
753	if ((iGprDst \| iGprSrc) >= 8)
754	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
755	: iGprSrc >= 8 ? X86_OP_REX_R \| X86_OP_REX_B
756	: X86_OP_REX_R;
757	pbCodeBuf[off++] = 0x0f;
758	pbCodeBuf[off++] = 0xb7;
759	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
760
761	/* shr Ev,8 */
762	if (iGprDst >= 8)
763	pbCodeBuf[off++] = X86_OP_REX_B;
764	pbCodeBuf[off++] = 0xc1;
765	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
766	pbCodeBuf[off++] = 8;
767
768	#elif defined(RT_ARCH_ARM64)
769	/* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
770	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
771	pu32CodeBuf[off++] = Armv8A64MkInstrUbfx(iGprDst, iGprSrc, 8, 8, false /f64Bit/);
772
773	#else
774	# error "port me"
775	#endif
776	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
777	return off;
778	}
779
780
781	/**
782	* Sign-extends 32-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
783	*/
784	DECL_INLINE_THROW(uint32_t)
785	iemNativeEmitLoadGprSignExtendedFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
786	{
787	#ifdef RT_ARCH_AMD64
788	/* movsxd r64, r/m32 */
789	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
790	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
791	pbCodeBuf[off++] = 0x63;
792	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
793
794	#elif defined(RT_ARCH_ARM64)
795	/* sxtw dst, src */
796	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
797	pu32CodeBuf[off++] = Armv8A64MkInstrSxtw(iGprDst, iGprSrc);
798
799	#else
800	# error "port me"
801	#endif
802	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
803	return off;
804	}
805
806
807	/**
808	* Sign-extends 16-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
809	*/
810	DECL_INLINE_THROW(uint32_t)
811	iemNativeEmitLoadGprSignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
812	{
813	#ifdef RT_ARCH_AMD64
814	/* movsx r64, r/m16 */
815	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
816	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
817	pbCodeBuf[off++] = 0x0f;
818	pbCodeBuf[off++] = 0xbf;
819	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
820
821	#elif defined(RT_ARCH_ARM64)
822	/* sxth dst, src */
823	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
824	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc);
825
826	#else
827	# error "port me"
828	#endif
829	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
830	return off;
831	}
832
833
834	/**
835	* Sign-extends 16-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
836	*/
837	DECL_INLINE_THROW(uint32_t)
838	iemNativeEmitLoadGpr32SignExtendedFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
839	{
840	#ifdef RT_ARCH_AMD64
841	/* movsx r64, r/m16 */
842	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
843	if (iGprDst >= 8 \|\| iGprSrc >= 8)
844	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
845	pbCodeBuf[off++] = 0x0f;
846	pbCodeBuf[off++] = 0xbf;
847	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
848
849	#elif defined(RT_ARCH_ARM64)
850	/* sxth dst32, src */
851	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
852	pu32CodeBuf[off++] = Armv8A64MkInstrSxth(iGprDst, iGprSrc, false /f64Bit/);
853
854	#else
855	# error "port me"
856	#endif
857	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
858	return off;
859	}
860
861
862	/**
863	* Sign-extends 8-bit value in @a iGprSrc into a 64-bit value in @a iGprDst.
864	*/
865	DECL_INLINE_THROW(uint32_t)
866	iemNativeEmitLoadGprSignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
867	{
868	#ifdef RT_ARCH_AMD64
869	/* movsx r64, r/m8 */
870	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
871	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
872	pbCodeBuf[off++] = 0x0f;
873	pbCodeBuf[off++] = 0xbe;
874	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
875
876	#elif defined(RT_ARCH_ARM64)
877	/* sxtb dst, src */
878	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
879	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc);
880
881	#else
882	# error "port me"
883	#endif
884	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
885	return off;
886	}
887
888
889	/**
890	* Sign-extends 8-bit value in @a iGprSrc into a 32-bit value in @a iGprDst.
891	* @note Bits 64 thru 32 are cleared.
892	*/
893	DECL_INLINE_THROW(uint32_t)
894	iemNativeEmitLoadGpr32SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
895	{
896	#ifdef RT_ARCH_AMD64
897	/* movsx r32, r/m8 */
898	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
899	if (iGprDst >= 8 \|\| iGprSrc >= 8)
900	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
901	pbCodeBuf[off++] = 0x0f;
902	pbCodeBuf[off++] = 0xbe;
903	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
904
905	#elif defined(RT_ARCH_ARM64)
906	/* sxtb dst32, src32 */
907	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
908	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
909
910	#else
911	# error "port me"
912	#endif
913	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
914	return off;
915	}
916
917
918	/**
919	* Sign-extends 8-bit value in @a iGprSrc into a 16-bit value in @a iGprDst.
920	* @note Bits 64 thru 16 are cleared.
921	*/
922	DECL_INLINE_THROW(uint32_t)
923	iemNativeEmitLoadGpr16SignExtendedFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
924	{
925	#ifdef RT_ARCH_AMD64
926	/* movsx r16, r/m8 */
927	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 9);
928	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
929	if (iGprDst >= 8 \|\| iGprSrc >= 8)
930	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
931	pbCodeBuf[off++] = 0x0f;
932	pbCodeBuf[off++] = 0xbe;
933	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
934
935	/* movzx r32, r/m16 */
936	if (iGprDst >= 8)
937	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
938	pbCodeBuf[off++] = 0x0f;
939	pbCodeBuf[off++] = 0xb7;
940	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
941
942	#elif defined(RT_ARCH_ARM64)
943	/* sxtb dst32, src32; and dst32, dst32, #0xffff */
944	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
945	pu32CodeBuf[off++] = Armv8A64MkInstrSxtb(iGprDst, iGprSrc, false /f64Bit/);
946	Assert(Armv8A64ConvertImmRImmS2Mask32(15, 0) == 0xffff);
947	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 15, 0, false /f64Bit/);
948
949	#else
950	# error "port me"
951	#endif
952	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
953	return off;
954	}
955
956
957	/**
958	* Emits a gprdst = gprsrc + addend load.
959	*/
960	DECL_INLINE_THROW(uint32_t)
961	iemNativeEmitLoadGprFromGprWithAddend(PIEMRECOMPILERSTATE pReNative, uint32_t off,
962	uint8_t iGprDst, uint8_t iGprSrc, int32_t iAddend)
963	{
964	Assert(iAddend != 0);
965
966	#ifdef RT_ARCH_AMD64
967	/* lea gprdst, [gprsrc + iAddend] */
968	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
969	if ((iGprDst \| iGprSrc) >= 8)
970	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
971	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
972	: X86_OP_REX_W \| X86_OP_REX_R;
973	else
974	pbCodeBuf[off++] = X86_OP_REX_W;
975	pbCodeBuf[off++] = 0x8d;
976	if (iAddend >= -128 && iAddend < 128)
977	{
978	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprDst & 7, iGprSrc & 7);
979	pbCodeBuf[off++] = (int8_t)iAddend;
980	}
981	else
982	{
983	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprDst & 7, iGprSrc & 7);
984	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
985	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
986	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
987	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
988	}
989
990	#elif defined(RT_ARCH_ARM64)
991	if ((uint32_t)iAddend < 4096)
992	{
993	/* add dst, src, uimm12 */
994	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
995	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprSrc, (uint32_t)iAddend);
996	}
997	else if ((uint32_t)-iAddend < 4096)
998	{
999	/* sub dst, src, uimm12 */
1000	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1001	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprSrc, (uint32_t)-iAddend);
1002	}
1003	else
1004	{
1005	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, (int64_t)iAddend);
1006	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1007	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprSrc, iGprDst);
1008	}
1009
1010	#else
1011	# error "port me"
1012	#endif
1013	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1014	return off;
1015	}
1016
1017
1018	#ifdef RT_ARCH_AMD64
1019	/**
1020	* Common bit of iemNativeEmitLoadGprByBp and friends.
1021	*/
1022	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1023	PIEMRECOMPILERSTATE pReNativeAssert)
1024	{
1025	if (offDisp < 128 && offDisp >= -128)
1026	{
1027	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1028	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1029	}
1030	else
1031	{
1032	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1033	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1034	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1035	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1036	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1037	}
1038	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1039	return off;
1040	}
1041	#elif defined(RT_ARCH_ARM64)
1042	/**
1043	* Common bit of iemNativeEmitLoadGprByBp and friends.
1044	*/
1045	DECL_FORCE_INLINE_THROW(uint32_t)
1046	iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1047	int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1048	{
1049	if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1050	{
1051	/* str w/ unsigned imm12 (scaled) */
1052	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1053	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1054	}
1055	else if (offDisp >= -256 && offDisp <= 256)
1056	{
1057	/* stur w/ signed imm9 (unscaled) */
1058	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1059	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1060	}
1061	else
1062	{
1063	/* Use temporary indexing register. */
1064	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1065	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1066	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1067	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1068	}
1069	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1070	return off;
1071	}
1072	#endif
1073
1074
1075	/**
1076	* Emits a 64-bit GRP load instruction with an BP relative source address.
1077	*/
1078	DECL_INLINE_THROW(uint32_t)
1079	iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1080	{
1081	#ifdef RT_ARCH_AMD64
1082	/* mov gprdst, qword [rbp + offDisp] */
1083	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1084	if (iGprDst < 8)
1085	pbCodeBuf[off++] = X86_OP_REX_W;
1086	else
1087	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1088	pbCodeBuf[off++] = 0x8b;
1089	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1090
1091	#elif defined(RT_ARCH_ARM64)
1092	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1093
1094	#else
1095	# error "port me"
1096	#endif
1097	}
1098
1099
1100	/**
1101	* Emits a 32-bit GRP load instruction with an BP relative source address.
1102	* @note Bits 63 thru 32 of the GPR will be cleared.
1103	*/
1104	DECL_INLINE_THROW(uint32_t)
1105	iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1106	{
1107	#ifdef RT_ARCH_AMD64
1108	/* mov gprdst, dword [rbp + offDisp] */
1109	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1110	if (iGprDst >= 8)
1111	pbCodeBuf[off++] = X86_OP_REX_R;
1112	pbCodeBuf[off++] = 0x8b;
1113	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1114
1115	#elif defined(RT_ARCH_ARM64)
1116	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1117
1118	#else
1119	# error "port me"
1120	#endif
1121	}
1122
1123
1124	/**
1125	* Emits a 16-bit GRP load instruction with an BP relative source address.
1126	* @note Bits 63 thru 16 of the GPR will be cleared.
1127	*/
1128	DECL_INLINE_THROW(uint32_t)
1129	iemNativeEmitLoadGprByBpU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1130	{
1131	#ifdef RT_ARCH_AMD64
1132	/* movzx gprdst, word [rbp + offDisp] */
1133	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1134	if (iGprDst >= 8)
1135	pbCodeBuf[off++] = X86_OP_REX_R;
1136	pbCodeBuf[off++] = 0x0f;
1137	pbCodeBuf[off++] = 0xb7;
1138	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1139
1140	#elif defined(RT_ARCH_ARM64)
1141	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint32_t));
1142
1143	#else
1144	# error "port me"
1145	#endif
1146	}
1147
1148
1149	/**
1150	* Emits a 8-bit GRP load instruction with an BP relative source address.
1151	* @note Bits 63 thru 8 of the GPR will be cleared.
1152	*/
1153	DECL_INLINE_THROW(uint32_t)
1154	iemNativeEmitLoadGprByBpU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1155	{
1156	#ifdef RT_ARCH_AMD64
1157	/* movzx gprdst, byte [rbp + offDisp] */
1158	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1159	if (iGprDst >= 8)
1160	pbCodeBuf[off++] = X86_OP_REX_R;
1161	pbCodeBuf[off++] = 0x0f;
1162	pbCodeBuf[off++] = 0xb6;
1163	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1164
1165	#elif defined(RT_ARCH_ARM64)
1166	return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint32_t));
1167
1168	#else
1169	# error "port me"
1170	#endif
1171	}
1172
1173
1174	/**
1175	* Emits a load effective address to a GRP with an BP relative source address.
1176	*/
1177	DECL_INLINE_THROW(uint32_t)
1178	iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1179	{
1180	#ifdef RT_ARCH_AMD64
1181	/* lea gprdst, [rbp + offDisp] */
1182	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1183	if (iGprDst < 8)
1184	pbCodeBuf[off++] = X86_OP_REX_W;
1185	else
1186	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1187	pbCodeBuf[off++] = 0x8d;
1188	off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1189
1190	#elif defined(RT_ARCH_ARM64)
1191	if ((uint32_t)offDisp < (unsigned)_4K)
1192	{
1193	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1194	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)offDisp);
1195	}
1196	else if ((uint32_t)-offDisp < (unsigned)_4K)
1197	{
1198	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1199	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1200	}
1201	else
1202	{
1203	Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1204	off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1205	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1206	if (offDisp >= 0)
1207	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1208	else
1209	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, iGprDst, ARMV8_A64_REG_BP, iGprDst);
1210	}
1211
1212	#else
1213	# error "port me"
1214	#endif
1215
1216	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1217	return off;
1218	}
1219
1220
1221	/**
1222	* Emits a 64-bit GPR store with an BP relative destination address.
1223	*
1224	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1225	*/
1226	DECL_INLINE_THROW(uint32_t)
1227	iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1228	{
1229	#ifdef RT_ARCH_AMD64
1230	/* mov qword [rbp + offDisp], gprdst */
1231	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1232	if (iGprSrc < 8)
1233	pbCodeBuf[off++] = X86_OP_REX_W;
1234	else
1235	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1236	pbCodeBuf[off++] = 0x89;
1237	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1238
1239	#elif defined(RT_ARCH_ARM64)
1240	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1241	{
1242	/* str w/ unsigned imm12 (scaled) */
1243	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1244	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1245	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1246	}
1247	else if (offDisp >= -256 && offDisp <= 256)
1248	{
1249	/* stur w/ signed imm9 (unscaled) */
1250	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1251	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1252	}
1253	else if ((uint32_t)-offDisp < (unsigned)_4K)
1254	{
1255	/* Use temporary indexing register w/ sub uimm12. */
1256	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1257	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, IEMNATIVE_REG_FIXED_TMP0,
1258	ARMV8_A64_REG_BP, (uint32_t)-offDisp);
1259	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc, IEMNATIVE_REG_FIXED_TMP0, 0);
1260	}
1261	else
1262	{
1263	/* Use temporary indexing register. */
1264	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1265	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1266	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1267	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1268	}
1269	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1270	return off;
1271
1272	#else
1273	# error "Port me!"
1274	#endif
1275	}
1276
1277
1278	/**
1279	* Emits a 64-bit immediate store with an BP relative destination address.
1280	*
1281	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1282	*/
1283	DECL_INLINE_THROW(uint32_t)
1284	iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1285	{
1286	#ifdef RT_ARCH_AMD64
1287	if ((int64_t)uImm64 == (int32_t)uImm64)
1288	{
1289	/* mov qword [rbp + offDisp], imm32 - sign extended */
1290	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1291	pbCodeBuf[off++] = X86_OP_REX_W;
1292	pbCodeBuf[off++] = 0xc7;
1293	if (offDisp < 128 && offDisp >= -128)
1294	{
1295	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1296	pbCodeBuf[off++] = (uint8_t)offDisp;
1297	}
1298	else
1299	{
1300	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1301	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1302	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1303	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1304	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1305	}
1306	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1307	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1308	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1309	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1310	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1311	return off;
1312	}
1313	#endif
1314
1315	/* Load tmp0, imm64; Store tmp to bp+disp. */
1316	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1317	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1318	}
1319
1320
1321	#ifdef RT_ARCH_AMD64
1322	/**
1323	* Common bit of iemNativeEmitLoadGprByGpr and friends.
1324	*/
1325	DECL_FORCE_INLINE(uint32_t)
1326	iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1327	{
1328	if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1329	{
1330	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1331	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1332	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1333	}
1334	else if (offDisp == (int8_t)offDisp)
1335	{
1336	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1337	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1338	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1339	pbCodeBuf[off++] = (uint8_t)offDisp;
1340	}
1341	else
1342	{
1343	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1344	if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1345	pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1346	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1347	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1348	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1349	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1350	}
1351	return off;
1352	}
1353	#elif defined(RT_ARCH_ARM64)
1354	/**
1355	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1356	*/
1357	DECL_FORCE_INLINE_THROW(uint32_t)
1358	iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1359	uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1360	{
1361	/*
1362	* There are a couple of ldr variants that takes an immediate offset, so
1363	* try use those if we can, otherwise we have to use the temporary register
1364	* help with the addressing.
1365	*/
1366	if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1367	{
1368	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
1369	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1370	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1371	}
1372	else
1373	{
1374	/* The offset is too large, so we must load it into a register and use
1375	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
1376	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1377	uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1378
1379	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1380	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1381
1382	iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1383	}
1384	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1385	return off;
1386	}
1387	#endif
1388
1389
1390	/**
1391	* Emits a 64-bit GPR load via a GPR base address with a displacement.
1392	*/
1393	DECL_INLINE_THROW(uint32_t)
1394	iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1395	{
1396	#ifdef RT_ARCH_AMD64
1397	/* mov reg64, mem64 */
1398	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1399	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1400	pbCodeBuf[off++] = 0x8b;
1401	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1402	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1403
1404	#elif defined(RT_ARCH_ARM64)
1405	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1406
1407	#else
1408	# error "port me"
1409	#endif
1410	return off;
1411	}
1412
1413
1414	/**
1415	* Emits a 32-bit GPR load via a GPR base address with a displacement.
1416	* @note Bits 63 thru 32 in @a iGprDst will be cleared.
1417	*/
1418	DECL_INLINE_THROW(uint32_t)
1419	iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1420	{
1421	#ifdef RT_ARCH_AMD64
1422	/* mov reg32, mem32 */
1423	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1424	if (iGprDst >= 8 \|\| iGprBase >= 8)
1425	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprBase < 8 ? 0 : X86_OP_REX_B);
1426	pbCodeBuf[off++] = 0x8b;
1427	off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1428	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1429
1430	#elif defined(RT_ARCH_ARM64)
1431	off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1432
1433	#else
1434	# error "port me"
1435	#endif
1436	return off;
1437	}
1438
1439
1440	/*********************************************************************************************************************************
1441	* Subtraction and Additions *
1442	*********************************************************************************************************************************/
1443
1444
1445	#ifdef RT_ARCH_AMD64
1446	/**
1447	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1448	*/
1449	DECL_INLINE_THROW(uint32_t)
1450	iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1451	{
1452	/* sub gprdst, imm8/imm32 */
1453	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1454	if (iGprDst < 8)
1455	pbCodeBuf[off++] = X86_OP_REX_W;
1456	else
1457	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1458	if (iSubtrahend < 128 && iSubtrahend >= -128)
1459	{
1460	pbCodeBuf[off++] = 0x83;
1461	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1462	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1463	}
1464	else
1465	{
1466	pbCodeBuf[off++] = 0x81;
1467	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1468	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1469	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1470	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1471	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1472	}
1473	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1474	return off;
1475	}
1476	#endif
1477
1478
1479	/**
1480	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1481	* @note The AMD64 version sets flags.
1482	*/
1483	DECL_INLINE_THROW(uint32_t)
1484	iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1485	{
1486	#if defined(RT_ARCH_AMD64)
1487	/* add Gv,Ev */
1488	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1489	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1490	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1491	pbCodeBuf[off++] = 0x04;
1492	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1493
1494	#elif defined(RT_ARCH_ARM64)
1495	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1496	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1497
1498	#else
1499	# error "Port me"
1500	#endif
1501	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1502	return off;
1503	}
1504
1505
1506	/**
1507	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1508	*/
1509	DECL_INLINE_THROW(uint32_t)
1510	iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1511	{
1512	#if defined(RT_ARCH_AMD64)
1513	/* add or inc */
1514	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1515	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1516	if (iImm8 != 1)
1517	{
1518	pbCodeBuf[off++] = 0x83;
1519	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1520	pbCodeBuf[off++] = (uint8_t)iImm8;
1521	}
1522	else
1523	{
1524	pbCodeBuf[off++] = 0xff;
1525	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1526	}
1527
1528	#elif defined(RT_ARCH_ARM64)
1529	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1530	if (iImm8 >= 0)
1531	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1532	else
1533	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1534
1535	#else
1536	# error "Port me"
1537	#endif
1538	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1539	return off;
1540	}
1541
1542
1543	/**
1544	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1545	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1546	*/
1547	DECL_INLINE_THROW(uint32_t)
1548	iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1549	{
1550	#if defined(RT_ARCH_AMD64)
1551	/* add or inc */
1552	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1553	if (iGprDst >= 8)
1554	pbCodeBuf[off++] = X86_OP_REX_B;
1555	if (iImm8 != 1)
1556	{
1557	pbCodeBuf[off++] = 0x83;
1558	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1559	pbCodeBuf[off++] = (uint8_t)iImm8;
1560	}
1561	else
1562	{
1563	pbCodeBuf[off++] = 0xff;
1564	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1565	}
1566
1567	#elif defined(RT_ARCH_ARM64)
1568	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1569	if (iImm8 >= 0)
1570	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1571	else
1572	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1573
1574	#else
1575	# error "Port me"
1576	#endif
1577	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1578	return off;
1579	}
1580
1581
1582	/**
1583	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1584	*/
1585	DECL_INLINE_THROW(uint32_t)
1586	iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1587	{
1588	#if defined(RT_ARCH_AMD64)
1589	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1590	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1591
1592	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1593	{
1594	/* add grp, imm32 */
1595	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1596	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1597	pbCodeBuf[off++] = 0x81;
1598	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1599	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1600	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1601	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1602	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1603	}
1604	else
1605	{
1606	/* Best to use a temporary register to deal with this in the simplest way: */
1607	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1608
1609	/* add dst, tmpreg */
1610	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1611	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1612	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1613	pbCodeBuf[off++] = 0x03;
1614	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1615
1616	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1617	}
1618
1619	#elif defined(RT_ARCH_ARM64)
1620	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1621	{
1622	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1623	if (iAddend >= 0)
1624	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1625	else
1626	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1627	}
1628	else
1629	{
1630	/* Use temporary register for the immediate. */
1631	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1632
1633	/* add gprdst, gprdst, tmpreg */
1634	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1635	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1636
1637	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1638	}
1639
1640	#else
1641	# error "Port me"
1642	#endif
1643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1644	return off;
1645	}
1646
1647
1648	/**
1649	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1650	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1651	*/
1652	DECL_INLINE_THROW(uint32_t)
1653	iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1654	{
1655	#if defined(RT_ARCH_AMD64)
1656	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1657	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1658
1659	/* add grp, imm32 */
1660	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1661	if (iGprDst >= 8)
1662	pbCodeBuf[off++] = X86_OP_REX_B;
1663	pbCodeBuf[off++] = 0x81;
1664	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1665	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1666	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1667	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1668	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1669
1670	#elif defined(RT_ARCH_ARM64)
1671	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1672	{
1673	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1674	if (iAddend >= 0)
1675	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1676	else
1677	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1678	}
1679	else
1680	{
1681	/* Use temporary register for the immediate. */
1682	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1683
1684	/* add gprdst, gprdst, tmpreg */
1685	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1686	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1687
1688	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1689	}
1690
1691	#else
1692	# error "Port me"
1693	#endif
1694	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1695	return off;
1696	}
1697
1698
1699
1700	/*********************************************************************************************************************************
1701	* Bit Operations *
1702	*********************************************************************************************************************************/
1703
1704	/**
1705	* Emits code for clearing bits 16 thru 63 in the GPR.
1706	*/
1707	DECL_INLINE_THROW(uint32_t)
1708	iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1709	{
1710	#if defined(RT_ARCH_AMD64)
1711	/* movzx Gv,Ew */
1712	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1713	if (iGprDst >= 8)
1714	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1715	pbCodeBuf[off++] = 0x0f;
1716	pbCodeBuf[off++] = 0xb7;
1717	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1718
1719	#elif defined(RT_ARCH_ARM64)
1720	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1721	# if 1
1722	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1723	# else
1724	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1725	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1726	# endif
1727	#else
1728	# error "Port me"
1729	#endif
1730	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1731	return off;
1732	}
1733
1734
1735	/**
1736	* Emits code for AND'ing two 64-bit GPRs.
1737	*
1738	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1739	* and ARM64 hosts.
1740	*/
1741	DECL_INLINE_THROW(uint32_t)
1742	iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1743	{
1744	#if defined(RT_ARCH_AMD64)
1745	/* and Gv, Ev */
1746	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1747	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1748	pbCodeBuf[off++] = 0x23;
1749	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1750	RT_NOREF(fSetFlags);
1751
1752	#elif defined(RT_ARCH_ARM64)
1753	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1754	if (!fSetFlags)
1755	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1756	else
1757	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
1758
1759	#else
1760	# error "Port me"
1761	#endif
1762	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1763	return off;
1764	}
1765
1766
1767	/**
1768	* Emits code for AND'ing two 32-bit GPRs.
1769	*/
1770	DECL_INLINE_THROW(uint32_t)
1771	iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
1772	{
1773	#if defined(RT_ARCH_AMD64)
1774	/* and Gv, Ev */
1775	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1776	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1777	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1778	pbCodeBuf[off++] = 0x23;
1779	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1780	RT_NOREF(fSetFlags);
1781
1782	#elif defined(RT_ARCH_ARM64)
1783	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1784	if (!fSetFlags)
1785	pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1786	else
1787	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1788
1789	#else
1790	# error "Port me"
1791	#endif
1792	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1793	return off;
1794	}
1795
1796
1797	/**
1798	* Emits code for AND'ing a 64-bit GPRs with a constant.
1799	*
1800	* @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
1801	* and ARM64 hosts.
1802	*/
1803	DECL_INLINE_THROW(uint32_t)
1804	iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
1805	{
1806	#if defined(RT_ARCH_AMD64)
1807	if ((int64_t)uImm == (int8_t)uImm)
1808	{
1809	/* and Ev, imm8 */
1810	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1811	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1812	pbCodeBuf[off++] = 0x83;
1813	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1814	pbCodeBuf[off++] = (uint8_t)uImm;
1815	}
1816	else if ((int64_t)uImm == (int32_t)uImm)
1817	{
1818	/* and Ev, imm32 */
1819	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1820	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_B);
1821	pbCodeBuf[off++] = 0x81;
1822	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1823	pbCodeBuf[off++] = RT_BYTE1(uImm);
1824	pbCodeBuf[off++] = RT_BYTE2(uImm);
1825	pbCodeBuf[off++] = RT_BYTE3(uImm);
1826	pbCodeBuf[off++] = RT_BYTE4(uImm);
1827	}
1828	else
1829	{
1830	/* Use temporary register for the 64-bit immediate. */
1831	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1832	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
1833	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1834	}
1835	RT_NOREF(fSetFlags);
1836
1837	#elif defined(RT_ARCH_ARM64)
1838	uint32_t uImmR = 0;
1839	uint32_t uImmNandS = 0;
1840	if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
1841	{
1842	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1843	if (!fSetFlags)
1844	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
1845	else
1846	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
1847	}
1848	else
1849	{
1850	/* Use temporary register for the 64-bit immediate. */
1851	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1852	off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1853	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1854	}
1855
1856	#else
1857	# error "Port me"
1858	#endif
1859	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1860	return off;
1861	}
1862
1863
1864	/**
1865	* Emits code for AND'ing an 32-bit GPRs with a constant.
1866	*/
1867	DECL_INLINE_THROW(uint32_t)
1868	iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
1869	{
1870	#if defined(RT_ARCH_AMD64)
1871	/* and Ev, imm */
1872	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1873	if (iGprDst >= 8)
1874	pbCodeBuf[off++] = X86_OP_REX_B;
1875	if ((int32_t)uImm == (int8_t)uImm)
1876	{
1877	pbCodeBuf[off++] = 0x83;
1878	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1879	pbCodeBuf[off++] = (uint8_t)uImm;
1880	}
1881	else
1882	{
1883	pbCodeBuf[off++] = 0x81;
1884	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1885	pbCodeBuf[off++] = RT_BYTE1(uImm);
1886	pbCodeBuf[off++] = RT_BYTE2(uImm);
1887	pbCodeBuf[off++] = RT_BYTE3(uImm);
1888	pbCodeBuf[off++] = RT_BYTE4(uImm);
1889	}
1890	RT_NOREF(fSetFlags);
1891
1892	#elif defined(RT_ARCH_ARM64)
1893	uint32_t uImmR = 0;
1894	uint32_t uImmNandS = 0;
1895	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
1896	{
1897	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1898	if (!fSetFlags)
1899	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1900	else
1901	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /f64Bit/);
1902	}
1903	else
1904	{
1905	/* Use temporary register for the 64-bit immediate. */
1906	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
1907	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
1908	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1909	}
1910
1911	#else
1912	# error "Port me"
1913	#endif
1914	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1915	return off;
1916	}
1917
1918
1919	/**
1920	* Emits code for XOR'ing two 64-bit GPRs.
1921	*/
1922	DECL_INLINE_THROW(uint32_t)
1923	iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1924	{
1925	#if defined(RT_ARCH_AMD64)
1926	/* and Gv, Ev */
1927	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1928	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1929	pbCodeBuf[off++] = 0x33;
1930	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1931
1932	#elif defined(RT_ARCH_ARM64)
1933	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1934	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1935
1936	#else
1937	# error "Port me"
1938	#endif
1939	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1940	return off;
1941	}
1942
1943
1944	/**
1945	* Emits code for XOR'ing two 32-bit GPRs.
1946	*/
1947	DECL_INLINE_THROW(uint32_t)
1948	iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1949	{
1950	#if defined(RT_ARCH_AMD64)
1951	/* and Gv, Ev */
1952	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1953	if (iGprDst >= 8 \|\| iGprSrc >= 8)
1954	pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1955	pbCodeBuf[off++] = 0x33;
1956	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1957
1958	#elif defined(RT_ARCH_ARM64)
1959	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1960	pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /f64Bit/);
1961
1962	#else
1963	# error "Port me"
1964	#endif
1965	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1966	return off;
1967	}
1968
1969
1970	/*********************************************************************************************************************************
1971	* Shifting *
1972	*********************************************************************************************************************************/
1973
1974	/**
1975	* Emits code for shifting a GPR a fixed number of bits to the left.
1976	*/
1977	DECL_INLINE_THROW(uint32_t)
1978	iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1979	{
1980	Assert(cShift > 0 && cShift < 64);
1981
1982	#if defined(RT_ARCH_AMD64)
1983	/* shl dst, cShift */
1984	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1985	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1986	if (cShift != 1)
1987	{
1988	pbCodeBuf[off++] = 0xc1;
1989	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1990	pbCodeBuf[off++] = cShift;
1991	}
1992	else
1993	{
1994	pbCodeBuf[off++] = 0xd1;
1995	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1996	}
1997
1998	#elif defined(RT_ARCH_ARM64)
1999	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2000	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2001
2002	#else
2003	# error "Port me"
2004	#endif
2005	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2006	return off;
2007	}
2008
2009
2010	/**
2011	* Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2012	*/
2013	DECL_INLINE_THROW(uint32_t)
2014	iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2015	{
2016	Assert(cShift > 0 && cShift < 32);
2017
2018	#if defined(RT_ARCH_AMD64)
2019	/* shl dst, cShift */
2020	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2021	if (iGprDst >= 8)
2022	pbCodeBuf[off++] = X86_OP_REX_B;
2023	if (cShift != 1)
2024	{
2025	pbCodeBuf[off++] = 0xc1;
2026	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2027	pbCodeBuf[off++] = cShift;
2028	}
2029	else
2030	{
2031	pbCodeBuf[off++] = 0xd1;
2032	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2033	}
2034
2035	#elif defined(RT_ARCH_ARM64)
2036	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2037	pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /64Bit/);
2038
2039	#else
2040	# error "Port me"
2041	#endif
2042	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2043	return off;
2044	}
2045
2046
2047	/**
2048	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2049	*/
2050	DECL_INLINE_THROW(uint32_t)
2051	iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2052	{
2053	Assert(cShift > 0 && cShift < 64);
2054
2055	#if defined(RT_ARCH_AMD64)
2056	/* shr dst, cShift */
2057	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2058	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
2059	if (cShift != 1)
2060	{
2061	pbCodeBuf[off++] = 0xc1;
2062	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2063	pbCodeBuf[off++] = cShift;
2064	}
2065	else
2066	{
2067	pbCodeBuf[off++] = 0xd1;
2068	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2069	}
2070
2071	#elif defined(RT_ARCH_ARM64)
2072	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2073	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2074
2075	#else
2076	# error "Port me"
2077	#endif
2078	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2079	return off;
2080	}
2081
2082
2083	/**
2084	* Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2085	* right.
2086	*/
2087	DECL_INLINE_THROW(uint32_t)
2088	iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2089	{
2090	Assert(cShift > 0 && cShift < 32);
2091
2092	#if defined(RT_ARCH_AMD64)
2093	/* shr dst, cShift */
2094	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2095	if (iGprDst >= 8)
2096	pbCodeBuf[off++] = X86_OP_REX_B;
2097	if (cShift != 1)
2098	{
2099	pbCodeBuf[off++] = 0xc1;
2100	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2101	pbCodeBuf[off++] = cShift;
2102	}
2103	else
2104	{
2105	pbCodeBuf[off++] = 0xd1;
2106	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2107	}
2108
2109	#elif defined(RT_ARCH_ARM64)
2110	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2111	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /64Bit/);
2112
2113	#else
2114	# error "Port me"
2115	#endif
2116	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2117	return off;
2118	}
2119
2120
2121
2122	/*********************************************************************************************************************************
2123	* Compare and Testing *
2124	*********************************************************************************************************************************/
2125
2126
2127	#ifdef RT_ARCH_ARM64
2128	/**
2129	* Emits an ARM64 compare instruction.
2130	*/
2131	DECL_INLINE_THROW(uint32_t)
2132	iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2133	bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2134	{
2135	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2136	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
2137	f64Bit, true /fSetFlags/, cShift, enmShift);
2138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2139	return off;
2140	}
2141	#endif
2142
2143
2144	/**
2145	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2146	* with conditional instruction.
2147	*/
2148	DECL_INLINE_THROW(uint32_t)
2149	iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2150	{
2151	#ifdef RT_ARCH_AMD64
2152	/* cmp Gv, Ev */
2153	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2154	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2155	pbCodeBuf[off++] = 0x3b;
2156	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2157	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2158
2159	#elif defined(RT_ARCH_ARM64)
2160	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2161
2162	#else
2163	# error "Port me!"
2164	#endif
2165	return off;
2166	}
2167
2168
2169	/**
2170	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2171	* with conditional instruction.
2172	*/
2173	DECL_INLINE_THROW(uint32_t)
2174	iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2175	{
2176	#ifdef RT_ARCH_AMD64
2177	/* cmp Gv, Ev */
2178	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2179	if (iGprLeft >= 8 \|\| iGprRight >= 8)
2180	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
2181	pbCodeBuf[off++] = 0x3b;
2182	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2183	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2184
2185	#elif defined(RT_ARCH_ARM64)
2186	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
2187
2188	#else
2189	# error "Port me!"
2190	#endif
2191	return off;
2192	}
2193
2194
2195	/**
2196	* Emits a compare of a 64-bit GPR with a constant value, settings status
2197	* flags/whatever for use with conditional instruction.
2198	*/
2199	DECL_INLINE_THROW(uint32_t)
2200	iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2201	{
2202	#ifdef RT_ARCH_AMD64
2203	if (uImm <= UINT32_C(0xff))
2204	{
2205	/* cmp Ev, Ib */
2206	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2207	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2208	pbCodeBuf[off++] = 0x83;
2209	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2210	pbCodeBuf[off++] = (uint8_t)uImm;
2211	}
2212	else if ((int64_t)uImm == (int32_t)uImm)
2213	{
2214	/* cmp Ev, imm */
2215	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2216	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2217	pbCodeBuf[off++] = 0x81;
2218	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2219	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2220	pbCodeBuf[off++] = RT_BYTE1(uImm);
2221	pbCodeBuf[off++] = RT_BYTE2(uImm);
2222	pbCodeBuf[off++] = RT_BYTE3(uImm);
2223	pbCodeBuf[off++] = RT_BYTE4(uImm);
2224	}
2225	else
2226	{
2227	/* Use temporary register for the immediate. */
2228	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2229	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2230	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2231	}
2232
2233	#elif defined(RT_ARCH_ARM64)
2234	/** @todo guess there are clevere things we can do here... */
2235	if (uImm < _4K)
2236	{
2237	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2238	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2239	true /64Bit/, true /fSetFlags/);
2240	}
2241	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2242	{
2243	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2244	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2245	true /64Bit/, true /fSetFlags/, true /fShift12/);
2246	}
2247	else
2248	{
2249	/* Use temporary register for the immediate. */
2250	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2251	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2252	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2253	}
2254
2255	#else
2256	# error "Port me!"
2257	#endif
2258
2259	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2260	return off;
2261	}
2262
2263
2264	/**
2265	* Emits a compare of a 32-bit GPR with a constant value, settings status
2266	* flags/whatever for use with conditional instruction.
2267	*/
2268	DECL_INLINE_THROW(uint32_t)
2269	iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2270	{
2271	#ifdef RT_ARCH_AMD64
2272	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2273	if (iGprLeft >= 8)
2274	pbCodeBuf[off++] = X86_OP_REX_B;
2275	if (uImm <= UINT32_C(0xff))
2276	{
2277	/* cmp Ev, Ib */
2278	pbCodeBuf[off++] = 0x83;
2279	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2280	pbCodeBuf[off++] = (uint8_t)uImm;
2281	}
2282	else
2283	{
2284	/* cmp Ev, imm */
2285	pbCodeBuf[off++] = 0x81;
2286	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2287	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2288	pbCodeBuf[off++] = RT_BYTE1(uImm);
2289	pbCodeBuf[off++] = RT_BYTE2(uImm);
2290	pbCodeBuf[off++] = RT_BYTE3(uImm);
2291	pbCodeBuf[off++] = RT_BYTE4(uImm);
2292	}
2293
2294	#elif defined(RT_ARCH_ARM64)
2295	/** @todo guess there are clevere things we can do here... */
2296	if (uImm < _4K)
2297	{
2298	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2299	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2300	false /64Bit/, true /fSetFlags/);
2301	}
2302	else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2303	{
2304	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2305	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2306	false /64Bit/, true /fSetFlags/, true /fShift12/);
2307	}
2308	else
2309	{
2310	/* Use temporary register for the immediate. */
2311	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2312	off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2313	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2314	}
2315
2316	#else
2317	# error "Port me!"
2318	#endif
2319
2320	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2321	return off;
2322	}
2323
2324
2325
2326	/*********************************************************************************************************************************
2327	* Branching *
2328	*********************************************************************************************************************************/
2329
2330	/**
2331	* Emits a JMP rel32 / B imm19 to the given label.
2332	*/
2333	DECL_INLINE_THROW(uint32_t)
2334	iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2335	{
2336	Assert(idxLabel < pReNative->cLabels);
2337
2338	#ifdef RT_ARCH_AMD64
2339	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2340	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2341	{
2342	uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2343	if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2344	{
2345	pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2346	pbCodeBuf[off++] = (uint8_t)offRel;
2347	}
2348	else
2349	{
2350	offRel -= 3;
2351	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2352	pbCodeBuf[off++] = RT_BYTE1(offRel);
2353	pbCodeBuf[off++] = RT_BYTE2(offRel);
2354	pbCodeBuf[off++] = RT_BYTE3(offRel);
2355	pbCodeBuf[off++] = RT_BYTE4(offRel);
2356	}
2357	}
2358	else
2359	{
2360	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2361	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2362	pbCodeBuf[off++] = 0xfe;
2363	pbCodeBuf[off++] = 0xff;
2364	pbCodeBuf[off++] = 0xff;
2365	pbCodeBuf[off++] = 0xff;
2366	}
2367	pbCodeBuf[off++] = 0xcc; /* int3 poison */
2368
2369	#elif defined(RT_ARCH_ARM64)
2370	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2371	if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2372	pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2373	else
2374	{
2375	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2376	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2377	}
2378
2379	#else
2380	# error "Port me!"
2381	#endif
2382	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2383	return off;
2384	}
2385
2386
2387	/**
2388	* Emits a JMP rel32 / B imm19 to a new undefined label.
2389	*/
2390	DECL_INLINE_THROW(uint32_t)
2391	iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2392	{
2393	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2394	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2395	}
2396
2397	/** Condition type. */
2398	#ifdef RT_ARCH_AMD64
2399	typedef enum IEMNATIVEINSTRCOND : uint8_t
2400	{
2401	kIemNativeInstrCond_o = 0,
2402	kIemNativeInstrCond_no,
2403	kIemNativeInstrCond_c,
2404	kIemNativeInstrCond_nc,
2405	kIemNativeInstrCond_e,
2406	kIemNativeInstrCond_ne,
2407	kIemNativeInstrCond_be,
2408	kIemNativeInstrCond_nbe,
2409	kIemNativeInstrCond_s,
2410	kIemNativeInstrCond_ns,
2411	kIemNativeInstrCond_p,
2412	kIemNativeInstrCond_np,
2413	kIemNativeInstrCond_l,
2414	kIemNativeInstrCond_nl,
2415	kIemNativeInstrCond_le,
2416	kIemNativeInstrCond_nle
2417	} IEMNATIVEINSTRCOND;
2418	#elif defined(RT_ARCH_ARM64)
2419	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2420	#else
2421	# error "Port me!"
2422	#endif
2423
2424
2425	/**
2426	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2427	*/
2428	DECL_INLINE_THROW(uint32_t)
2429	iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2430	{
2431	Assert(idxLabel < pReNative->cLabels);
2432
2433	#ifdef RT_ARCH_AMD64
2434	/* jcc rel32 */
2435	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2436	pbCodeBuf[off++] = 0x0f;
2437	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2438	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2439	pbCodeBuf[off++] = 0x00;
2440	pbCodeBuf[off++] = 0x00;
2441	pbCodeBuf[off++] = 0x00;
2442	pbCodeBuf[off++] = 0x00;
2443
2444	#elif defined(RT_ARCH_ARM64)
2445	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2446	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2447	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2448
2449	#else
2450	# error "Port me!"
2451	#endif
2452	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2453	return off;
2454	}
2455
2456
2457	/**
2458	* Emits a Jcc rel32 / B.cc imm19 to a new label.
2459	*/
2460	DECL_INLINE_THROW(uint32_t)
2461	iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2462	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2463	{
2464	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
2465	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2466	}
2467
2468
2469	/**
2470	* Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2471	*/
2472	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2473	{
2474	#ifdef RT_ARCH_AMD64
2475	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2476	#elif defined(RT_ARCH_ARM64)
2477	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2478	#else
2479	# error "Port me!"
2480	#endif
2481	}
2482
2483	/**
2484	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2485	*/
2486	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2487	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2488	{
2489	#ifdef RT_ARCH_AMD64
2490	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2491	#elif defined(RT_ARCH_ARM64)
2492	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2493	#else
2494	# error "Port me!"
2495	#endif
2496	}
2497
2498
2499	/**
2500	* Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2501	*/
2502	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2503	{
2504	#ifdef RT_ARCH_AMD64
2505	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2506	#elif defined(RT_ARCH_ARM64)
2507	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2508	#else
2509	# error "Port me!"
2510	#endif
2511	}
2512
2513	/**
2514	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2515	*/
2516	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2517	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2518	{
2519	#ifdef RT_ARCH_AMD64
2520	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2521	#elif defined(RT_ARCH_ARM64)
2522	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2523	#else
2524	# error "Port me!"
2525	#endif
2526	}
2527
2528
2529	/**
2530	* Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2531	*/
2532	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2533	{
2534	#ifdef RT_ARCH_AMD64
2535	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2536	#elif defined(RT_ARCH_ARM64)
2537	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2538	#else
2539	# error "Port me!"
2540	#endif
2541	}
2542
2543	/**
2544	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2545	*/
2546	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2547	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2548	{
2549	#ifdef RT_ARCH_AMD64
2550	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2551	#elif defined(RT_ARCH_ARM64)
2552	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2553	#else
2554	# error "Port me!"
2555	#endif
2556	}
2557
2558
2559	/**
2560	* Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2561	*/
2562	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2563	{
2564	#ifdef RT_ARCH_AMD64
2565	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2566	#elif defined(RT_ARCH_ARM64)
2567	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2568	#else
2569	# error "Port me!"
2570	#endif
2571	}
2572
2573	/**
2574	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2575	*/
2576	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2577	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2578	{
2579	#ifdef RT_ARCH_AMD64
2580	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2581	#elif defined(RT_ARCH_ARM64)
2582	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2583	#else
2584	# error "Port me!"
2585	#endif
2586	}
2587
2588
2589	/**
2590	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2591	* How @a offJmp is applied is are target specific.
2592	*/
2593	DECL_INLINE_THROW(uint32_t)
2594	iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2595	{
2596	#ifdef RT_ARCH_AMD64
2597	/* jcc rel32 */
2598	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2599	if (offTarget < 128 && offTarget >= -128)
2600	{
2601	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x70;
2602	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2603	}
2604	else
2605	{
2606	pbCodeBuf[off++] = 0x0f;
2607	pbCodeBuf[off++] = (uint8_t)enmCond \| 0x80;
2608	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2609	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2610	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2611	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2612	}
2613
2614	#elif defined(RT_ARCH_ARM64)
2615	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2616	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2617
2618	#else
2619	# error "Port me!"
2620	#endif
2621	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2622	return off;
2623	}
2624
2625
2626	/**
2627	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2628	* How @a offJmp is applied is are target specific.
2629	*/
2630	DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2631	{
2632	#ifdef RT_ARCH_AMD64
2633	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2634	#elif defined(RT_ARCH_ARM64)
2635	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2636	#else
2637	# error "Port me!"
2638	#endif
2639	}
2640
2641
2642	/**
2643	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2644	* How @a offJmp is applied is are target specific.
2645	*/
2646	DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2647	{
2648	#ifdef RT_ARCH_AMD64
2649	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2650	#elif defined(RT_ARCH_ARM64)
2651	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2652	#else
2653	# error "Port me!"
2654	#endif
2655	}
2656
2657
2658	/**
2659	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2660	* How @a offJmp is applied is are target specific.
2661	*/
2662	DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2663	{
2664	#ifdef RT_ARCH_AMD64
2665	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2666	#elif defined(RT_ARCH_ARM64)
2667	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2668	#else
2669	# error "Port me!"
2670	#endif
2671	}
2672
2673
2674	/**
2675	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2676	* How @a offJmp is applied is are target specific.
2677	*/
2678	DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2679	{
2680	#ifdef RT_ARCH_AMD64
2681	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2682	#elif defined(RT_ARCH_ARM64)
2683	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2684	#else
2685	# error "Port me!"
2686	#endif
2687	}
2688
2689
2690	/**
2691	* Fixes up a conditional jump to a fixed label.
2692	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2693	*/
2694	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2695	{
2696	# if defined(RT_ARCH_AMD64)
2697	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2698	if (pbCodeBuf[offFixup] != 0x0f)
2699	{
2700	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2701	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2702	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2703	}
2704	else
2705	{
2706	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2707	uint32_t const offRel32 = offTarget - (offFixup + 6);
2708	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2709	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2710	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2711	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2712	}
2713
2714	# elif defined(RT_ARCH_ARM64)
2715	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2716
2717	int32_t const offDisp = offTarget - offFixup;
2718	Assert(offDisp >= -262144 && offDisp < 262144);
2719	Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
2720
2721	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
2722	\| (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
2723
2724	# endif
2725	}
2726
2727
2728	/**
2729	* Internal helper, don't call directly.
2730	*/
2731	DECL_INLINE_THROW(uint32_t)
2732	iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2733	uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
2734	{
2735	Assert(iBitNo < 64);
2736	#ifdef RT_ARCH_AMD64
2737	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2738	if (iBitNo < 8)
2739	{
2740	/* test Eb, imm8 */
2741	if (iGprSrc >= 4)
2742	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2743	pbCodeBuf[off++] = 0xf6;
2744	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2745	pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2746	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2747	}
2748	else
2749	{
2750	/* bt Ev, imm8 */
2751	if (iBitNo >= 32)
2752	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2753	else if (iGprSrc >= 8)
2754	pbCodeBuf[off++] = X86_OP_REX_B;
2755	pbCodeBuf[off++] = 0x0f;
2756	pbCodeBuf[off++] = 0xba;
2757	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2758	pbCodeBuf[off++] = iBitNo;
2759	off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2760	}
2761
2762	#elif defined(RT_ARCH_ARM64)
2763	/* Use the TBNZ instruction here. */
2764	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2765	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
2766	pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2767
2768	#else
2769	# error "Port me!"
2770	#endif
2771	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2772	return off;
2773	}
2774
2775
2776	/**
2777	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2778	* @a iGprSrc.
2779	*
2780	* @note On ARM64 the range is only +/-8191 instructions.
2781	*/
2782	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2783	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2784	{
2785	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /fJmpIfSet/);
2786	}
2787
2788
2789	/**
2790	* Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2791	* _set_ in @a iGprSrc.
2792	*
2793	* @note On ARM64 the range is only +/-8191 instructions.
2794	*/
2795	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2796	uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2797	{
2798	return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /fJmpIfSet/);
2799	}
2800
2801
2802	/**
2803	* Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2804	* flags accordingly.
2805	*/
2806	DECL_INLINE_THROW(uint32_t)
2807	iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2808	{
2809	Assert(fBits != 0);
2810	#ifdef RT_ARCH_AMD64
2811
2812	if (fBits >= UINT32_MAX)
2813	{
2814	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2815
2816	/* test Ev,Gv */
2817	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2818	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R) \| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2819	pbCodeBuf[off++] = 0x85;
2820	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2821
2822	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2823	}
2824	else if (fBits <= UINT32_MAX)
2825	{
2826	/* test Eb, imm8 or test Ev, imm32 */
2827	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2828	if (fBits <= UINT8_MAX)
2829	{
2830	if (iGprSrc >= 4)
2831	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2832	pbCodeBuf[off++] = 0xf6;
2833	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2834	pbCodeBuf[off++] = (uint8_t)fBits;
2835	}
2836	else
2837	{
2838	if (iGprSrc >= 8)
2839	pbCodeBuf[off++] = X86_OP_REX_B;
2840	pbCodeBuf[off++] = 0xf7;
2841	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2842	pbCodeBuf[off++] = RT_BYTE1(fBits);
2843	pbCodeBuf[off++] = RT_BYTE2(fBits);
2844	pbCodeBuf[off++] = RT_BYTE3(fBits);
2845	pbCodeBuf[off++] = RT_BYTE4(fBits);
2846	}
2847	}
2848	/** @todo implement me. */
2849	else
2850	AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
2851
2852	#elif defined(RT_ARCH_ARM64)
2853
2854	if (false)
2855	{
2856	/** @todo figure out how to work the immr / N:imms constants. */
2857	}
2858	else
2859	{
2860	/* ands Zr, iGprSrc, iTmpReg */
2861	uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2862	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2863	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2864	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2865	}
2866
2867	#else
2868	# error "Port me!"
2869	#endif
2870	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2871	return off;
2872	}
2873
2874
2875	/**
2876	* Emits a test for any of the bits from @a fBits in the lower 8 bits of
2877	* @a iGprSrc, setting CPU flags accordingly.
2878	*/
2879	DECL_INLINE_THROW(uint32_t)
2880	iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
2881	{
2882	Assert(fBits != 0);
2883
2884	#ifdef RT_ARCH_AMD64
2885	/* test Eb, imm8 */
2886	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2887	if (iGprSrc >= 4)
2888	pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
2889	pbCodeBuf[off++] = 0xf6;
2890	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2891	pbCodeBuf[off++] = fBits;
2892
2893	#elif defined(RT_ARCH_ARM64)
2894
2895	/* ands xzr, src, [tmp\|#imm] */
2896	uint32_t uImmR = 0;
2897	uint32_t uImmNandS = 0;
2898	if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2899	{
2900	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2901	pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprDst, uImmNandS, uImmR, false /f64Bit/);
2902	}
2903	else
2904	{
2905	/* Use temporary register for the 64-bit immediate. */
2906	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2907	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2908	pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /f64Bit/);
2909	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2910	}
2911
2912	#else
2913	# error "Port me!"
2914	#endif
2915	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2916	return off;
2917	}
2918
2919
2920	/**
2921	* Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2922	* are set in @a iGprSrc.
2923	*/
2924	DECL_INLINE_THROW(uint32_t)
2925	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2926	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2927	{
2928	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2929
2930	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2931	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2932
2933	return off;
2934	}
2935
2936
2937	/**
2938	* Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2939	* are set in @a iGprSrc.
2940	*/
2941	DECL_INLINE_THROW(uint32_t)
2942	iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2943	uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2944	{
2945	Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2946
2947	off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2948	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2949
2950	return off;
2951	}
2952
2953
2954	/**
2955	* Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
2956	*
2957	* The operand size is given by @a f64Bit.
2958	*/
2959	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2960	uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
2961	{
2962	Assert(idxLabel < pReNative->cLabels);
2963
2964	#ifdef RT_ARCH_AMD64
2965	/* test reg32,reg32 / test reg64,reg64 */
2966	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2967	if (f64Bit)
2968	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprSrc < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
2969	else if (iGprSrc >= 8)
2970	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
2971	pbCodeBuf[off++] = 0x85;
2972	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
2973	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2974
2975	/* jz idxLabel */
2976	off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2977
2978	#elif defined(RT_ARCH_ARM64)
2979	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2980	iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2981	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 0, iGprSrc, f64Bit);
2982	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2983
2984	#else
2985	# error "Port me!"
2986	#endif
2987	return off;
2988	}
2989
2990
2991	/**
2992	* Emits code that jumps to a new label if @a iGprSrc is zero.
2993	*
2994	* The operand size is given by @a f64Bit.
2995	*/
2996	DECL_INLINE_THROW(uint32_t)
2997	iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
2998	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2999	{
3000	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3001	return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3002	}
3003
3004
3005	/**
3006	* Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3007	* differs.
3008	*/
3009	DECL_INLINE_THROW(uint32_t)
3010	iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3011	uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3012	{
3013	off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3014	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3015	return off;
3016	}
3017
3018
3019	/**
3020	* Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3021	*/
3022	DECL_INLINE_THROW(uint32_t)
3023	iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3024	uint8_t iGprLeft, uint8_t iGprRight,
3025	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3026	{
3027	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3028	return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3029	}
3030
3031
3032	/**
3033	* Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3034	*/
3035	DECL_INLINE_THROW(uint32_t)
3036	iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3037	uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3038	{
3039	off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3040	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3041	return off;
3042	}
3043
3044
3045	/**
3046	* Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3047	*/
3048	DECL_INLINE_THROW(uint32_t)
3049	iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3050	uint8_t iGprSrc, uint64_t uImm,
3051	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3052	{
3053	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3054	return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3055	}
3056
3057
3058	/**
3059	* Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3060	* @a uImm.
3061	*/
3062	DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3063	uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3064	{
3065	off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3066	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3067	return off;
3068	}
3069
3070
3071	/**
3072	* Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3073	* @a uImm.
3074	*/
3075	DECL_INLINE_THROW(uint32_t)
3076	iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3077	uint8_t iGprSrc, uint32_t uImm,
3078	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3079	{
3080	uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
3081	return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3082	}
3083
3084
3085	/*********************************************************************************************************************************
3086	* Calls. *
3087	*********************************************************************************************************************************/
3088
3089	/**
3090	* Emits a call to a 64-bit address.
3091	*/
3092	DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3093	{
3094	#ifdef RT_ARCH_AMD64
3095	off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3096
3097	/* call rax */
3098	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3099	pbCodeBuf[off++] = 0xff;
3100	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3101
3102	#elif defined(RT_ARCH_ARM64)
3103	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3104
3105	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3106	pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3107
3108	#else
3109	# error "port me"
3110	#endif
3111	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3112	return off;
3113	}
3114
3115
3116	/**
3117	* Emits code to load a stack variable into an argument GPR.
3118	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3119	*/
3120	DECL_FORCE_INLINE_THROW(uint32_t)
3121	iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3122	int32_t offAddend = 0)
3123	{
3124	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3125	AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3126	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3127
3128	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3129	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3130	{
3131	Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK));
3132	if (!offAddend)
3133	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
3134	else
3135	off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
3136	}
3137	else
3138	{
3139	uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
3140	AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
3141	off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
3142	if (offAddend)
3143	off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
3144	}
3145	return off;
3146	}
3147
3148
3149	/**
3150	* Emits code to load a stack or immediate variable value into an argument GPR,
3151	* optional with a addend.
3152	* @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
3153	*/
3154	DECL_FORCE_INLINE_THROW(uint32_t)
3155	iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3156	int32_t offAddend = 0)
3157	{
3158	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3159	if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
3160	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
3161	else
3162	off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend);
3163	return off;
3164	}
3165
3166
3167	/**
3168	* Emits code to load the variable address into an argument GRP.
3169	*
3170	* This only works for uninitialized and stack variables.
3171	*/
3172	DECL_FORCE_INLINE_THROW(uint32_t)
3173	iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
3174	bool fFlushShadows)
3175	{
3176	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
3177	AssertStmt( pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
3178	\|\| pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
3179	IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
3180
3181	uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
3182	int32_t const offBpDisp = iemNativeStackCalcBpDisp(idxStackSlot);
3183
3184	uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
3185	if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
3186	{
3187	off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
3188	iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
3189	Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3190	}
3191	Assert( pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
3192	&& pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
3193
3194	return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
3195	}
3196
3197
3198	/** @} */
3199
3200	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */
3201

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h@ 102479

Download in other formats: