VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101913

Last change on this file since 101913 was 101913, checked in by vboxsync, 17 months ago

VMM/IEM: Release build fix for arm64. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 124.7 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101913 2023-11-07 01:45:38Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46
47/** @name Stack Frame Layout
48 *
49 * @{ */
50/** The size of the area for stack variables and spills and stuff.
51 * @note This limit is duplicated in the python script(s). We add 0x40 for
52 * alignment padding. */
53#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
54/** Number of 64-bit variable slots (0x100 / 8 = 32. */
55#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
56AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
57
58#ifdef RT_ARCH_AMD64
59/** An stack alignment adjustment (between non-volatile register pushes and
60 * the stack variable area, so the latter better aligned). */
61# define IEMNATIVE_FRAME_ALIGN_SIZE 8
62
63/** Number of stack arguments slots for calls made from the frame. */
64# ifdef RT_OS_WINDOWS
65# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
66# else
67# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
68# endif
69/** Number of any shadow arguments (spill area) for calls we make. */
70# ifdef RT_OS_WINDOWS
71# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
72# else
73# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
74# endif
75
76/** Frame pointer (RBP) relative offset of the last push. */
77# ifdef RT_OS_WINDOWS
78# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
79# else
80# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
81# endif
82/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
83 * address for it). */
84# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
85/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
86# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
87/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
88# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
89# ifdef RT_OS_WINDOWS
90/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
91# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
92/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
93# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
94# endif
95
96# ifdef RT_OS_WINDOWS
97/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
98# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
99/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
100# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
101/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
102# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
103/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
104# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
105# endif
106
107#elif RT_ARCH_ARM64
108/** No alignment padding needed for arm64. */
109# define IEMNATIVE_FRAME_ALIGN_SIZE 0
110/** No stack argument slots, got 8 registers for arguments will suffice. */
111# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
112/** There are no argument spill area. */
113# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
114
115/** Number of saved registers at the top of our stack frame.
116 * This includes the return address and old frame pointer, so x19 thru x30. */
117# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
118/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
119# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
120
121/** Frame pointer (BP) relative offset of the last push. */
122# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
123
124/** Frame pointer (BP) relative offset of the stack variable area (the lowest
125 * address for it). */
126# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
127
128#else
129# error "port me"
130#endif
131/** @} */
132
133
134/** @name Fixed Register Allocation(s)
135 * @{ */
136/** @def IEMNATIVE_REG_FIXED_PVMCPU
137 * The number of the register holding the pVCpu pointer. */
138/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
139 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
140 * @note This not available on AMD64, only ARM64. */
141/** @def IEMNATIVE_REG_FIXED_TMP0
142 * Dedicated temporary register.
143 * @todo replace this by a register allocator and content tracker. */
144/** @def IEMNATIVE_REG_FIXED_MASK
145 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
146 * architecture. */
147#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
148# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
149# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
150# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
151 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
152 | RT_BIT_32(X86_GREG_xSP) \
153 | RT_BIT_32(X86_GREG_xBP) )
154
155#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
156# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
157# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
158# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
159# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
160 | RT_BIT_32(ARMV8_A64_REG_LR) \
161 | RT_BIT_32(ARMV8_A64_REG_BP) \
162 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
163 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
164 | RT_BIT_32(ARMV8_A64_REG_X18) \
165 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
166
167#else
168# error "port me"
169#endif
170/** @} */
171
172/** @name Call related registers.
173 * @{ */
174/** @def IEMNATIVE_CALL_RET_GREG
175 * The return value register. */
176/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
177 * Number of arguments in registers. */
178/** @def IEMNATIVE_CALL_ARG0_GREG
179 * The general purpose register carrying argument \#0. */
180/** @def IEMNATIVE_CALL_ARG1_GREG
181 * The general purpose register carrying argument \#1. */
182/** @def IEMNATIVE_CALL_ARG2_GREG
183 * The general purpose register carrying argument \#2. */
184/** @def IEMNATIVE_CALL_ARG3_GREG
185 * The general purpose register carrying argument \#3. */
186/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
187 * Mask of registers the callee will not save and may trash. */
188#ifdef RT_ARCH_AMD64
189# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
190
191# ifdef RT_OS_WINDOWS
192# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
193# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
194# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
195# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
196# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
197# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
198 | RT_BIT_32(X86_GREG_xCX) \
199 | RT_BIT_32(X86_GREG_xDX) \
200 | RT_BIT_32(X86_GREG_x8) \
201 | RT_BIT_32(X86_GREG_x9) \
202 | RT_BIT_32(X86_GREG_x10) \
203 | RT_BIT_32(X86_GREG_x11) )
204# else
205# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
206# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
207# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
208# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
209# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
210# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
211# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
212# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
213 | RT_BIT_32(X86_GREG_xCX) \
214 | RT_BIT_32(X86_GREG_xDX) \
215 | RT_BIT_32(X86_GREG_xDI) \
216 | RT_BIT_32(X86_GREG_xSI) \
217 | RT_BIT_32(X86_GREG_x8) \
218 | RT_BIT_32(X86_GREG_x9) \
219 | RT_BIT_32(X86_GREG_x10) \
220 | RT_BIT_32(X86_GREG_x11) )
221# endif
222
223#elif defined(RT_ARCH_ARM64)
224# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
225# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
226# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
227# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
228# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
229# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
230# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
231# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
232# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
233# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
234# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
235 | RT_BIT_32(ARMV8_A64_REG_X1) \
236 | RT_BIT_32(ARMV8_A64_REG_X2) \
237 | RT_BIT_32(ARMV8_A64_REG_X3) \
238 | RT_BIT_32(ARMV8_A64_REG_X4) \
239 | RT_BIT_32(ARMV8_A64_REG_X5) \
240 | RT_BIT_32(ARMV8_A64_REG_X6) \
241 | RT_BIT_32(ARMV8_A64_REG_X7) \
242 | RT_BIT_32(ARMV8_A64_REG_X8) \
243 | RT_BIT_32(ARMV8_A64_REG_X9) \
244 | RT_BIT_32(ARMV8_A64_REG_X10) \
245 | RT_BIT_32(ARMV8_A64_REG_X11) \
246 | RT_BIT_32(ARMV8_A64_REG_X12) \
247 | RT_BIT_32(ARMV8_A64_REG_X13) \
248 | RT_BIT_32(ARMV8_A64_REG_X14) \
249 | RT_BIT_32(ARMV8_A64_REG_X15) \
250 | RT_BIT_32(ARMV8_A64_REG_X16) \
251 | RT_BIT_32(ARMV8_A64_REG_X17) )
252
253#endif
254
255/** This is the maximum argument count we'll ever be needing. */
256#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
257/** @} */
258
259
260/** @def IEMNATIVE_HST_GREG_COUNT
261 * Number of host general purpose registers we tracker. */
262/** @def IEMNATIVE_HST_GREG_MASK
263 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
264 * inverted register masks and such to get down to a correct set of regs. */
265#ifdef RT_ARCH_AMD64
266# define IEMNATIVE_HST_GREG_COUNT 16
267# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
268
269#elif defined(RT_ARCH_ARM64)
270# define IEMNATIVE_HST_GREG_COUNT 32
271# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
272#else
273# error "Port me!"
274#endif
275
276
277/** Native code generator label types. */
278typedef enum
279{
280 kIemNativeLabelType_Invalid = 0,
281 /* Labels w/o data, only once instance per TB: */
282 kIemNativeLabelType_Return,
283 kIemNativeLabelType_ReturnBreak,
284 kIemNativeLabelType_ReturnWithFlags,
285 kIemNativeLabelType_NonZeroRetOrPassUp,
286 kIemNativeLabelType_RaiseGp0,
287 /* Labels with data, potentially multiple instances per TB: */
288 kIemNativeLabelType_If,
289 kIemNativeLabelType_Else,
290 kIemNativeLabelType_Endif,
291 kIemNativeLabelType_CheckIrq,
292 kIemNativeLabelType_End
293} IEMNATIVELABELTYPE;
294
295/** Native code generator label definition. */
296typedef struct IEMNATIVELABEL
297{
298 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
299 * the epilog. */
300 uint32_t off;
301 /** The type of label (IEMNATIVELABELTYPE). */
302 uint16_t enmType;
303 /** Additional label data, type specific. */
304 uint16_t uData;
305} IEMNATIVELABEL;
306/** Pointer to a label. */
307typedef IEMNATIVELABEL *PIEMNATIVELABEL;
308
309
310/** Native code generator fixup types. */
311typedef enum
312{
313 kIemNativeFixupType_Invalid = 0,
314#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
315 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
316 kIemNativeFixupType_Rel32,
317#elif defined(RT_ARCH_ARM64)
318 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
319 kIemNativeFixupType_RelImm26At0,
320 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
321 kIemNativeFixupType_RelImm19At5,
322 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
323 kIemNativeFixupType_RelImm14At5,
324#endif
325 kIemNativeFixupType_End
326} IEMNATIVEFIXUPTYPE;
327
328/** Native code generator fixup. */
329typedef struct IEMNATIVEFIXUP
330{
331 /** Code offset of the fixup location. */
332 uint32_t off;
333 /** The IEMNATIVELABEL this is a fixup for. */
334 uint16_t idxLabel;
335 /** The fixup type (IEMNATIVEFIXUPTYPE). */
336 uint8_t enmType;
337 /** Addend or other data. */
338 int8_t offAddend;
339} IEMNATIVEFIXUP;
340/** Pointer to a native code generator fixup. */
341typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
342
343
344/**
345 * Guest registers that can be shadowed in GPRs.
346 */
347typedef enum IEMNATIVEGSTREG : uint8_t
348{
349 kIemNativeGstReg_GprFirst = 0,
350 kIemNativeGstReg_GprLast = 15,
351 kIemNativeGstReg_Pc,
352 kIemNativeGstReg_EFlags, /**< This one is problematic since the higher bits are used internally. */
353 /* gap: 18..23 */
354 kIemNativeGstReg_SegSelFirst = 24,
355 kIemNativeGstReg_SegSelLast = 29,
356 kIemNativeGstReg_SegBaseFirst = 30,
357 kIemNativeGstReg_SegBaseLast = 35,
358 kIemNativeGstReg_SegLimitFirst = 36,
359 kIemNativeGstReg_SegLimitLast = 41,
360 kIemNativeGstReg_End
361} IEMNATIVEGSTREG;
362
363/**
364 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
365 */
366typedef enum IEMNATIVEGSTREGUSE
367{
368 /** The usage is read-only, the register holding the guest register
369 * shadow copy will not be modified by the caller. */
370 kIemNativeGstRegUse_ReadOnly = 0,
371 /** The caller will update the guest register (think: PC += cbInstr).
372 * The guest shadow copy will follow the returned register. */
373 kIemNativeGstRegUse_ForUpdate,
374 /** The caller will use the guest register value as input in a calculation
375 * and the host register will be modified.
376 * This means that the returned host register will not be marked as a shadow
377 * copy of the guest register. */
378 kIemNativeGstRegUse_Calculation
379} IEMNATIVEGSTREGUSE;
380
381/**
382 * Guest registers (classes) that can be referenced.
383 */
384typedef enum IEMNATIVEGSTREGREF : uint8_t
385{
386 kIemNativeGstRegRef_Invalid = 0,
387 kIemNativeGstRegRef_Gpr,
388 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
389 kIemNativeGstRegRef_EFlags,
390 kIemNativeGstRegRef_MxCsr,
391 kIemNativeGstRegRef_FpuReg,
392 kIemNativeGstRegRef_MReg,
393 kIemNativeGstRegRef_XReg,
394 //kIemNativeGstRegRef_YReg, - doesn't work.
395 kIemNativeGstRegRef_End
396} IEMNATIVEGSTREGREF;
397
398
399/** Variable kinds. */
400typedef enum IEMNATIVEVARKIND : uint8_t
401{
402 /** Customary invalid zero value. */
403 kIemNativeVarKind_Invalid = 0,
404 /** This is either in a register or on the stack. */
405 kIemNativeVarKind_Stack,
406 /** Immediate value - loaded into register when needed, or can live on the
407 * stack if referenced (in theory). */
408 kIemNativeVarKind_Immediate,
409 /** Variable reference - loaded into register when needed, never stack. */
410 kIemNativeVarKind_VarRef,
411 /** Guest register reference - loaded into register when needed, never stack. */
412 kIemNativeVarKind_GstRegRef,
413 /** End of valid values. */
414 kIemNativeVarKind_End
415} IEMNATIVEVARKIND;
416
417
418/** Variable or argument. */
419typedef struct IEMNATIVEVAR
420{
421 /** The kind of variable. */
422 IEMNATIVEVARKIND enmKind;
423 /** The variable size in bytes. */
424 uint8_t cbVar;
425 /** The first stack slot (uint64_t), except for immediate and references
426 * where it usually is UINT8_MAX. */
427 uint8_t idxStackSlot;
428 /** The host register allocated for the variable, UINT8_MAX if not. */
429 uint8_t idxReg;
430 /** The argument number if argument, UINT8_MAX if regular variable. */
431 uint8_t uArgNo;
432 /** If referenced, the index of the variable referencing this one, otherwise
433 * UINT8_MAX. A referenced variable must only be placed on the stack and
434 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
435 uint8_t idxReferrerVar;
436 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
437 IEMNATIVEGSTREG enmGstReg;
438 uint8_t bAlign;
439
440 union
441 {
442 /** kIemNativeVarKind_Immediate: The immediate value. */
443 uint64_t uValue;
444 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
445 uint8_t idxRefVar;
446 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
447 struct
448 {
449 /** The class of register. */
450 IEMNATIVEGSTREGREF enmClass;
451 /** Index within the class. */
452 uint8_t idx;
453 } GstRegRef;
454 } u;
455} IEMNATIVEVAR;
456
457/** What is being kept in a host register. */
458typedef enum IEMNATIVEWHAT : uint8_t
459{
460 /** The traditional invalid zero value. */
461 kIemNativeWhat_Invalid = 0,
462 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
463 kIemNativeWhat_Var,
464 /** Temporary register, this is typically freed when a MC completes. */
465 kIemNativeWhat_Tmp,
466 /** Call argument w/o a variable mapping. This is free (via
467 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
468 kIemNativeWhat_Arg,
469 /** Return status code.
470 * @todo not sure if we need this... */
471 kIemNativeWhat_rc,
472 /** The fixed pVCpu (PVMCPUCC) register.
473 * @todo consider offsetting this on amd64 to use negative offsets to access
474 * more members using 8-byte disp. */
475 kIemNativeWhat_pVCpuFixed,
476 /** The fixed pCtx (PCPUMCTX) register.
477 * @todo consider offsetting this on amd64 to use negative offsets to access
478 * more members using 8-byte disp. */
479 kIemNativeWhat_pCtxFixed,
480 /** Fixed temporary register. */
481 kIemNativeWhat_FixedTmp,
482 /** Register reserved by the CPU or OS architecture. */
483 kIemNativeWhat_FixedReserved,
484 /** End of valid values. */
485 kIemNativeWhat_End
486} IEMNATIVEWHAT;
487
488/**
489 * Host general register entry.
490 *
491 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
492 *
493 * @todo Track immediate values in host registers similarlly to how we track the
494 * guest register shadow copies. For it to be real helpful, though,
495 * we probably need to know which will be reused and put them into
496 * non-volatile registers, otherwise it's going to be more or less
497 * restricted to an instruction or two.
498 */
499typedef struct IEMNATIVEHSTREG
500{
501 /** Set of guest registers this one shadows.
502 *
503 * Using a bitmap here so we can designate the same host register as a copy
504 * for more than one guest register. This is expected to be useful in
505 * situations where one value is copied to several registers in a sequence.
506 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
507 * sequence we'd want to let this register follow to be a copy of and there
508 * will always be places where we'd be picking the wrong one.
509 */
510 uint64_t fGstRegShadows;
511 /** What is being kept in this register. */
512 IEMNATIVEWHAT enmWhat;
513 /** Variable index if holding a variable, otherwise UINT8_MAX. */
514 uint8_t idxVar;
515 /** Alignment padding. */
516 uint8_t abAlign[6];
517} IEMNATIVEHSTREG;
518
519
520/**
521 * Core state for the native recompiler, that is, things that needs careful
522 * handling when dealing with branches.
523 */
524typedef struct IEMNATIVECORESTATE
525{
526 /** Allocation bitmap for aHstRegs. */
527 uint32_t bmHstRegs;
528
529 /** Bitmap marking which host register contains guest register shadow copies.
530 * This is used during register allocation to try preserve copies. */
531 uint32_t bmHstRegsWithGstShadow;
532 /** Bitmap marking valid entries in aidxGstRegShadows. */
533 uint64_t bmGstRegShadows;
534
535 union
536 {
537 /** Index of variable arguments, UINT8_MAX if not valid. */
538 uint8_t aidxArgVars[8];
539 /** For more efficient resetting. */
540 uint64_t u64ArgVars;
541 };
542
543 /** Allocation bitmap for the stack. */
544 uint32_t bmStack;
545 /** Allocation bitmap for aVars. */
546 uint32_t bmVars;
547
548 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
549 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
550 * (A shadow copy of a guest register can only be held in a one host register,
551 * there are no duplicate copies or ambiguities like that). */
552 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
553
554 /** Host register allocation tracking. */
555 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
556
557 /** Variables and arguments. */
558 IEMNATIVEVAR aVars[9];
559} IEMNATIVECORESTATE;
560/** Pointer to core state. */
561typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
562/** Pointer to const core state. */
563typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
564
565
566/**
567 * Conditional stack entry.
568 */
569typedef struct IEMNATIVECOND
570{
571 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
572 bool fInElse;
573 /** The label for the IEM_MC_ELSE. */
574 uint32_t idxLabelElse;
575 /** The label for the IEM_MC_ENDIF. */
576 uint32_t idxLabelEndIf;
577 /** The initial state snapshot as the if-block starts executing. */
578 IEMNATIVECORESTATE InitialState;
579 /** The state snapshot at the end of the if-block. */
580 IEMNATIVECORESTATE IfFinalState;
581} IEMNATIVECOND;
582/** Pointer to a condition stack entry. */
583typedef IEMNATIVECOND *PIEMNATIVECOND;
584
585
586/**
587 * Native recompiler state.
588 */
589typedef struct IEMRECOMPILERSTATE
590{
591 /** Size of the buffer that pbNativeRecompileBufR3 points to in
592 * IEMNATIVEINSTR units. */
593 uint32_t cInstrBufAlloc;
594#ifdef VBOX_STRICT
595 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
596 uint32_t offInstrBufChecked;
597#else
598 uint32_t uPadding1; /* We don't keep track of the size here... */
599#endif
600 /** Fixed temporary code buffer for native recompilation. */
601 PIEMNATIVEINSTR pInstrBuf;
602
603 /** Bitmaps with the label types used. */
604 uint64_t bmLabelTypes;
605 /** Actual number of labels in paLabels. */
606 uint32_t cLabels;
607 /** Max number of entries allowed in paLabels before reallocating it. */
608 uint32_t cLabelsAlloc;
609 /** Labels defined while recompiling (referenced by fixups). */
610 PIEMNATIVELABEL paLabels;
611
612 /** Actual number of fixups paFixups. */
613 uint32_t cFixups;
614 /** Max number of entries allowed in paFixups before reallocating it. */
615 uint32_t cFixupsAlloc;
616 /** Buffer used by the recompiler for recording fixups when generating code. */
617 PIEMNATIVEFIXUP paFixups;
618
619#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
620 /** Number of debug info entries allocated for pDbgInfo. */
621 uint32_t cDbgInfoAlloc;
622 uint32_t uPadding;
623 /** Debug info. */
624 PIEMTBDBG pDbgInfo;
625#endif
626
627 /** The translation block being recompiled. */
628 PCIEMTB pTbOrg;
629
630 /** The current condition stack depth (aCondStack). */
631 uint8_t cCondDepth;
632 uint8_t bPadding2;
633 /** Condition sequence number (for generating unique labels). */
634 uint16_t uCondSeqNo;
635 /** Check IRQ seqeunce number (for generating unique lables). */
636 uint16_t uCheckIrqSeqNo;
637 uint8_t bPadding3;
638
639 /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
640 uint8_t cArgs;
641 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
642 uint32_t fCImpl;
643 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
644 uint32_t fMc;
645
646 /** Core state requiring care with branches. */
647 IEMNATIVECORESTATE Core;
648
649 /** The condition nesting stack. */
650 IEMNATIVECOND aCondStack[2];
651
652#ifndef IEM_WITH_THROW_CATCH
653 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
654 * for recompilation error handling. */
655 jmp_buf JmpBuf;
656#endif
657} IEMRECOMPILERSTATE;
658/** Pointer to a native recompiler state. */
659typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
660
661
662/** @def IEMNATIVE_TRY_SETJMP
663 * Wrapper around setjmp / try, hiding all the ugly differences.
664 *
665 * @note Use with extreme care as this is a fragile macro.
666 * @param a_pReNative The native recompile state.
667 * @param a_rcTarget The variable that should receive the status code in case
668 * of a longjmp/throw.
669 */
670/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
671 * Start wrapper for catch / setjmp-else.
672 *
673 * This will set up a scope.
674 *
675 * @note Use with extreme care as this is a fragile macro.
676 * @param a_pReNative The native recompile state.
677 * @param a_rcTarget The variable that should receive the status code in case
678 * of a longjmp/throw.
679 */
680/** @def IEMNATIVE_CATCH_LONGJMP_END
681 * End wrapper for catch / setjmp-else.
682 *
683 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
684 * up the state.
685 *
686 * @note Use with extreme care as this is a fragile macro.
687 * @param a_pReNative The native recompile state.
688 */
689/** @def IEMNATIVE_DO_LONGJMP
690 *
691 * Wrapper around longjmp / throw.
692 *
693 * @param a_pReNative The native recompile state.
694 * @param a_rc The status code jump back with / throw.
695 */
696#ifdef IEM_WITH_THROW_CATCH
697# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
698 a_rcTarget = VINF_SUCCESS; \
699 try
700# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
701 catch (int rcThrown) \
702 { \
703 a_rcTarget = rcThrown
704# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
705 } \
706 ((void)0)
707# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
708#else /* !IEM_WITH_THROW_CATCH */
709# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
710 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
711# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
712 else \
713 { \
714 ((void)0)
715# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
716 }
717# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
718#endif /* !IEM_WITH_THROW_CATCH */
719
720
721/**
722 * Native recompiler worker for a threaded function.
723 *
724 * @returns New code buffer offset; throws VBox status code in case of a failure.
725 * @param pReNative The native recompiler state.
726 * @param off The current code buffer offset.
727 * @param pCallEntry The threaded call entry.
728 *
729 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
730 */
731typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
732/** Pointer to a native recompiler worker for a threaded function. */
733typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
734
735/** Defines a native recompiler worker for a threaded function.
736 * @see FNIEMNATIVERECOMPFUNC */
737#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
738 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
739
740/** Prototypes a native recompiler function for a threaded function.
741 * @see FNIEMNATIVERECOMPFUNC */
742#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
743
744DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
745 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
746DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
747DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
748 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
749DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
750
751DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
752DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
753 bool fPreferVolatile = true);
754DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
755 IEMNATIVEGSTREG enmGstReg, IEMNATIVEGSTREGUSE enmIntendedUse);
756DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
757 IEMNATIVEGSTREG enmGstReg);
758
759DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
760DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
761DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
762DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
763DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
764DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
765DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
766DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
767
768DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
769 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
770DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
771
772extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
773
774
775/**
776 * Ensures that there is sufficient space in the instruction output buffer.
777 *
778 * This will reallocate the buffer if needed and allowed.
779 *
780 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
781 * allocation size.
782 *
783 * @returns Pointer to the instruction output buffer on success; throws VBox
784 * status code on failure, so no need to check it.
785 * @param pReNative The native recompile state.
786 * @param off Current instruction offset. Works safely for UINT32_MAX
787 * as well.
788 * @param cInstrReq Number of instruction about to be added. It's okay to
789 * overestimate this a bit.
790 */
791DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
792iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
793{
794 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
795 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
796 {
797#ifdef VBOX_STRICT
798 pReNative->offInstrBufChecked = offChecked;
799#endif
800 return pReNative->pInstrBuf;
801 }
802 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
803}
804
805/**
806 * Checks that we didn't exceed the space requested in the last
807 * iemNativeInstrBufEnsure() call. */
808#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
809 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
810 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
811
812
813/**
814 * Emit a simple marker instruction to more easily tell where something starts
815 * in the disassembly.
816 */
817DECL_INLINE_THROW(uint32_t)
818iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
819{
820#ifdef RT_ARCH_AMD64
821 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
822 if (uInfo == 0)
823 {
824 /* nop */
825 pbCodeBuf[off++] = 0x90;
826 }
827 else
828 {
829 /* nop [disp32] */
830 pbCodeBuf[off++] = 0x0f;
831 pbCodeBuf[off++] = 0x1f;
832 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
833 pbCodeBuf[off++] = RT_BYTE1(uInfo);
834 pbCodeBuf[off++] = RT_BYTE2(uInfo);
835 pbCodeBuf[off++] = RT_BYTE3(uInfo);
836 pbCodeBuf[off++] = RT_BYTE4(uInfo);
837 }
838#elif RT_ARCH_ARM64
839 /* nop */
840 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
841 pu32CodeBuf[off++] = 0xd503201f;
842
843 RT_NOREF(uInfo);
844#else
845# error "port me"
846#endif
847 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
848 return off;
849}
850
851
852/*********************************************************************************************************************************
853* Loads, Stores and Related Stuff. *
854*********************************************************************************************************************************/
855
856/**
857 * Emits setting a GPR to zero.
858 */
859DECL_INLINE_THROW(uint32_t)
860iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
861{
862#ifdef RT_ARCH_AMD64
863 /* xor gpr32, gpr32 */
864 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
865 if (iGpr >= 8)
866 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
867 pbCodeBuf[off++] = 0x33;
868 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
869
870#elif RT_ARCH_ARM64
871 /* mov gpr, #0x0 */
872 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
873 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
874
875#else
876# error "port me"
877#endif
878 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
879 return off;
880}
881
882
883/**
884 * Emits loading a constant into a 64-bit GPR
885 */
886DECL_INLINE_THROW(uint32_t)
887iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
888{
889 if (!uImm64)
890 return iemNativeEmitGprZero(pReNative, off, iGpr);
891
892#ifdef RT_ARCH_AMD64
893 if (uImm64 <= UINT32_MAX)
894 {
895 /* mov gpr, imm32 */
896 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
897 if (iGpr >= 8)
898 pbCodeBuf[off++] = X86_OP_REX_B;
899 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
900 pbCodeBuf[off++] = RT_BYTE1(uImm64);
901 pbCodeBuf[off++] = RT_BYTE2(uImm64);
902 pbCodeBuf[off++] = RT_BYTE3(uImm64);
903 pbCodeBuf[off++] = RT_BYTE4(uImm64);
904 }
905 else
906 {
907 /* mov gpr, imm64 */
908 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
909 if (iGpr < 8)
910 pbCodeBuf[off++] = X86_OP_REX_W;
911 else
912 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
913 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
914 pbCodeBuf[off++] = RT_BYTE1(uImm64);
915 pbCodeBuf[off++] = RT_BYTE2(uImm64);
916 pbCodeBuf[off++] = RT_BYTE3(uImm64);
917 pbCodeBuf[off++] = RT_BYTE4(uImm64);
918 pbCodeBuf[off++] = RT_BYTE5(uImm64);
919 pbCodeBuf[off++] = RT_BYTE6(uImm64);
920 pbCodeBuf[off++] = RT_BYTE7(uImm64);
921 pbCodeBuf[off++] = RT_BYTE8(uImm64);
922 }
923
924#elif RT_ARCH_ARM64
925 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
926
927 /*
928 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
929 * supply remaining bits using 'movk grp, imm16, lsl #x'.
930 *
931 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
932 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
933 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
934 * after the first non-zero immediate component so we switch to movk for
935 * the remainder.
936 */
937 uint32_t fMovK = 0;
938 /* mov gpr, imm16 */
939 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
940 if (uImmPart)
941 {
942 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
943 fMovK |= RT_BIT_32(29);
944 }
945 /* mov[k] gpr, imm16, lsl #16 */
946 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
947 if (uImmPart)
948 {
949 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
950 fMovK |= RT_BIT_32(29);
951 }
952 /* mov[k] gpr, imm16, lsl #32 */
953 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
954 if (uImmPart)
955 {
956 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
957 fMovK |= RT_BIT_32(29);
958 }
959 /* mov[k] gpr, imm16, lsl #48 */
960 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
961 if (uImmPart)
962 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
963
964 /** @todo there is an inverted mask variant we might want to explore if it
965 * reduces the number of instructions... */
966 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
967 * clang 12.x does that, only to use the 'x' version for the
968 * addressing in the following ldr). */
969
970#else
971# error "port me"
972#endif
973 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
974 return off;
975}
976
977
978/**
979 * Emits loading a constant into a 8-bit GPR
980 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
981 * only the ARM64 version does that.
982 */
983DECL_INLINE_THROW(uint32_t)
984iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
985{
986#ifdef RT_ARCH_AMD64
987 /* mov gpr, imm8 */
988 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
989 if (iGpr >= 8)
990 pbCodeBuf[off++] = X86_OP_REX_B;
991 else if (iGpr >= 4)
992 pbCodeBuf[off++] = X86_OP_REX;
993 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
994 pbCodeBuf[off++] = RT_BYTE1(uImm8);
995
996#elif RT_ARCH_ARM64
997 /* movz gpr, imm16, lsl #0 */
998 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
999 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
1000
1001#else
1002# error "port me"
1003#endif
1004 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1005 return off;
1006}
1007
1008
1009#ifdef RT_ARCH_AMD64
1010/**
1011 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1012 */
1013DECL_FORCE_INLINE(uint32_t)
1014iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
1015{
1016 if (offVCpu < 128)
1017 {
1018 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
1019 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
1020 }
1021 else
1022 {
1023 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
1024 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
1025 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
1026 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
1027 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
1028 }
1029 return off;
1030}
1031#elif RT_ARCH_ARM64
1032/**
1033 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1034 */
1035DECL_FORCE_INLINE_THROW(uint32_t)
1036iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1037 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1038{
1039 /*
1040 * There are a couple of ldr variants that takes an immediate offset, so
1041 * try use those if we can, otherwise we have to use the temporary register
1042 * help with the addressing.
1043 */
1044 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
1045 {
1046 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
1047 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1048 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
1049 }
1050 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
1051 {
1052 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1053 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
1054 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
1055 }
1056 else
1057 {
1058 /* The offset is too large, so we must load it into a register and use
1059 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
1060 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1061 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
1062
1063 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1064 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
1065 IEMNATIVE_REG_FIXED_TMP0);
1066 }
1067 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1068 return off;
1069}
1070#endif
1071
1072
1073/**
1074 * Emits a 64-bit GPR load of a VCpu value.
1075 */
1076DECL_INLINE_THROW(uint32_t)
1077iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1078{
1079#ifdef RT_ARCH_AMD64
1080 /* mov reg64, mem64 */
1081 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1082 if (iGpr < 8)
1083 pbCodeBuf[off++] = X86_OP_REX_W;
1084 else
1085 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1086 pbCodeBuf[off++] = 0x8b;
1087 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
1088 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1089
1090#elif RT_ARCH_ARM64
1091 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1092
1093#else
1094# error "port me"
1095#endif
1096 return off;
1097}
1098
1099
1100/**
1101 * Emits a 32-bit GPR load of a VCpu value.
1102 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1103 */
1104DECL_INLINE_THROW(uint32_t)
1105iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1106{
1107#ifdef RT_ARCH_AMD64
1108 /* mov reg32, mem32 */
1109 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1110 if (iGpr >= 8)
1111 pbCodeBuf[off++] = X86_OP_REX_R;
1112 pbCodeBuf[off++] = 0x8b;
1113 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1114 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1115
1116#elif RT_ARCH_ARM64
1117 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1118
1119#else
1120# error "port me"
1121#endif
1122 return off;
1123}
1124
1125
1126/**
1127 * Emits a 16-bit GPR load of a VCpu value.
1128 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
1129 */
1130DECL_INLINE_THROW(uint32_t)
1131iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1132{
1133#ifdef RT_ARCH_AMD64
1134 /* movzx reg32, mem16 */
1135 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1136 if (iGpr >= 8)
1137 pbCodeBuf[off++] = X86_OP_REX_R;
1138 pbCodeBuf[off++] = 0x0f;
1139 pbCodeBuf[off++] = 0xb7;
1140 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1141 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1142
1143#elif RT_ARCH_ARM64
1144 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1145
1146#else
1147# error "port me"
1148#endif
1149 return off;
1150}
1151
1152
1153/**
1154 * Emits a 8-bit GPR load of a VCpu value.
1155 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
1156 */
1157DECL_INLINE_THROW(uint32_t)
1158iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1159{
1160#ifdef RT_ARCH_AMD64
1161 /* movzx reg32, mem8 */
1162 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1163 if (iGpr >= 8)
1164 pbCodeBuf[off++] = X86_OP_REX_R;
1165 pbCodeBuf[off++] = 0x0f;
1166 pbCodeBuf[off++] = 0xb6;
1167 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1168 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1169
1170#elif RT_ARCH_ARM64
1171 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1172
1173#else
1174# error "port me"
1175#endif
1176 return off;
1177}
1178
1179
1180/**
1181 * Emits a store of a GPR value to a 64-bit VCpu field.
1182 */
1183DECL_INLINE_THROW(uint32_t)
1184iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1185{
1186#ifdef RT_ARCH_AMD64
1187 /* mov mem64, reg64 */
1188 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1189 if (iGpr < 8)
1190 pbCodeBuf[off++] = X86_OP_REX_W;
1191 else
1192 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1193 pbCodeBuf[off++] = 0x89;
1194 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1195 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1196
1197#elif RT_ARCH_ARM64
1198 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1199
1200#else
1201# error "port me"
1202#endif
1203 return off;
1204}
1205
1206
1207/**
1208 * Emits a store of a GPR value to a 32-bit VCpu field.
1209 */
1210DECL_INLINE_THROW(uint32_t)
1211iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1212{
1213#ifdef RT_ARCH_AMD64
1214 /* mov mem32, reg32 */
1215 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1216 if (iGpr >= 8)
1217 pbCodeBuf[off++] = X86_OP_REX_R;
1218 pbCodeBuf[off++] = 0x89;
1219 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1220 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1221
1222#elif RT_ARCH_ARM64
1223 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1224
1225#else
1226# error "port me"
1227#endif
1228 return off;
1229}
1230
1231
1232/**
1233 * Emits a store of a GPR value to a 16-bit VCpu field.
1234 */
1235DECL_INLINE_THROW(uint32_t)
1236iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1237{
1238#ifdef RT_ARCH_AMD64
1239 /* mov mem16, reg16 */
1240 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1241 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1242 if (iGpr >= 8)
1243 pbCodeBuf[off++] = X86_OP_REX_R;
1244 pbCodeBuf[off++] = 0x89;
1245 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1246 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1247
1248#elif RT_ARCH_ARM64
1249 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1250
1251#else
1252# error "port me"
1253#endif
1254 return off;
1255}
1256
1257
1258/**
1259 * Emits a store of a GPR value to a 8-bit VCpu field.
1260 */
1261DECL_INLINE_THROW(uint32_t)
1262iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1263{
1264#ifdef RT_ARCH_AMD64
1265 /* mov mem8, reg8 */
1266 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1267 if (iGpr >= 8)
1268 pbCodeBuf[off++] = X86_OP_REX_R;
1269 pbCodeBuf[off++] = 0x88;
1270 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1271 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1272
1273#elif RT_ARCH_ARM64
1274 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1275
1276#else
1277# error "port me"
1278#endif
1279 return off;
1280}
1281
1282
1283/**
1284 * Emits a load effective address to a GRP of a VCpu field.
1285 */
1286DECL_INLINE_THROW(uint32_t)
1287iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
1288{
1289#ifdef RT_ARCH_AMD64
1290 /* lea gprdst, [rbx + offDisp] */
1291 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1292 if (iGprDst < 8)
1293 pbCodeBuf[off++] = X86_OP_REX_W;
1294 else
1295 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1296 pbCodeBuf[off++] = 0x8d;
1297 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
1298
1299#elif defined(RT_ARCH_ARM64)
1300 if (offVCpu < (unsigned)_4K)
1301 {
1302 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1303 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
1304 }
1305 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
1306 {
1307 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1308 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
1309 offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
1310 }
1311 else
1312 {
1313 Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1314 off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
1315 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1316 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
1317 }
1318
1319#else
1320# error "port me"
1321#endif
1322 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1323 return off;
1324}
1325
1326
1327/**
1328 * Emits a gprdst = gprsrc load.
1329 */
1330DECL_INLINE_THROW(uint32_t)
1331iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1332{
1333#ifdef RT_ARCH_AMD64
1334 /* mov gprdst, gprsrc */
1335 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1336 if ((iGprDst | iGprSrc) >= 8)
1337 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1338 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1339 : X86_OP_REX_W | X86_OP_REX_R;
1340 else
1341 pbCodeBuf[off++] = X86_OP_REX_W;
1342 pbCodeBuf[off++] = 0x8b;
1343 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1344
1345#elif RT_ARCH_ARM64
1346 /* mov dst, src; alias for: orr dst, xzr, src */
1347 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1348 pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
1349
1350#else
1351# error "port me"
1352#endif
1353 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1354 return off;
1355}
1356
1357
1358/**
1359 * Emits a gprdst = gprsrc[31:0] load.
1360 * @note Bits 63 thru 32 are cleared.
1361 */
1362DECL_INLINE_THROW(uint32_t)
1363iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1364{
1365#ifdef RT_ARCH_AMD64
1366 /* mov gprdst, gprsrc */
1367 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1368 if ((iGprDst | iGprSrc) >= 8)
1369 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1370 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
1371 : X86_OP_REX_R;
1372 pbCodeBuf[off++] = 0x8b;
1373 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1374
1375#elif RT_ARCH_ARM64
1376 /* mov dst32, src32; alias for: orr dst32, wzr, src32 */
1377 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1378 pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);
1379
1380#else
1381# error "port me"
1382#endif
1383 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1384 return off;
1385}
1386
1387
1388/**
1389 * Emits a gprdst = gprsrc[15:0] load.
1390 * @note Bits 63 thru 15 are cleared.
1391 */
1392DECL_INLINE_THROW(uint32_t)
1393iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1394{
1395#ifdef RT_ARCH_AMD64
1396 /* movzx Gv,Ew */
1397 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1398 if ((iGprDst | iGprSrc) >= 8)
1399 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1400 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
1401 : X86_OP_REX_R;
1402 pbCodeBuf[off++] = 0x0f;
1403 pbCodeBuf[off++] = 0xb7;
1404 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1405
1406#elif RT_ARCH_ARM64
1407 /* and gprdst, gprsrc, #0xffff */
1408 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1409# if 1
1410 Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
1411 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /*f64Bit*/);
1412# else
1413 Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
1414 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
1415# endif
1416
1417#else
1418# error "port me"
1419#endif
1420 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1421 return off;
1422}
1423
1424
1425/**
1426 * Emits a gprdst = gprsrc[7:0] load.
1427 * @note Bits 63 thru 8 are cleared.
1428 */
1429DECL_INLINE_THROW(uint32_t)
1430iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1431{
1432#ifdef RT_ARCH_AMD64
1433 /* movzx Gv,Eb */
1434 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1435 if ((iGprDst | iGprSrc) >= 8)
1436 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1437 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
1438 : X86_OP_REX_R;
1439 pbCodeBuf[off++] = 0x0f;
1440 pbCodeBuf[off++] = 0xb6;
1441 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1442
1443#elif RT_ARCH_ARM64
1444 /* and gprdst, gprsrc, #0xff */
1445 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1446# if 1
1447 Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
1448 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);
1449# else
1450 Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
1451 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
1452# endif
1453
1454#else
1455# error "port me"
1456#endif
1457 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1458 return off;
1459}
1460
1461#if 0 /** @todo */
1462/**
1463 * Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
1464 * @note Bits 63 thru 8 are cleared.
1465 */
1466DECL_INLINE_THROW(uint32_t)
1467iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1468{
1469#ifdef RT_ARCH_AMD64
1470 /* movzx Gv,Eb */
1471 /** @todo */
1472 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1473 if ((iGprDst | iGprSrc) >= 8)
1474 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B
1475 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
1476 : X86_OP_REX_R;
1477 pbCodeBuf[off++] = 0x0f;
1478 pbCodeBuf[off++] = 0xb6;
1479 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1480
1481#elif RT_ARCH_ARM64
1482 /* ubfx gprdst, gprsrc, #8, #8 */
1483 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1484 Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
1485 pu32CodeBuf[off++] = /** @todo ubfx */;
1486
1487#else
1488# error "port me"
1489#endif
1490 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1491 return off;
1492}
1493#endif
1494
1495#ifdef RT_ARCH_AMD64
1496/**
1497 * Common bit of iemNativeEmitLoadGprByBp and friends.
1498 */
1499DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1500 PIEMRECOMPILERSTATE pReNativeAssert)
1501{
1502 if (offDisp < 128 && offDisp >= -128)
1503 {
1504 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1505 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1506 }
1507 else
1508 {
1509 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1510 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1511 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1512 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1513 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1514 }
1515 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1516 return off;
1517}
1518#elif defined(RT_ARCH_ARM64)
1519/**
1520 * Common bit of iemNativeEmitLoadGprByBp and friends.
1521 */
1522DECL_FORCE_INLINE_THROW(uint32_t)
1523iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1524 int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1525{
1526 if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))
1527 {
1528 /* str w/ unsigned imm12 (scaled) */
1529 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1530 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);
1531 }
1532 else if (offDisp >= -256 && offDisp <= 256)
1533 {
1534 /* stur w/ signed imm9 (unscaled) */
1535 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1536 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);
1537 }
1538 else
1539 {
1540 /* Use temporary indexing register. */
1541 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1542 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1543 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,
1544 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1545 }
1546 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1547 return off;
1548}
1549#endif
1550
1551
1552/**
1553 * Emits a 64-bit GRP load instruction with an BP relative source address.
1554 */
1555DECL_INLINE_THROW(uint32_t)
1556iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1557{
1558#ifdef RT_ARCH_AMD64
1559 /* mov gprdst, qword [rbp + offDisp] */
1560 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1561 if (iGprDst < 8)
1562 pbCodeBuf[off++] = X86_OP_REX_W;
1563 else
1564 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1565 pbCodeBuf[off++] = 0x8b;
1566 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1567
1568#elif defined(RT_ARCH_ARM64)
1569 return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1570
1571#else
1572# error "port me"
1573#endif
1574}
1575
1576
1577/**
1578 * Emits a 32-bit GRP load instruction with an BP relative source address.
1579 */
1580DECL_INLINE_THROW(uint32_t)
1581iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1582{
1583#ifdef RT_ARCH_AMD64
1584 /* mov gprdst, dword [rbp + offDisp] */
1585 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1586 if (iGprDst >= 8)
1587 pbCodeBuf[off++] = X86_OP_REX_R;
1588 pbCodeBuf[off++] = 0x8b;
1589 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1590
1591#elif defined(RT_ARCH_ARM64)
1592 return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1593
1594#else
1595# error "port me"
1596#endif
1597}
1598
1599
1600/**
1601 * Emits a load effective address to a GRP with an BP relative source address.
1602 */
1603DECL_INLINE_THROW(uint32_t)
1604iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1605{
1606#ifdef RT_ARCH_AMD64
1607 /* lea gprdst, [rbp + offDisp] */
1608 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1609 if (iGprDst < 8)
1610 pbCodeBuf[off++] = X86_OP_REX_W;
1611 else
1612 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1613 pbCodeBuf[off++] = 0x8d;
1614 off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1615
1616#elif defined(RT_ARCH_ARM64)
1617 if ((uint32_t)offDisp < (unsigned)_4K)
1618 {
1619 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1620 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);
1621 }
1622 else if ((uint32_t)-offDisp < (unsigned)_4K)
1623 {
1624 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1625 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);
1626 }
1627 else
1628 {
1629 Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
1630 off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
1631 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1632 if (offDisp >= 0)
1633 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
1634 else
1635 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
1636 }
1637
1638#else
1639# error "port me"
1640#endif
1641
1642 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1643 return off;
1644}
1645
1646
1647/**
1648 * Emits a 64-bit GPR store with an BP relative destination address.
1649 *
1650 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1651 */
1652DECL_INLINE_THROW(uint32_t)
1653iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1654{
1655#ifdef RT_ARCH_AMD64
1656 /* mov qword [rbp + offDisp], gprdst */
1657 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1658 if (iGprSrc < 8)
1659 pbCodeBuf[off++] = X86_OP_REX_W;
1660 else
1661 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1662 pbCodeBuf[off++] = 0x89;
1663 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1664
1665#elif defined(RT_ARCH_ARM64)
1666 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1667 {
1668 /* str w/ unsigned imm12 (scaled) */
1669 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1670 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1671 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1672 }
1673 else if (offDisp >= -256 && offDisp <= 256)
1674 {
1675 /* stur w/ signed imm9 (unscaled) */
1676 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1677 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1678 }
1679 else
1680 {
1681 /* Use temporary indexing register. */
1682 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1683 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1684 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1685 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1686 }
1687 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1688 return off;
1689
1690#else
1691# error "Port me!"
1692#endif
1693}
1694
1695
1696/**
1697 * Emits a 64-bit immediate store with an BP relative destination address.
1698 *
1699 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1700 */
1701DECL_INLINE_THROW(uint32_t)
1702iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1703{
1704#ifdef RT_ARCH_AMD64
1705 if ((int64_t)uImm64 == (int32_t)uImm64)
1706 {
1707 /* mov qword [rbp + offDisp], imm32 - sign extended */
1708 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1709 pbCodeBuf[off++] = X86_OP_REX_W;
1710 pbCodeBuf[off++] = 0xc7;
1711 if (offDisp < 128 && offDisp >= -128)
1712 {
1713 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1714 pbCodeBuf[off++] = (uint8_t)offDisp;
1715 }
1716 else
1717 {
1718 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1719 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1720 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1721 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1722 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1723 }
1724 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1725 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1726 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1727 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1728 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1729 return off;
1730 }
1731#endif
1732
1733 /* Load tmp0, imm64; Store tmp to bp+disp. */
1734 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1735 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1736}
1737
1738
1739#ifdef RT_ARCH_AMD64
1740/**
1741 * Common bit of iemNativeEmitLoadGprByGpr and friends.
1742 */
1743DECL_FORCE_INLINE(uint32_t)
1744iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
1745{
1746 if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
1747 {
1748 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
1749 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1750 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1751 }
1752 else if (offDisp == (int8_t)offDisp)
1753 {
1754 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1755 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1756 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1757 pbCodeBuf[off++] = (uint8_t)offDisp;
1758 }
1759 else
1760 {
1761 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
1762 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
1763 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
1764 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1765 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1766 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1767 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1768 }
1769 return off;
1770}
1771#elif RT_ARCH_ARM64
1772/**
1773 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
1774 */
1775DECL_FORCE_INLINE_THROW(uint32_t)
1776iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
1777 uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
1778{
1779 /*
1780 * There are a couple of ldr variants that takes an immediate offset, so
1781 * try use those if we can, otherwise we have to use the temporary register
1782 * help with the addressing.
1783 */
1784 if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
1785 {
1786 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
1787 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1788 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
1789 }
1790 else
1791 {
1792 /* The offset is too large, so we must load it into a register and use
1793 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
1794 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
1795 uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
1796
1797 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1798 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
1799
1800 iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
1801 }
1802 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1803 return off;
1804}
1805#endif
1806
1807
1808/**
1809 * Emits a 64-bit GPR load via a GPR base address with a displacement.
1810 */
1811DECL_INLINE_THROW(uint32_t)
1812iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1813{
1814#ifdef RT_ARCH_AMD64
1815 /* mov reg64, mem64 */
1816 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1817 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
1818 pbCodeBuf[off++] = 0x8b;
1819 off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1820 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1821
1822#elif RT_ARCH_ARM64
1823 off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
1824
1825#else
1826# error "port me"
1827#endif
1828 return off;
1829}
1830
1831
1832/**
1833 * Emits a 32-bit GPR load via a GPR base address with a displacement.
1834 * @note Bits 63 thru 32 in @a iGprDst will be cleared.
1835 */
1836DECL_INLINE_THROW(uint32_t)
1837iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
1838{
1839#ifdef RT_ARCH_AMD64
1840 /* mov reg32, mem32 */
1841 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1842 if (iGprDst >= 8 || iGprBase >= 8)
1843 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
1844 pbCodeBuf[off++] = 0x8b;
1845 off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
1846 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1847
1848#elif RT_ARCH_ARM64
1849 off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1850
1851#else
1852# error "port me"
1853#endif
1854 return off;
1855}
1856
1857
1858/*********************************************************************************************************************************
1859* Subtraction and Additions *
1860*********************************************************************************************************************************/
1861
1862
1863#ifdef RT_ARCH_AMD64
1864/**
1865 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1866 */
1867DECL_INLINE_THROW(uint32_t)
1868iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1869{
1870 /* sub gprdst, imm8/imm32 */
1871 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1872 if (iGprDst < 8)
1873 pbCodeBuf[off++] = X86_OP_REX_W;
1874 else
1875 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1876 if (iSubtrahend < 128 && iSubtrahend >= -128)
1877 {
1878 pbCodeBuf[off++] = 0x83;
1879 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1880 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1881 }
1882 else
1883 {
1884 pbCodeBuf[off++] = 0x81;
1885 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1886 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1887 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1888 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1889 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1890 }
1891 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1892 return off;
1893}
1894#endif
1895
1896
1897/**
1898 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1899 * @note The AMD64 version sets flags.
1900 */
1901DECL_INLINE_THROW(uint32_t)
1902iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1903{
1904#if defined(RT_ARCH_AMD64)
1905 /* add Gv,Ev */
1906 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1907 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1908 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1909 pbCodeBuf[off++] = 0x04;
1910 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1911
1912#elif defined(RT_ARCH_ARM64)
1913 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1914 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1915
1916#else
1917# error "Port me"
1918#endif
1919 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1920 return off;
1921}
1922
1923
1924/**
1925 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1926 */
1927DECL_INLINE_THROW(uint32_t)
1928iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1929{
1930#if defined(RT_ARCH_AMD64)
1931 /* add or inc */
1932 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1933 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1934 if (iImm8 != 1)
1935 {
1936 pbCodeBuf[off++] = 0x83;
1937 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1938 pbCodeBuf[off++] = (uint8_t)iImm8;
1939 }
1940 else
1941 {
1942 pbCodeBuf[off++] = 0xff;
1943 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1944 }
1945
1946#elif defined(RT_ARCH_ARM64)
1947 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1948 if (iImm8 >= 0)
1949 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1950 else
1951 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1952
1953#else
1954# error "Port me"
1955#endif
1956 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1957 return off;
1958}
1959
1960
1961/**
1962 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1963 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1964 */
1965DECL_INLINE_THROW(uint32_t)
1966iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1967{
1968#if defined(RT_ARCH_AMD64)
1969 /* add or inc */
1970 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1971 if (iGprDst >= 8)
1972 pbCodeBuf[off++] = X86_OP_REX_B;
1973 if (iImm8 != 1)
1974 {
1975 pbCodeBuf[off++] = 0x83;
1976 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1977 pbCodeBuf[off++] = (uint8_t)iImm8;
1978 }
1979 else
1980 {
1981 pbCodeBuf[off++] = 0xff;
1982 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1983 }
1984
1985#elif defined(RT_ARCH_ARM64)
1986 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1987 if (iImm8 >= 0)
1988 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1989 else
1990 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1991
1992#else
1993# error "Port me"
1994#endif
1995 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1996 return off;
1997}
1998
1999
2000/**
2001 * Emits a 64-bit GPR additions with a 64-bit signed addend.
2002 */
2003DECL_INLINE_THROW(uint32_t)
2004iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
2005{
2006#if defined(RT_ARCH_AMD64)
2007 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2008 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
2009
2010 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
2011 {
2012 /* add grp, imm32 */
2013 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2014 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
2015 pbCodeBuf[off++] = 0x81;
2016 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2017 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2018 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2019 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2020 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2021 }
2022 else
2023 {
2024 /* Best to use a temporary register to deal with this in the simplest way: */
2025 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2026
2027 /* add dst, tmpreg */
2028 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2029 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
2030 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2031 pbCodeBuf[off++] = 0x03;
2032 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
2033
2034 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2035 }
2036
2037#elif defined(RT_ARCH_ARM64)
2038 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2039 {
2040 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2041 if (iAddend >= 0)
2042 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
2043 else
2044 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
2045 }
2046 else
2047 {
2048 /* Use temporary register for the immediate. */
2049 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
2050
2051 /* add gprdst, gprdst, tmpreg */
2052 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2053 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
2054
2055 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2056 }
2057
2058#else
2059# error "Port me"
2060#endif
2061 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2062 return off;
2063}
2064
2065
2066/**
2067 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
2068 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
2069 */
2070DECL_INLINE_THROW(uint32_t)
2071iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
2072{
2073#if defined(RT_ARCH_AMD64)
2074 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
2075 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
2076
2077 /* add grp, imm32 */
2078 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2079 if (iGprDst >= 8)
2080 pbCodeBuf[off++] = X86_OP_REX_B;
2081 pbCodeBuf[off++] = 0x81;
2082 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
2083 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
2084 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
2085 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
2086 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
2087
2088#elif defined(RT_ARCH_ARM64)
2089 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
2090 {
2091 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2092 if (iAddend >= 0)
2093 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
2094 else
2095 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
2096 }
2097 else
2098 {
2099 /* Use temporary register for the immediate. */
2100 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
2101
2102 /* add gprdst, gprdst, tmpreg */
2103 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2104 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
2105
2106 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2107 }
2108
2109#else
2110# error "Port me"
2111#endif
2112 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2113 return off;
2114}
2115
2116
2117
2118/*********************************************************************************************************************************
2119* Bit Operations *
2120*********************************************************************************************************************************/
2121
2122/**
2123 * Emits code for clearing bits 16 thru 63 in the GPR.
2124 */
2125DECL_INLINE_THROW(uint32_t)
2126iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
2127{
2128#if defined(RT_ARCH_AMD64)
2129 /* movzx Gv,Ew */
2130 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2131 if (iGprDst >= 8)
2132 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
2133 pbCodeBuf[off++] = 0x0f;
2134 pbCodeBuf[off++] = 0xb7;
2135 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
2136
2137#elif defined(RT_ARCH_ARM64)
2138 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2139# if 1
2140 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
2141# else
2142 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
2143 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
2144# endif
2145#else
2146# error "Port me"
2147#endif
2148 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2149 return off;
2150}
2151
2152
2153/**
2154 * Emits code for AND'ing two 64-bit GPRs.
2155 *
2156 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2157 * and ARM64 hosts.
2158 */
2159DECL_INLINE_THROW(uint32_t)
2160iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2161{
2162#if defined(RT_ARCH_AMD64)
2163 /* and Gv, Ev */
2164 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2165 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2166 pbCodeBuf[off++] = 0x23;
2167 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2168 RT_NOREF(fSetFlags);
2169
2170#elif defined(RT_ARCH_ARM64)
2171 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2172 if (!fSetFlags)
2173 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
2174 else
2175 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
2176
2177#else
2178# error "Port me"
2179#endif
2180 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2181 return off;
2182}
2183
2184
2185/**
2186 * Emits code for AND'ing two 32-bit GPRs.
2187 */
2188DECL_INLINE_THROW(uint32_t)
2189iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
2190{
2191#if defined(RT_ARCH_AMD64)
2192 /* and Gv, Ev */
2193 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2194 if (iGprDst >= 8 || iGprSrc >= 8)
2195 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2196 pbCodeBuf[off++] = 0x23;
2197 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2198 RT_NOREF(fSetFlags);
2199
2200#elif defined(RT_ARCH_ARM64)
2201 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2202 if (fSetFlags)
2203 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
2204 else
2205 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
2206
2207#else
2208# error "Port me"
2209#endif
2210 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2211 return off;
2212}
2213
2214
2215/**
2216 * Emits code for AND'ing a 64-bit GPRs with a constant.
2217 *
2218 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
2219 * and ARM64 hosts.
2220 */
2221DECL_INLINE_THROW(uint32_t)
2222iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)
2223{
2224#if defined(RT_ARCH_AMD64)
2225 if ((int64_t)uImm == (int8_t)uImm)
2226 {
2227 /* and Ev, imm8 */
2228 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2229 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_B);
2230 pbCodeBuf[off++] = 0x83;
2231 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2232 pbCodeBuf[off++] = (uint8_t)uImm;
2233 }
2234 else if ((int64_t)uImm == (int32_t)uImm)
2235 {
2236 /* and Ev, imm32 */
2237 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2238 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_B);
2239 pbCodeBuf[off++] = 0x81;
2240 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2241 pbCodeBuf[off++] = RT_BYTE1(uImm);
2242 pbCodeBuf[off++] = RT_BYTE2(uImm);
2243 pbCodeBuf[off++] = RT_BYTE3(uImm);
2244 pbCodeBuf[off++] = RT_BYTE4(uImm);
2245 }
2246 else
2247 {
2248 /* Use temporary register for the 64-bit immediate. */
2249 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2250 off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
2251 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2252 }
2253 RT_NOREF(fSetFlags);
2254
2255#elif defined(RT_ARCH_ARM64)
2256 uint32_t uImmR = 0;
2257 uint32_t uImmNandS = 0;
2258 if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
2259 {
2260 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2261 if (!fSetFlags)
2262 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
2263 else
2264 pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
2265 }
2266 else
2267 {
2268 /* Use temporary register for the 64-bit immediate. */
2269 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2270 off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2271 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2272 }
2273
2274#else
2275# error "Port me"
2276#endif
2277 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2278 return off;
2279}
2280
2281
2282/**
2283 * Emits code for AND'ing an 32-bit GPRs with a constant.
2284 */
2285DECL_INLINE_THROW(uint32_t)
2286iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
2287{
2288#if defined(RT_ARCH_AMD64)
2289 /* and Ev, imm */
2290 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2291 if (iGprDst >= 8)
2292 pbCodeBuf[off++] = X86_OP_REX_B;
2293 if ((int32_t)uImm == (int8_t)uImm)
2294 {
2295 pbCodeBuf[off++] = 0x83;
2296 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2297 pbCodeBuf[off++] = (uint8_t)uImm;
2298 }
2299 else
2300 {
2301 pbCodeBuf[off++] = 0x81;
2302 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2303 pbCodeBuf[off++] = RT_BYTE1(uImm);
2304 pbCodeBuf[off++] = RT_BYTE2(uImm);
2305 pbCodeBuf[off++] = RT_BYTE3(uImm);
2306 pbCodeBuf[off++] = RT_BYTE4(uImm);
2307 }
2308 RT_NOREF(fSetFlags);
2309
2310#elif defined(RT_ARCH_ARM64)
2311 uint32_t uImmR = 0;
2312 uint32_t uImmNandS = 0;
2313 if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
2314 {
2315 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2316 if (!fSetFlags)
2317 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
2318 else
2319 pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
2320 }
2321 else
2322 {
2323 /* Use temporary register for the 64-bit immediate. */
2324 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2325 off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);
2326 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2327 }
2328
2329#else
2330# error "Port me"
2331#endif
2332 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2333 return off;
2334}
2335
2336
2337/**
2338 * Emits code for XOR'ing two 64-bit GPRs.
2339 */
2340DECL_INLINE_THROW(uint32_t)
2341iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2342{
2343#if defined(RT_ARCH_AMD64)
2344 /* and Gv, Ev */
2345 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2346 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2347 pbCodeBuf[off++] = 0x33;
2348 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2349
2350#elif defined(RT_ARCH_ARM64)
2351 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2352 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
2353
2354#else
2355# error "Port me"
2356#endif
2357 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2358 return off;
2359}
2360
2361
2362/**
2363 * Emits code for XOR'ing two 32-bit GPRs.
2364 */
2365DECL_INLINE_THROW(uint32_t)
2366iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
2367{
2368#if defined(RT_ARCH_AMD64)
2369 /* and Gv, Ev */
2370 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2371 if (iGprDst >= 8 || iGprSrc >= 8)
2372 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2373 pbCodeBuf[off++] = 0x33;
2374 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
2375
2376#elif defined(RT_ARCH_ARM64)
2377 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2378 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
2379
2380#else
2381# error "Port me"
2382#endif
2383 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2384 return off;
2385}
2386
2387
2388/*********************************************************************************************************************************
2389* Shifting *
2390*********************************************************************************************************************************/
2391
2392/**
2393 * Emits code for shifting a GPR a fixed number of bits to the left.
2394 */
2395DECL_INLINE_THROW(uint32_t)
2396iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2397{
2398 Assert(cShift > 0 && cShift < 64);
2399
2400#if defined(RT_ARCH_AMD64)
2401 /* shl dst, cShift */
2402 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2403 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
2404 if (cShift != 1)
2405 {
2406 pbCodeBuf[off++] = 0xc1;
2407 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2408 pbCodeBuf[off++] = cShift;
2409 }
2410 else
2411 {
2412 pbCodeBuf[off++] = 0xd1;
2413 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2414 }
2415
2416#elif defined(RT_ARCH_ARM64)
2417 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2418 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
2419
2420#else
2421# error "Port me"
2422#endif
2423 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2424 return off;
2425}
2426
2427
2428/**
2429 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
2430 */
2431DECL_INLINE_THROW(uint32_t)
2432iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2433{
2434 Assert(cShift > 0 && cShift < 32);
2435
2436#if defined(RT_ARCH_AMD64)
2437 /* shl dst, cShift */
2438 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2439 if (iGprDst >= 8)
2440 pbCodeBuf[off++] = X86_OP_REX_B;
2441 if (cShift != 1)
2442 {
2443 pbCodeBuf[off++] = 0xc1;
2444 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2445 pbCodeBuf[off++] = cShift;
2446 }
2447 else
2448 {
2449 pbCodeBuf[off++] = 0xd1;
2450 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
2451 }
2452
2453#elif defined(RT_ARCH_ARM64)
2454 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2455 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
2456
2457#else
2458# error "Port me"
2459#endif
2460 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2461 return off;
2462}
2463
2464
2465/**
2466 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
2467 */
2468DECL_INLINE_THROW(uint32_t)
2469iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2470{
2471 Assert(cShift > 0 && cShift < 64);
2472
2473#if defined(RT_ARCH_AMD64)
2474 /* shr dst, cShift */
2475 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2476 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
2477 if (cShift != 1)
2478 {
2479 pbCodeBuf[off++] = 0xc1;
2480 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2481 pbCodeBuf[off++] = cShift;
2482 }
2483 else
2484 {
2485 pbCodeBuf[off++] = 0xd1;
2486 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2487 }
2488
2489#elif defined(RT_ARCH_ARM64)
2490 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2491 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
2492
2493#else
2494# error "Port me"
2495#endif
2496 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2497 return off;
2498}
2499
2500
2501/**
2502 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
2503 * right.
2504 */
2505DECL_INLINE_THROW(uint32_t)
2506iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
2507{
2508 Assert(cShift > 0 && cShift < 32);
2509
2510#if defined(RT_ARCH_AMD64)
2511 /* shr dst, cShift */
2512 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2513 if (iGprDst >= 8)
2514 pbCodeBuf[off++] = X86_OP_REX_B;
2515 if (cShift != 1)
2516 {
2517 pbCodeBuf[off++] = 0xc1;
2518 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2519 pbCodeBuf[off++] = cShift;
2520 }
2521 else
2522 {
2523 pbCodeBuf[off++] = 0xd1;
2524 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
2525 }
2526
2527#elif defined(RT_ARCH_ARM64)
2528 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2529 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
2530
2531#else
2532# error "Port me"
2533#endif
2534 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2535 return off;
2536}
2537
2538
2539
2540/*********************************************************************************************************************************
2541* Compare and Testing *
2542*********************************************************************************************************************************/
2543
2544
2545#ifdef RT_ARCH_ARM64
2546/**
2547 * Emits an ARM64 compare instruction.
2548 */
2549DECL_INLINE_THROW(uint32_t)
2550iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
2551 bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
2552{
2553 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2554 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
2555 f64Bit, true /*fSetFlags*/, cShift, enmShift);
2556 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2557 return off;
2558}
2559#endif
2560
2561
2562/**
2563 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
2564 * with conditional instruction.
2565 */
2566DECL_INLINE_THROW(uint32_t)
2567iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2568{
2569#ifdef RT_ARCH_AMD64
2570 /* cmp Gv, Ev */
2571 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2572 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
2573 pbCodeBuf[off++] = 0x3b;
2574 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2575 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2576
2577#elif defined(RT_ARCH_ARM64)
2578 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
2579
2580#else
2581# error "Port me!"
2582#endif
2583 return off;
2584}
2585
2586
2587/**
2588 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
2589 * with conditional instruction.
2590 */
2591DECL_INLINE_THROW(uint32_t)
2592iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
2593{
2594#ifdef RT_ARCH_AMD64
2595 /* cmp Gv, Ev */
2596 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2597 if (iGprLeft >= 8 || iGprRight >= 8)
2598 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
2599 pbCodeBuf[off++] = 0x3b;
2600 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2601 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2602
2603#elif defined(RT_ARCH_ARM64)
2604 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
2605
2606#else
2607# error "Port me!"
2608#endif
2609 return off;
2610}
2611
2612
2613/**
2614 * Emits a compare of a 64-bit GPR with a constant value, settings status
2615 * flags/whatever for use with conditional instruction.
2616 */
2617DECL_INLINE_THROW(uint32_t)
2618iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
2619{
2620#ifdef RT_ARCH_AMD64
2621 if (uImm <= UINT32_C(0xff))
2622 {
2623 /* cmp Ev, Ib */
2624 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
2625 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2626 pbCodeBuf[off++] = 0x83;
2627 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2628 pbCodeBuf[off++] = (uint8_t)uImm;
2629 }
2630 else if ((int64_t)uImm == (int32_t)uImm)
2631 {
2632 /* cmp Ev, imm */
2633 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2634 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
2635 pbCodeBuf[off++] = 0x81;
2636 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2637 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2638 pbCodeBuf[off++] = RT_BYTE1(uImm);
2639 pbCodeBuf[off++] = RT_BYTE2(uImm);
2640 pbCodeBuf[off++] = RT_BYTE3(uImm);
2641 pbCodeBuf[off++] = RT_BYTE4(uImm);
2642 }
2643 else
2644 {
2645 /* Use temporary register for the immediate. */
2646 uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2647 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2648 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2649 }
2650
2651#elif defined(RT_ARCH_ARM64)
2652 /** @todo guess there are clevere things we can do here... */
2653 if (uImm < _4K)
2654 {
2655 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2656 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2657 true /*64Bit*/, true /*fSetFlags*/);
2658 }
2659 else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2660 {
2661 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2662 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2663 true /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);
2664 }
2665 else
2666 {
2667 /* Use temporary register for the immediate. */
2668 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2669 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);
2670 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2671 }
2672
2673#else
2674# error "Port me!"
2675#endif
2676
2677 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2678 return off;
2679}
2680
2681
2682/**
2683 * Emits a compare of a 32-bit GPR with a constant value, settings status
2684 * flags/whatever for use with conditional instruction.
2685 */
2686DECL_INLINE_THROW(uint32_t)
2687iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
2688{
2689#ifdef RT_ARCH_AMD64
2690 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2691 if (iGprLeft >= 8)
2692 pbCodeBuf[off++] = X86_OP_REX_B;
2693 if (uImm <= UINT32_C(0xff))
2694 {
2695 /* cmp Ev, Ib */
2696 pbCodeBuf[off++] = 0x83;
2697 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2698 pbCodeBuf[off++] = (uint8_t)uImm;
2699 }
2700 else
2701 {
2702 /* cmp Ev, imm */
2703 pbCodeBuf[off++] = 0x81;
2704 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
2705 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2706 pbCodeBuf[off++] = RT_BYTE1(uImm);
2707 pbCodeBuf[off++] = RT_BYTE2(uImm);
2708 pbCodeBuf[off++] = RT_BYTE3(uImm);
2709 pbCodeBuf[off++] = RT_BYTE4(uImm);
2710 }
2711
2712#elif defined(RT_ARCH_ARM64)
2713 /** @todo guess there are clevere things we can do here... */
2714 if (uImm < _4K)
2715 {
2716 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2717 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2718 false /*64Bit*/, true /*fSetFlags*/);
2719 }
2720 else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
2721 {
2722 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2723 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
2724 false /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);
2725 }
2726 else
2727 {
2728 /* Use temporary register for the immediate. */
2729 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
2730 off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);
2731 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2732 }
2733
2734#else
2735# error "Port me!"
2736#endif
2737
2738 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2739 return off;
2740}
2741
2742
2743
2744/*********************************************************************************************************************************
2745* Branching *
2746*********************************************************************************************************************************/
2747
2748/**
2749 * Emits a JMP rel32 / B imm19 to the given label.
2750 */
2751DECL_INLINE_THROW(uint32_t)
2752iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2753{
2754 Assert(idxLabel < pReNative->cLabels);
2755
2756#ifdef RT_ARCH_AMD64
2757 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2758 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2759 {
2760 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2761 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2762 {
2763 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2764 pbCodeBuf[off++] = (uint8_t)offRel;
2765 }
2766 else
2767 {
2768 offRel -= 3;
2769 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2770 pbCodeBuf[off++] = RT_BYTE1(offRel);
2771 pbCodeBuf[off++] = RT_BYTE2(offRel);
2772 pbCodeBuf[off++] = RT_BYTE3(offRel);
2773 pbCodeBuf[off++] = RT_BYTE4(offRel);
2774 }
2775 }
2776 else
2777 {
2778 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2779 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2780 pbCodeBuf[off++] = 0xfe;
2781 pbCodeBuf[off++] = 0xff;
2782 pbCodeBuf[off++] = 0xff;
2783 pbCodeBuf[off++] = 0xff;
2784 }
2785 pbCodeBuf[off++] = 0xcc; /* int3 poison */
2786
2787#elif defined(RT_ARCH_ARM64)
2788 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2789 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2790 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
2791 else
2792 {
2793 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
2794 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2795 }
2796
2797#else
2798# error "Port me!"
2799#endif
2800 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2801 return off;
2802}
2803
2804
2805/**
2806 * Emits a JMP rel32 / B imm19 to a new undefined label.
2807 */
2808DECL_INLINE_THROW(uint32_t)
2809iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2810{
2811 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2812 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2813}
2814
2815/** Condition type. */
2816#ifdef RT_ARCH_AMD64
2817typedef enum IEMNATIVEINSTRCOND : uint8_t
2818{
2819 kIemNativeInstrCond_o = 0,
2820 kIemNativeInstrCond_no,
2821 kIemNativeInstrCond_c,
2822 kIemNativeInstrCond_nc,
2823 kIemNativeInstrCond_e,
2824 kIemNativeInstrCond_ne,
2825 kIemNativeInstrCond_be,
2826 kIemNativeInstrCond_nbe,
2827 kIemNativeInstrCond_s,
2828 kIemNativeInstrCond_ns,
2829 kIemNativeInstrCond_p,
2830 kIemNativeInstrCond_np,
2831 kIemNativeInstrCond_l,
2832 kIemNativeInstrCond_nl,
2833 kIemNativeInstrCond_le,
2834 kIemNativeInstrCond_nle
2835} IEMNATIVEINSTRCOND;
2836#elif defined(RT_ARCH_ARM64)
2837typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2838#else
2839# error "Port me!"
2840#endif
2841
2842
2843/**
2844 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2845 */
2846DECL_INLINE_THROW(uint32_t)
2847iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2848{
2849 Assert(idxLabel < pReNative->cLabels);
2850
2851#ifdef RT_ARCH_AMD64
2852 /* jcc rel32 */
2853 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2854 pbCodeBuf[off++] = 0x0f;
2855 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2856 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
2857 pbCodeBuf[off++] = 0x00;
2858 pbCodeBuf[off++] = 0x00;
2859 pbCodeBuf[off++] = 0x00;
2860 pbCodeBuf[off++] = 0x00;
2861
2862#elif defined(RT_ARCH_ARM64)
2863 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2864 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
2865 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2866
2867#else
2868# error "Port me!"
2869#endif
2870 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2871 return off;
2872}
2873
2874
2875/**
2876 * Emits a Jcc rel32 / B.cc imm19 to a new label.
2877 */
2878DECL_INLINE_THROW(uint32_t)
2879iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2880 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2881{
2882 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2883 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2884}
2885
2886
2887/**
2888 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2889 */
2890DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2891{
2892#ifdef RT_ARCH_AMD64
2893 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2894#elif defined(RT_ARCH_ARM64)
2895 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2896#else
2897# error "Port me!"
2898#endif
2899}
2900
2901/**
2902 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2903 */
2904DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2905 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2906{
2907#ifdef RT_ARCH_AMD64
2908 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2909#elif defined(RT_ARCH_ARM64)
2910 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2911#else
2912# error "Port me!"
2913#endif
2914}
2915
2916
2917/**
2918 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2919 */
2920DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2921{
2922#ifdef RT_ARCH_AMD64
2923 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2924#elif defined(RT_ARCH_ARM64)
2925 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2926#else
2927# error "Port me!"
2928#endif
2929}
2930
2931/**
2932 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2933 */
2934DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2935 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2936{
2937#ifdef RT_ARCH_AMD64
2938 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2939#elif defined(RT_ARCH_ARM64)
2940 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2941#else
2942# error "Port me!"
2943#endif
2944}
2945
2946
2947/**
2948 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2949 */
2950DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2951{
2952#ifdef RT_ARCH_AMD64
2953 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2954#elif defined(RT_ARCH_ARM64)
2955 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2956#else
2957# error "Port me!"
2958#endif
2959}
2960
2961/**
2962 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2963 */
2964DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2965 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2966{
2967#ifdef RT_ARCH_AMD64
2968 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2969#elif defined(RT_ARCH_ARM64)
2970 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2971#else
2972# error "Port me!"
2973#endif
2974}
2975
2976
2977/**
2978 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2979 */
2980DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2981{
2982#ifdef RT_ARCH_AMD64
2983 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2984#elif defined(RT_ARCH_ARM64)
2985 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2986#else
2987# error "Port me!"
2988#endif
2989}
2990
2991/**
2992 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2993 */
2994DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2995 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2996{
2997#ifdef RT_ARCH_AMD64
2998 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2999#elif defined(RT_ARCH_ARM64)
3000 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
3001#else
3002# error "Port me!"
3003#endif
3004}
3005
3006
3007/**
3008 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
3009 * How @a offJmp is applied is are target specific.
3010 */
3011DECL_INLINE_THROW(uint32_t)
3012iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
3013{
3014#ifdef RT_ARCH_AMD64
3015 /* jcc rel32 */
3016 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
3017 if (offTarget < 128 && offTarget >= -128)
3018 {
3019 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
3020 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
3021 }
3022 else
3023 {
3024 pbCodeBuf[off++] = 0x0f;
3025 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
3026 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
3027 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
3028 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
3029 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
3030 }
3031
3032#elif defined(RT_ARCH_ARM64)
3033 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3034 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
3035
3036#else
3037# error "Port me!"
3038#endif
3039 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3040 return off;
3041}
3042
3043
3044/**
3045 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
3046 * How @a offJmp is applied is are target specific.
3047 */
3048DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3049{
3050#ifdef RT_ARCH_AMD64
3051 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
3052#elif defined(RT_ARCH_ARM64)
3053 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
3054#else
3055# error "Port me!"
3056#endif
3057}
3058
3059
3060/**
3061 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
3062 * How @a offJmp is applied is are target specific.
3063 */
3064DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3065{
3066#ifdef RT_ARCH_AMD64
3067 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
3068#elif defined(RT_ARCH_ARM64)
3069 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
3070#else
3071# error "Port me!"
3072#endif
3073}
3074
3075
3076/**
3077 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
3078 * How @a offJmp is applied is are target specific.
3079 */
3080DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3081{
3082#ifdef RT_ARCH_AMD64
3083 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
3084#elif defined(RT_ARCH_ARM64)
3085 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
3086#else
3087# error "Port me!"
3088#endif
3089}
3090
3091
3092/**
3093 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
3094 * How @a offJmp is applied is are target specific.
3095 */
3096DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
3097{
3098#ifdef RT_ARCH_AMD64
3099 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
3100#elif defined(RT_ARCH_ARM64)
3101 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
3102#else
3103# error "Port me!"
3104#endif
3105}
3106
3107
3108/**
3109 * Fixes up a conditional jump to a fixed label.
3110 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
3111 */
3112DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
3113{
3114# if defined(RT_ARCH_AMD64)
3115 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
3116 if (pbCodeBuf[offFixup] != 0x0f)
3117 {
3118 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
3119 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
3120 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
3121 }
3122 else
3123 {
3124 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
3125 uint32_t const offRel32 = offTarget - (offFixup + 6);
3126 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
3127 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
3128 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
3129 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
3130 }
3131
3132# elif defined(RT_ARCH_ARM64)
3133 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
3134
3135 int32_t const offDisp = offTarget - offFixup;
3136 Assert(offDisp >= -262144 && offDisp < 262144);
3137 Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */
3138
3139 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
3140 | (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);
3141
3142# endif
3143}
3144
3145
3146/**
3147 * Internal helper, don't call directly.
3148 */
3149DECL_INLINE_THROW(uint32_t)
3150iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
3151 uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
3152{
3153 Assert(iBitNo < 64);
3154#ifdef RT_ARCH_AMD64
3155 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3156 if (iBitNo < 8)
3157 {
3158 /* test Eb, imm8 */
3159 if (iGprSrc >= 4)
3160 pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3161 pbCodeBuf[off++] = 0xf6;
3162 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3163 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
3164 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
3165 }
3166 else
3167 {
3168 /* bt Ev, imm8 */
3169 if (iBitNo >= 32)
3170 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
3171 else if (iGprSrc >= 8)
3172 pbCodeBuf[off++] = X86_OP_REX_B;
3173 pbCodeBuf[off++] = 0x0f;
3174 pbCodeBuf[off++] = 0xba;
3175 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
3176 pbCodeBuf[off++] = iBitNo;
3177 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
3178 }
3179
3180#elif defined(RT_ARCH_ARM64)
3181 /* Use the TBNZ instruction here. */
3182 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3183 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
3184 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
3185
3186#else
3187# error "Port me!"
3188#endif
3189 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3190 return off;
3191}
3192
3193
3194/**
3195 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
3196 * @a iGprSrc.
3197 *
3198 * @note On ARM64 the range is only +/-8191 instructions.
3199 */
3200DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3201 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3202{
3203 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
3204}
3205
3206
3207/**
3208 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
3209 * _set_ in @a iGprSrc.
3210 *
3211 * @note On ARM64 the range is only +/-8191 instructions.
3212 */
3213DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3214 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
3215{
3216 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
3217}
3218
3219
3220/**
3221 * Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
3222 * flags accordingly.
3223 */
3224DECL_INLINE_THROW(uint32_t)
3225iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
3226{
3227 Assert(fBits != 0);
3228#ifdef RT_ARCH_AMD64
3229
3230 if (fBits >= UINT32_MAX)
3231 {
3232 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3233
3234 /* test Ev,Gv */
3235 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
3236 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R) | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
3237 pbCodeBuf[off++] = 0x85;
3238 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
3239
3240 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3241 }
3242 else if (fBits <= UINT32_MAX)
3243 {
3244 /* test Eb, imm8 or test Ev, imm32 */
3245 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
3246 if (fBits <= UINT8_MAX)
3247 {
3248 if (iGprSrc >= 4)
3249 pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
3250 pbCodeBuf[off++] = 0xf6;
3251 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3252 pbCodeBuf[off++] = (uint8_t)fBits;
3253 }
3254 else
3255 {
3256 if (iGprSrc >= 8)
3257 pbCodeBuf[off++] = X86_OP_REX_B;
3258 pbCodeBuf[off++] = 0xf7;
3259 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
3260 pbCodeBuf[off++] = RT_BYTE1(fBits);
3261 pbCodeBuf[off++] = RT_BYTE2(fBits);
3262 pbCodeBuf[off++] = RT_BYTE3(fBits);
3263 pbCodeBuf[off++] = RT_BYTE4(fBits);
3264 }
3265 }
3266 /** @todo implement me. */
3267 else
3268 AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));
3269
3270#elif defined(RT_ARCH_ARM64)
3271
3272 if (false)
3273 {
3274 /** @todo figure out how to work the immr / N:imms constants. */
3275 }
3276 else
3277 {
3278 /* ands Zr, iGprSrc, iTmpReg */
3279 uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
3280 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3281 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
3282 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
3283 }
3284
3285#else
3286# error "Port me!"
3287#endif
3288 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3289 return off;
3290}
3291
3292
3293/**
3294 * Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
3295 * are set in @a iGprSrc.
3296 */
3297DECL_INLINE_THROW(uint32_t)
3298iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3299 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3300{
3301 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3302
3303 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3304 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3305
3306 return off;
3307}
3308
3309
3310/**
3311 * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
3312 * are set in @a iGprSrc.
3313 */
3314DECL_INLINE_THROW(uint32_t)
3315iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3316 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
3317{
3318 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
3319
3320 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
3321 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3322
3323 return off;
3324}
3325
3326
3327/**
3328 * Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
3329 *
3330 * The operand size is given by @a f64Bit.
3331 */
3332DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3333 uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
3334{
3335 Assert(idxLabel < pReNative->cLabels);
3336
3337#ifdef RT_ARCH_AMD64
3338 /* test reg32,reg32 / test reg64,reg64 */
3339 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
3340 if (f64Bit)
3341 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
3342 else if (iGprSrc >= 8)
3343 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
3344 pbCodeBuf[off++] = 0x85;
3345 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
3346 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3347
3348 /* jz idxLabel */
3349 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
3350
3351#elif defined(RT_ARCH_ARM64)
3352 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3353 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
3354 pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 0, iGprSrc, f64Bit);
3355 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3356
3357#else
3358# error "Port me!"
3359#endif
3360 return off;
3361}
3362
3363
3364/**
3365 * Emits code that jumps to a new label if @a iGprSrc is zero.
3366 *
3367 * The operand size is given by @a f64Bit.
3368 */
3369DECL_INLINE_THROW(uint32_t)
3370iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,
3371 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3372{
3373 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
3374 return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
3375}
3376
3377
3378/**
3379 * Emits code that jumps to the given label if @a iGprLeft and @a iGprRight
3380 * differs.
3381 */
3382DECL_INLINE_THROW(uint32_t)
3383iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3384 uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)
3385{
3386 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
3387 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3388 return off;
3389}
3390
3391
3392/**
3393 * Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.
3394 */
3395DECL_INLINE_THROW(uint32_t)
3396iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3397 uint8_t iGprLeft, uint8_t iGprRight,
3398 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3399{
3400 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
3401 return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);
3402}
3403
3404
3405/**
3406 * Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.
3407 */
3408DECL_INLINE_THROW(uint32_t)
3409iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3410 uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)
3411{
3412 off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
3413 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3414 return off;
3415}
3416
3417
3418/**
3419 * Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.
3420 */
3421DECL_INLINE_THROW(uint32_t)
3422iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3423 uint8_t iGprSrc, uint64_t uImm,
3424 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3425{
3426 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
3427 return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3428}
3429
3430
3431/**
3432 * Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
3433 * @a uImm.
3434 */
3435DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3436 uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)
3437{
3438 off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
3439 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
3440 return off;
3441}
3442
3443
3444/**
3445 * Emits code that jumps to a new label if 32-bit @a iGprSrc differs from
3446 * @a uImm.
3447 */
3448DECL_INLINE_THROW(uint32_t)
3449iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
3450 uint8_t iGprSrc, uint32_t uImm,
3451 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
3452{
3453 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
3454 return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);
3455}
3456
3457
3458
3459/**
3460 * Emits a call to a 64-bit address.
3461 */
3462DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
3463{
3464#ifdef RT_ARCH_AMD64
3465 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
3466
3467 /* call rax */
3468 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
3469 pbCodeBuf[off++] = 0xff;
3470 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
3471
3472#elif defined(RT_ARCH_ARM64)
3473 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
3474
3475 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
3476 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
3477
3478#else
3479# error "port me"
3480#endif
3481 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
3482 return off;
3483}
3484
3485
3486/** @} */
3487
3488#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3489
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette