VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101576

Last change on this file since 101576 was 101576, checked in by vboxsync, 18 months ago

VMM/IEM: Native IEM_MC_IF_EFL_BITS_NE translation. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 95.3 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101576 2023-10-24 09:40:58Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46
47/** @name Stack Frame Layout
48 *
49 * @{ */
50/** The size of the area for stack variables and spills and stuff.
51 * @note This limit is duplicated in the python script(s). */
52#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
53#ifdef RT_ARCH_AMD64
54/** Number of stack arguments slots for calls made from the frame. */
55# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
56/** An stack alignment adjustment (between non-volatile register pushes and
57 * the stack variable area, so the latter better aligned). */
58# define IEMNATIVE_FRAME_ALIGN_SIZE 8
59/** Number of any shadow arguments (spill area) for calls we make. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
62# else
63# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
64# endif
65
66/** Frame pointer (RBP) relative offset of the last push. */
67# ifdef RT_OS_WINDOWS
68# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
69# else
70# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
71# endif
72/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
73 * address for it). */
74# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
75/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
76# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
77/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
78# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
79/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
80# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
81/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
82# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
83
84# ifdef RT_OS_WINDOWS
85/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
86# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
87/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
88# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
89/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
90# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
91/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
92# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
93# endif
94
95#elif RT_ARCH_ARM64
96/** No stack argument slots, enough got 8 registers for arguments. */
97# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
98/** There are no argument spill area. */
99# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100
101/** Number of saved registers at the top of our stack frame.
102 * This includes the return address and old frame pointer, so x19 thru x30. */
103# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
104/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
105# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
106
107/** Frame pointer (BP) relative offset of the last push. */
108# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
109
110/** Frame pointer (BP) relative offset of the stack variable area (the lowest
111 * address for it). */
112# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
113
114#else
115# error "port me"
116#endif
117/** @} */
118
119
120/** @name Fixed Register Allocation(s)
121 * @{ */
122/** @def IEMNATIVE_REG_FIXED_PVMCPU
123 * The number of the register holding the pVCpu pointer. */
124/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
125 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
126 * @note This not available on AMD64, only ARM64. */
127/** @def IEMNATIVE_REG_FIXED_TMP0
128 * Dedicated temporary register.
129 * @todo replace this by a register allocator and content tracker. */
130/** @def IEMNATIVE_REG_FIXED_MASK
131 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
132 * architecture. */
133#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
134# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
135# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
136# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
137 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
138 | RT_BIT_32(X86_GREG_xSP) \
139 | RT_BIT_32(X86_GREG_xBP) )
140
141#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
142# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
143# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
144# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
145# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
146 | RT_BIT_32(ARMV8_A64_REG_LR) \
147 | RT_BIT_32(ARMV8_A64_REG_BP) \
148 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
149 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
150 | RT_BIT_32(ARMV8_A64_REG_X18) \
151 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
152
153#else
154# error "port me"
155#endif
156/** @} */
157
158/** @name Call related registers.
159 * @{ */
160/** @def IEMNATIVE_CALL_RET_GREG
161 * The return value register. */
162/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
163 * Number of arguments in registers. */
164/** @def IEMNATIVE_CALL_ARG0_GREG
165 * The general purpose register carrying argument \#0. */
166/** @def IEMNATIVE_CALL_ARG1_GREG
167 * The general purpose register carrying argument \#1. */
168/** @def IEMNATIVE_CALL_ARG2_GREG
169 * The general purpose register carrying argument \#2. */
170/** @def IEMNATIVE_CALL_ARG3_GREG
171 * The general purpose register carrying argument \#3. */
172/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
173 * Mask of registers the callee will not save and may trash. */
174#ifdef RT_ARCH_AMD64
175# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
176
177# ifdef RT_OS_WINDOWS
178# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
179# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
180# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
181# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
182# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
183# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
184 | RT_BIT_32(X86_GREG_xCX) \
185 | RT_BIT_32(X86_GREG_xDX) \
186 | RT_BIT_32(X86_GREG_x8) \
187 | RT_BIT_32(X86_GREG_x9) \
188 | RT_BIT_32(X86_GREG_x10) \
189 | RT_BIT_32(X86_GREG_x11) )
190# else
191# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
192# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
193# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
194# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
195# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
196# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
197# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
198# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
199 | RT_BIT_32(X86_GREG_xCX) \
200 | RT_BIT_32(X86_GREG_xDX) \
201 | RT_BIT_32(X86_GREG_xDI) \
202 | RT_BIT_32(X86_GREG_xSI) \
203 | RT_BIT_32(X86_GREG_x8) \
204 | RT_BIT_32(X86_GREG_x9) \
205 | RT_BIT_32(X86_GREG_x10) \
206 | RT_BIT_32(X86_GREG_x11) )
207# endif
208
209#elif defined(RT_ARCH_ARM64)
210# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
211# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
212# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
213# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
214# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
215# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
216# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
217# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
218# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
219# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
220# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
221 | RT_BIT_32(ARMV8_A64_REG_X1) \
222 | RT_BIT_32(ARMV8_A64_REG_X2) \
223 | RT_BIT_32(ARMV8_A64_REG_X3) \
224 | RT_BIT_32(ARMV8_A64_REG_X4) \
225 | RT_BIT_32(ARMV8_A64_REG_X5) \
226 | RT_BIT_32(ARMV8_A64_REG_X6) \
227 | RT_BIT_32(ARMV8_A64_REG_X7) \
228 | RT_BIT_32(ARMV8_A64_REG_X8) \
229 | RT_BIT_32(ARMV8_A64_REG_X9) \
230 | RT_BIT_32(ARMV8_A64_REG_X10) \
231 | RT_BIT_32(ARMV8_A64_REG_X11) \
232 | RT_BIT_32(ARMV8_A64_REG_X12) \
233 | RT_BIT_32(ARMV8_A64_REG_X13) \
234 | RT_BIT_32(ARMV8_A64_REG_X14) \
235 | RT_BIT_32(ARMV8_A64_REG_X15) \
236 | RT_BIT_32(ARMV8_A64_REG_X16) \
237 | RT_BIT_32(ARMV8_A64_REG_X17) )
238
239#endif
240
241/** @} */
242
243
244/** @def IEMNATIVE_HST_GREG_COUNT
245 * Number of host general purpose registers we tracker. */
246/** @def IEMNATIVE_HST_GREG_MASK
247 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
248 * inverted register masks and such to get down to a correct set of regs. */
249#ifdef RT_ARCH_AMD64
250# define IEMNATIVE_HST_GREG_COUNT 16
251# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
252
253#elif defined(RT_ARCH_ARM64)
254# define IEMNATIVE_HST_GREG_COUNT 32
255# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
256#else
257# error "Port me!"
258#endif
259
260
261/** Native code generator label types. */
262typedef enum
263{
264 kIemNativeLabelType_Invalid = 0,
265 kIemNativeLabelType_Return,
266#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
267 kIemNativeLabelType_If,
268#endif
269 kIemNativeLabelType_Else,
270 kIemNativeLabelType_Endif,
271 kIemNativeLabelType_NonZeroRetOrPassUp,
272 kIemNativeLabelType_RaiseGp0,
273 kIemNativeLabelType_End
274} IEMNATIVELABELTYPE;
275
276/** Native code generator label definition. */
277typedef struct IEMNATIVELABEL
278{
279 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
280 * the epilog. */
281 uint32_t off;
282 /** The type of label (IEMNATIVELABELTYPE). */
283 uint16_t enmType;
284 /** Additional label data, type specific. */
285 uint16_t uData;
286} IEMNATIVELABEL;
287/** Pointer to a label. */
288typedef IEMNATIVELABEL *PIEMNATIVELABEL;
289
290
291/** Native code generator fixup types. */
292typedef enum
293{
294 kIemNativeFixupType_Invalid = 0,
295#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
296 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
297 kIemNativeFixupType_Rel32,
298#elif defined(RT_ARCH_ARM64)
299 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
300 kIemNativeFixupType_RelImm19At5,
301 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
302 kIemNativeFixupType_RelImm14At5,
303#endif
304 kIemNativeFixupType_End
305} IEMNATIVEFIXUPTYPE;
306
307/** Native code generator fixup. */
308typedef struct IEMNATIVEFIXUP
309{
310 /** Code offset of the fixup location. */
311 uint32_t off;
312 /** The IEMNATIVELABEL this is a fixup for. */
313 uint16_t idxLabel;
314 /** The fixup type (IEMNATIVEFIXUPTYPE). */
315 uint8_t enmType;
316 /** Addend or other data. */
317 int8_t offAddend;
318} IEMNATIVEFIXUP;
319/** Pointer to a native code generator fixup. */
320typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
321
322
323/**
324 * Guest registers that can be shadowed in GPRs.
325 */
326typedef enum IEMNATIVEGSTREG : uint8_t
327{
328 kIemNativeGstReg_GprFirst = 0,
329 kIemNativeGstReg_GprLast = 15,
330 kIemNativeGstReg_Pc,
331 kIemNativeGstReg_EFlags, /**< This one is problematic since the higher bits are used internally. */
332 /* gap: 18..23 */
333 kIemNativeGstReg_SegSelFirst = 24,
334 kIemNativeGstReg_SegSelLast = 29,
335 kIemNativeGstReg_SegBaseFirst = 30,
336 kIemNativeGstReg_SegBaseLast = 35,
337 kIemNativeGstReg_SegLimitFirst = 36,
338 kIemNativeGstReg_SegLimitLast = 41,
339 kIemNativeGstReg_End
340} IEMNATIVEGSTREG;
341
342/**
343 * Guest registers (classes) that can be referenced.
344 */
345typedef enum IEMNATIVEGSTREGREF : uint8_t
346{
347 kIemNativeGstRegRef_Invalid = 0,
348 kIemNativeGstRegRef_Gpr,
349 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
350 kIemNativeGstRegRef_EFlags,
351 kIemNativeGstRegRef_MxCsr,
352 kIemNativeGstRegRef_FpuReg,
353 kIemNativeGstRegRef_MReg,
354 kIemNativeGstRegRef_XReg,
355 kIemNativeGstRegRef_YReg,
356 kIemNativeGstRegRef_End
357} IEMNATIVEGSTREGREF;
358
359
360/** Variable kinds. */
361typedef enum IEMNATIVEVARKIND : uint8_t
362{
363 /** Customary invalid zero value. */
364 kIemNativeVarKind_Invalid = 0,
365 /** This is either in a register or on the stack. */
366 kIemNativeVarKind_Stack,
367 /** Immediate value - loaded into register when needed, or can live on the
368 * stack if referenced (in theory). */
369 kIemNativeVarKind_Immediate,
370 /** Variable reference - loaded into register when needed, never stack. */
371 kIemNativeVarKind_VarRef,
372 /** Guest register reference - loaded into register when needed, never stack. */
373 kIemNativeVarKind_GstRegRef,
374 /** End of valid values. */
375 kIemNativeVarKind_End
376} IEMNATIVEVARKIND;
377
378
379/** Variable or argument. */
380typedef struct IEMNATIVEVAR
381{
382 /** The kind of variable. */
383 IEMNATIVEVARKIND enmKind;
384 /** The variable size in bytes. */
385 uint8_t cbVar;
386 /** The first stack slot (uint64_t), except for immediate and references
387 * where it usually is UINT8_MAX. */
388 uint8_t idxStackSlot;
389 /** The host register allocated for the variable, UINT8_MAX if not. */
390 uint8_t idxReg;
391 /** The argument number if argument, UINT8_MAX if regular variable. */
392 uint8_t uArgNo;
393 /** If referenced, the index of the variable referencing this one, otherwise
394 * UINT8_MAX. A referenced variable must only be placed on the stack and
395 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
396 uint8_t idxReferrerVar;
397 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
398 IEMNATIVEGSTREG enmGstReg;
399 uint8_t bAlign;
400
401 union
402 {
403 /** kIemNativeVarKind_Immediate: The immediate value. */
404 uint64_t uValue;
405 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
406 uint8_t idxRefVar;
407 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
408 struct
409 {
410 /** The class of register. */
411 IEMNATIVEGSTREGREF enmClass;
412 /** Index within the class. */
413 uint8_t idx;
414 } GstRegRef;
415 } u;
416} IEMNATIVEVAR;
417
418/** What is being kept in a host register. */
419typedef enum IEMNATIVEWHAT : uint8_t
420{
421 /** The traditional invalid zero value. */
422 kIemNativeWhat_Invalid = 0,
423 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
424 kIemNativeWhat_Var,
425 /** Temporary register, this is typically freed when a MC completes. */
426 kIemNativeWhat_Tmp,
427 /** Call argument w/o a variable mapping. This is free (via
428 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
429 kIemNativeWhat_Arg,
430 /** Return status code.
431 * @todo not sure if we need this... */
432 kIemNativeWhat_rc,
433 /** The fixed pVCpu (PVMCPUCC) register.
434 * @todo consider offsetting this on amd64 to use negative offsets to access
435 * more members using 8-byte disp. */
436 kIemNativeWhat_pVCpuFixed,
437 /** The fixed pCtx (PCPUMCTX) register.
438 * @todo consider offsetting this on amd64 to use negative offsets to access
439 * more members using 8-byte disp. */
440 kIemNativeWhat_pCtxFixed,
441 /** Fixed temporary register. */
442 kIemNativeWhat_FixedTmp,
443 /** Register reserved by the CPU or OS architecture. */
444 kIemNativeWhat_FixedReserved,
445 /** End of valid values. */
446 kIemNativeWhat_End
447} IEMNATIVEWHAT;
448
449/**
450 * Host general register entry.
451 *
452 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
453 *
454 * @todo Track immediate values in host registers similarlly to how we track the
455 * guest register shadow copies. For it to be real helpful, though,
456 * we probably need to know which will be reused and put them into
457 * non-volatile registers, otherwise it's going to be more or less
458 * restricted to an instruction or two.
459 */
460typedef struct IEMNATIVEHSTREG
461{
462 /** Set of guest registers this one shadows.
463 *
464 * Using a bitmap here so we can designate the same host register as a copy
465 * for more than one guest register. This is expected to be useful in
466 * situations where one value is copied to several registers in a sequence.
467 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
468 * sequence we'd want to let this register follow to be a copy of and there
469 * will always be places where we'd be picking the wrong one.
470 */
471 uint64_t fGstRegShadows;
472 /** What is being kept in this register. */
473 IEMNATIVEWHAT enmWhat;
474 /** Variable index if holding a variable, otherwise UINT8_MAX. */
475 uint8_t idxVar;
476 /** Alignment padding. */
477 uint8_t abAlign[6];
478} IEMNATIVEHSTREG;
479
480
481/**
482 * Core state for the native recompiler, that is, things that needs careful
483 * handling when dealing with branches.
484 */
485typedef struct IEMNATIVECORESTATE
486{
487 /** Allocation bitmap for aHstRegs. */
488 uint32_t bmHstRegs;
489
490 /** Bitmap marking which host register contains guest register shadow copies.
491 * This is used during register allocation to try preserve copies. */
492 uint32_t bmHstRegsWithGstShadow;
493 /** Bitmap marking valid entries in aidxGstRegShadows. */
494 uint64_t bmGstRegShadows;
495
496 union
497 {
498 /** Index of variable arguments, UINT8_MAX if not valid. */
499 uint8_t aidxArgVars[8];
500 /** For more efficient resetting. */
501 uint64_t u64ArgVars;
502 };
503
504 /** Allocation bitmap for aVars. */
505 uint32_t bmVars;
506
507 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
508 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
509 * (A shadow copy of a guest register can only be held in a one host register,
510 * there are no duplicate copies or ambiguities like that). */
511 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
512
513 /** Host register allocation tracking. */
514 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
515
516 /** Variables and arguments. */
517 IEMNATIVEVAR aVars[9];
518} IEMNATIVECORESTATE;
519/** Pointer to core state. */
520typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
521/** Pointer to const core state. */
522typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
523
524
525/**
526 * Conditional stack entry.
527 */
528typedef struct IEMNATIVECOND
529{
530 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
531 bool fInElse;
532 /** The label for the IEM_MC_ELSE. */
533 uint32_t idxLabelElse;
534 /** The label for the IEM_MC_ENDIF. */
535 uint32_t idxLabelEndIf;
536 /** The initial state snapshot as the if-block starts executing. */
537 IEMNATIVECORESTATE InitialState;
538 /** The state snapshot at the end of the if-block. */
539 IEMNATIVECORESTATE IfFinalState;
540} IEMNATIVECOND;
541/** Pointer to a condition stack entry. */
542typedef IEMNATIVECOND *PIEMNATIVECOND;
543
544
545/**
546 * Native recompiler state.
547 */
548typedef struct IEMRECOMPILERSTATE
549{
550 /** Size of the buffer that pbNativeRecompileBufR3 points to in
551 * IEMNATIVEINSTR units. */
552 uint32_t cInstrBufAlloc;
553#ifdef VBOX_STRICT
554 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
555 uint32_t offInstrBufChecked;
556#else
557 uint32_t uPadding1; /* We don't keep track of the size here... */
558#endif
559 /** Fixed temporary code buffer for native recompilation. */
560 PIEMNATIVEINSTR pInstrBuf;
561
562 /** Bitmaps with the label types used. */
563 uint64_t bmLabelTypes;
564 /** Actual number of labels in paLabels. */
565 uint32_t cLabels;
566 /** Max number of entries allowed in paLabels before reallocating it. */
567 uint32_t cLabelsAlloc;
568 /** Labels defined while recompiling (referenced by fixups). */
569 PIEMNATIVELABEL paLabels;
570
571 /** Actual number of fixups paFixups. */
572 uint32_t cFixups;
573 /** Max number of entries allowed in paFixups before reallocating it. */
574 uint32_t cFixupsAlloc;
575 /** Buffer used by the recompiler for recording fixups when generating code. */
576 PIEMNATIVEFIXUP paFixups;
577
578#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
579 /** Number of debug info entries allocated for pDbgInfo. */
580 uint32_t cDbgInfoAlloc;
581 uint32_t uPadding;
582 /** Debug info. */
583 PIEMTBDBG pDbgInfo;
584#endif
585
586 /** The translation block being recompiled. */
587 PCIEMTB pTbOrg;
588
589 /** The current condition stack depth (aCondStack). */
590 uint8_t cCondDepth;
591 uint8_t bPadding2;
592 /** Condition sequence number (for generating unique labels). */
593 uint16_t uCondSeqNo;
594 uint32_t uPadding3;
595
596 /** Core state requiring care with branches. */
597 IEMNATIVECORESTATE Core;
598
599 /** The condition nesting stack. */
600 IEMNATIVECOND aCondStack[2];
601} IEMRECOMPILERSTATE;
602/** Pointer to a native recompiler state. */
603typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
604
605
606/**
607 * Native recompiler worker for a threaded function.
608 *
609 * @returns New code buffer offset, UINT32_MAX in case of failure.
610 * @param pReNative The native recompiler state.
611 * @param off The current code buffer offset.
612 * @param pCallEntry The threaded call entry.
613 *
614 * @note This is not allowed to throw anything atm.
615 */
616typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
617 PCIEMTHRDEDCALLENTRY pCallEntry));
618/** Pointer to a native recompiler worker for a threaded function. */
619typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
620
621/** Defines a native recompiler worker for a threaded function. */
622#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
623 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
624/** Prototypes a native recompiler function for a threaded function. */
625#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
626
627
628DECLHIDDEN(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
629 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
630DECLHIDDEN(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere) RT_NOEXCEPT;
631DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
632 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
633DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
634 uint32_t cInstrReq) RT_NOEXCEPT;
635
636DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
637 bool fPreferVolatile = true) RT_NOEXCEPT;
638DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
639 bool fPreferVolatile = true) RT_NOEXCEPT;
640DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
641 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
642DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
643DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
644DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
645DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
646DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
647DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
648DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
649DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
650
651DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
652 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
653DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
654 uint8_t idxInstr) RT_NOEXCEPT;
655
656
657/**
658 * Ensures that there is sufficient space in the instruction output buffer.
659 *
660 * This will reallocate the buffer if needed and allowed.
661 *
662 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
663 * allocation size.
664 *
665 * @returns Pointer to the instruction output buffer on success, NULL on
666 * failure.
667 * @param pReNative The native recompile state.
668 * @param off Current instruction offset. Works safely for UINT32_MAX
669 * as well.
670 * @param cInstrReq Number of instruction about to be added. It's okay to
671 * overestimate this a bit.
672 */
673DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
674{
675 uint64_t const offChecked = off + (uint64_t)cInstrReq;
676 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
677 {
678#ifdef VBOX_STRICT
679 pReNative->offInstrBufChecked = offChecked;
680#endif
681 return pReNative->pInstrBuf;
682 }
683 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
684}
685
686/**
687 * Checks that we didn't exceed the space requested in the last
688 * iemNativeInstrBufEnsure() call. */
689#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
690 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
691 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
692
693
694/**
695 * Emit a simple marker instruction to more easily tell where something starts
696 * in the disassembly.
697 */
698DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
699{
700#ifdef RT_ARCH_AMD64
701 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
702 AssertReturn(pbCodeBuf, UINT32_MAX);
703 if (uInfo == 0)
704 {
705 /* nop */
706 pbCodeBuf[off++] = 0x90;
707 }
708 else
709 {
710 /* nop [disp32] */
711 pbCodeBuf[off++] = 0x0f;
712 pbCodeBuf[off++] = 0x1f;
713 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
714 pbCodeBuf[off++] = RT_BYTE1(uInfo);
715 pbCodeBuf[off++] = RT_BYTE2(uInfo);
716 pbCodeBuf[off++] = RT_BYTE3(uInfo);
717 pbCodeBuf[off++] = RT_BYTE4(uInfo);
718 }
719#elif RT_ARCH_ARM64
720 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
721 AssertReturn(pu32CodeBuf, UINT32_MAX);
722 /* nop */
723 pu32CodeBuf[off++] = 0xd503201f;
724
725 RT_NOREF(uInfo);
726#else
727# error "port me"
728#endif
729 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
730 return off;
731}
732
733
734/*********************************************************************************************************************************
735* Loads, Stores and Related Stuff. *
736*********************************************************************************************************************************/
737
738/**
739 * Emits setting a GPR to zero.
740 */
741DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
742{
743#ifdef RT_ARCH_AMD64
744 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
745 AssertReturn(pbCodeBuf, UINT32_MAX);
746 /* xor gpr32, gpr32 */
747 if (iGpr >= 8)
748 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
749 pbCodeBuf[off++] = 0x33;
750 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
751
752#elif RT_ARCH_ARM64
753 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
754 AssertReturn(pu32CodeBuf, UINT32_MAX);
755 /* mov gpr, #0x0 */
756 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
757
758#else
759# error "port me"
760#endif
761 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
762 return off;
763}
764
765
766/**
767 * Emits loading a constant into a 64-bit GPR
768 */
769DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
770{
771 if (!uImm64)
772 return iemNativeEmitGprZero(pReNative, off, iGpr);
773
774#ifdef RT_ARCH_AMD64
775 if (uImm64 <= UINT32_MAX)
776 {
777 /* mov gpr, imm32 */
778 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
779 AssertReturn(pbCodeBuf, UINT32_MAX);
780 if (iGpr >= 8)
781 pbCodeBuf[off++] = X86_OP_REX_B;
782 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
783 pbCodeBuf[off++] = RT_BYTE1(uImm64);
784 pbCodeBuf[off++] = RT_BYTE2(uImm64);
785 pbCodeBuf[off++] = RT_BYTE3(uImm64);
786 pbCodeBuf[off++] = RT_BYTE4(uImm64);
787 }
788 else
789 {
790 /* mov gpr, imm64 */
791 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
792 AssertReturn(pbCodeBuf, UINT32_MAX);
793 if (iGpr < 8)
794 pbCodeBuf[off++] = X86_OP_REX_W;
795 else
796 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
797 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
798 pbCodeBuf[off++] = RT_BYTE1(uImm64);
799 pbCodeBuf[off++] = RT_BYTE2(uImm64);
800 pbCodeBuf[off++] = RT_BYTE3(uImm64);
801 pbCodeBuf[off++] = RT_BYTE4(uImm64);
802 pbCodeBuf[off++] = RT_BYTE5(uImm64);
803 pbCodeBuf[off++] = RT_BYTE6(uImm64);
804 pbCodeBuf[off++] = RT_BYTE7(uImm64);
805 pbCodeBuf[off++] = RT_BYTE8(uImm64);
806 }
807
808#elif RT_ARCH_ARM64
809 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
810 AssertReturn(pu32CodeBuf, UINT32_MAX);
811
812 /*
813 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
814 * supply remaining bits using 'movk grp, imm16, lsl #x'.
815 *
816 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
817 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
818 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
819 * after the first non-zero immediate component so we switch to movk for
820 * the remainder.
821 */
822 uint32_t fMovK = 0;
823 /* mov gpr, imm16 */
824 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
825 if (uImmPart)
826 {
827 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
828 fMovK |= RT_BIT_32(29);
829 }
830 /* mov[k] gpr, imm16, lsl #16 */
831 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
832 if (uImmPart)
833 {
834 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
835 fMovK |= RT_BIT_32(29);
836 }
837 /* mov[k] gpr, imm16, lsl #32 */
838 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
839 if (uImmPart)
840 {
841 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
842 fMovK |= RT_BIT_32(29);
843 }
844 /* mov[k] gpr, imm16, lsl #48 */
845 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
846 if (uImmPart)
847 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
848
849 /** @todo there is an inverted mask variant we might want to explore if it
850 * reduces the number of instructions... */
851 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
852 * clang 12.x does that, only to use the 'x' version for the
853 * addressing in the following ldr). */
854
855#else
856# error "port me"
857#endif
858 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
859 return off;
860}
861
862
863/**
864 * Emits loading a constant into a 8-bit GPR
865 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
866 * only the ARM64 version does that.
867 */
868DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
869{
870#ifdef RT_ARCH_AMD64
871 /* mov gpr, imm8 */
872 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
873 AssertReturn(pbCodeBuf, UINT32_MAX);
874 if (iGpr >= 8)
875 pbCodeBuf[off++] = X86_OP_REX_B;
876 else if (iGpr >= 4)
877 pbCodeBuf[off++] = X86_OP_REX;
878 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
879 pbCodeBuf[off++] = RT_BYTE1(uImm8);
880
881#elif RT_ARCH_ARM64
882 /* movz gpr, imm16, lsl #0 */
883 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
884 AssertReturn(pu32CodeBuf, UINT32_MAX);
885 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
886
887#else
888# error "port me"
889#endif
890 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
891 return off;
892}
893
894
895#ifdef RT_ARCH_AMD64
896/**
897 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
898 */
899DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
900{
901 if (offVCpu < 128)
902 {
903 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
904 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
905 }
906 else
907 {
908 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
909 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
910 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
911 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
912 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
913 }
914 return off;
915}
916#elif RT_ARCH_ARM64
917/**
918 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
919 */
920DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
921 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
922{
923 /*
924 * There are a couple of ldr variants that takes an immediate offset, so
925 * try use those if we can, otherwise we have to use the temporary register
926 * help with the addressing.
927 */
928 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
929 {
930 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
931 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
932 AssertReturn(pu32CodeBuf, UINT32_MAX);
933 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
934 }
935 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
936 {
937 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
938 AssertReturn(pu32CodeBuf, UINT32_MAX);
939 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
940 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
941 }
942 else
943 {
944 /* The offset is too large, so we must load it into a register and use
945 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
946 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
947 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
948
949 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
950 AssertReturn(pu32CodeBuf, UINT32_MAX);
951 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
952 }
953 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
954 return off;
955}
956#endif
957
958
959/**
960 * Emits a 64-bit GPR load of a VCpu value.
961 */
962DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
963{
964#ifdef RT_ARCH_AMD64
965 /* mov reg64, mem64 */
966 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
967 AssertReturn(pbCodeBuf, UINT32_MAX);
968 if (iGpr < 8)
969 pbCodeBuf[off++] = X86_OP_REX_W;
970 else
971 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
972 pbCodeBuf[off++] = 0x8b;
973 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
974 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
975
976#elif RT_ARCH_ARM64
977 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
978
979#else
980# error "port me"
981#endif
982 return off;
983}
984
985
986/**
987 * Emits a 32-bit GPR load of a VCpu value.
988 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
989 */
990DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
991{
992#ifdef RT_ARCH_AMD64
993 /* mov reg32, mem32 */
994 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
995 AssertReturn(pbCodeBuf, UINT32_MAX);
996 if (iGpr >= 8)
997 pbCodeBuf[off++] = X86_OP_REX_R;
998 pbCodeBuf[off++] = 0x8b;
999 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1000 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1001
1002#elif RT_ARCH_ARM64
1003 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1004
1005#else
1006# error "port me"
1007#endif
1008 return off;
1009}
1010
1011
1012/**
1013 * Emits a 16-bit GPR load of a VCpu value.
1014 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
1015 */
1016DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1017{
1018#ifdef RT_ARCH_AMD64
1019 /* movzx reg32, mem16 */
1020 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1021 AssertReturn(pbCodeBuf, UINT32_MAX);
1022 if (iGpr >= 8)
1023 pbCodeBuf[off++] = X86_OP_REX_R;
1024 pbCodeBuf[off++] = 0x0f;
1025 pbCodeBuf[off++] = 0xb7;
1026 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1027 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1028
1029#elif RT_ARCH_ARM64
1030 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1031
1032#else
1033# error "port me"
1034#endif
1035 return off;
1036}
1037
1038
1039/**
1040 * Emits a 8-bit GPR load of a VCpu value.
1041 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
1042 */
1043DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1044{
1045#ifdef RT_ARCH_AMD64
1046 /* movzx reg32, mem8 */
1047 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1048 AssertReturn(pbCodeBuf, UINT32_MAX);
1049 if (iGpr >= 8)
1050 pbCodeBuf[off++] = X86_OP_REX_R;
1051 pbCodeBuf[off++] = 0x0f;
1052 pbCodeBuf[off++] = 0xb6;
1053 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1054 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1055
1056#elif RT_ARCH_ARM64
1057 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1058
1059#else
1060# error "port me"
1061#endif
1062 return off;
1063}
1064
1065
1066/**
1067 * Emits a store of a GPR value to a 64-bit VCpu field.
1068 */
1069DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1070{
1071#ifdef RT_ARCH_AMD64
1072 /* mov mem64, reg64 */
1073 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1074 AssertReturn(pbCodeBuf, UINT32_MAX);
1075 if (iGpr < 8)
1076 pbCodeBuf[off++] = X86_OP_REX_W;
1077 else
1078 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1079 pbCodeBuf[off++] = 0x89;
1080 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1081 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1082
1083#elif RT_ARCH_ARM64
1084 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1085
1086#else
1087# error "port me"
1088#endif
1089 return off;
1090}
1091
1092
1093/**
1094 * Emits a store of a GPR value to a 32-bit VCpu field.
1095 */
1096DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1097{
1098#ifdef RT_ARCH_AMD64
1099 /* mov mem32, reg32 */
1100 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1101 AssertReturn(pbCodeBuf, UINT32_MAX);
1102 if (iGpr >= 8)
1103 pbCodeBuf[off++] = X86_OP_REX_R;
1104 pbCodeBuf[off++] = 0x89;
1105 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1106 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1107
1108#elif RT_ARCH_ARM64
1109 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1110
1111#else
1112# error "port me"
1113#endif
1114 return off;
1115}
1116
1117
1118/**
1119 * Emits a store of a GPR value to a 16-bit VCpu field.
1120 */
1121DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1122{
1123#ifdef RT_ARCH_AMD64
1124 /* mov mem16, reg16 */
1125 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1126 AssertReturn(pbCodeBuf, UINT32_MAX);
1127 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1128 if (iGpr >= 8)
1129 pbCodeBuf[off++] = X86_OP_REX_R;
1130 pbCodeBuf[off++] = 0x89;
1131 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1132 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1133
1134#elif RT_ARCH_ARM64
1135 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1136
1137#else
1138# error "port me"
1139#endif
1140 return off;
1141}
1142
1143
1144/**
1145 * Emits a store of a GPR value to a 8-bit VCpu field.
1146 */
1147DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1148{
1149#ifdef RT_ARCH_AMD64
1150 /* mov mem8, reg8 */
1151 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1152 AssertReturn(pbCodeBuf, UINT32_MAX);
1153 if (iGpr >= 8)
1154 pbCodeBuf[off++] = X86_OP_REX_R;
1155 pbCodeBuf[off++] = 0x88;
1156 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1157 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1158
1159#elif RT_ARCH_ARM64
1160 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1161
1162#else
1163# error "port me"
1164#endif
1165 return off;
1166}
1167
1168
1169/**
1170 * Emits a gprdst = gprsrc load.
1171 */
1172DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1173{
1174#ifdef RT_ARCH_AMD64
1175 /* mov gprdst, gprsrc */
1176 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1177 AssertReturn(pbCodeBuf, UINT32_MAX);
1178 if ((iGprDst | iGprSrc) >= 8)
1179 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1180 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1181 : X86_OP_REX_W | X86_OP_REX_R;
1182 else
1183 pbCodeBuf[off++] = X86_OP_REX_W;
1184 pbCodeBuf[off++] = 0x8b;
1185 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1186
1187#elif RT_ARCH_ARM64
1188 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1189 AssertReturn(pu32CodeBuf, UINT32_MAX);
1190 /* mov dst, src; alias for: orr dst, xzr, src */
1191 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1192
1193#else
1194# error "port me"
1195#endif
1196 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1197 return off;
1198}
1199
1200#ifdef RT_ARCH_AMD64
1201/**
1202 * Common bit of iemNativeEmitLoadGprByBp and friends.
1203 */
1204DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1205 PIEMRECOMPILERSTATE pReNativeAssert)
1206{
1207 if (offDisp < 128 && offDisp >= -128)
1208 {
1209 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1210 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1211 }
1212 else
1213 {
1214 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1215 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1216 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1217 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1218 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1219 }
1220 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1221 return off;
1222}
1223#endif
1224
1225
1226#ifdef RT_ARCH_AMD64
1227/**
1228 * Emits a 64-bit GRP load instruction with an BP relative source address.
1229 */
1230DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1231{
1232 /* mov gprdst, qword [rbp + offDisp] */
1233 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1234 AssertReturn(pbCodeBuf, UINT32_MAX);
1235 if (iGprDst < 8)
1236 pbCodeBuf[off++] = X86_OP_REX_W;
1237 else
1238 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1239 pbCodeBuf[off++] = 0x8b;
1240 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1241}
1242#endif
1243
1244
1245#ifdef RT_ARCH_AMD64
1246/**
1247 * Emits a 32-bit GRP load instruction with an BP relative source address.
1248 */
1249DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1250{
1251 /* mov gprdst, dword [rbp + offDisp] */
1252 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1253 AssertReturn(pbCodeBuf, UINT32_MAX);
1254 if (iGprDst >= 8)
1255 pbCodeBuf[off++] = X86_OP_REX_R;
1256 pbCodeBuf[off++] = 0x8b;
1257 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1258}
1259#endif
1260
1261
1262#ifdef RT_ARCH_AMD64
1263/**
1264 * Emits a load effective address to a GRP with an BP relative source address.
1265 */
1266DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1267{
1268 /* lea gprdst, [rbp + offDisp] */
1269 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1270 AssertReturn(pbCodeBuf, UINT32_MAX);
1271 if (iGprDst < 8)
1272 pbCodeBuf[off++] = X86_OP_REX_W;
1273 else
1274 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1275 pbCodeBuf[off++] = 0x8d;
1276 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1277}
1278#endif
1279
1280
1281/**
1282 * Emits a 64-bit GPR store with an BP relative destination address.
1283 *
1284 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1285 */
1286DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1287{
1288#ifdef RT_ARCH_AMD64
1289 /* mov qword [rbp + offDisp], gprdst */
1290 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1291 AssertReturn(pbCodeBuf, UINT32_MAX);
1292 if (iGprSrc < 8)
1293 pbCodeBuf[off++] = X86_OP_REX_W;
1294 else
1295 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1296 pbCodeBuf[off++] = 0x89;
1297 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1298
1299#elif defined(RT_ARCH_ARM64)
1300 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1301 {
1302 /* str w/ unsigned imm12 (scaled) */
1303 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1304 AssertReturn(pu32CodeBuf, UINT32_MAX);
1305 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1306 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1307 }
1308 else if (offDisp >= -256 && offDisp <= 256)
1309 {
1310 /* stur w/ signed imm9 (unscaled) */
1311 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1312 AssertReturn(pu32CodeBuf, UINT32_MAX);
1313 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1314 }
1315 else
1316 {
1317 /* Use temporary indexing register. */
1318 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1319 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1320 AssertReturn(pu32CodeBuf, UINT32_MAX);
1321 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1322 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1323 }
1324 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1325 return off;
1326
1327#else
1328# error "Port me!"
1329#endif
1330}
1331
1332
1333/**
1334 * Emits a 64-bit immediate store with an BP relative destination address.
1335 *
1336 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1337 */
1338DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1339{
1340#ifdef RT_ARCH_AMD64
1341 if ((int64_t)uImm64 == (int32_t)uImm64)
1342 {
1343 /* mov qword [rbp + offDisp], imm32 - sign extended */
1344 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1345 AssertReturn(pbCodeBuf, UINT32_MAX);
1346
1347 pbCodeBuf[off++] = X86_OP_REX_W;
1348 pbCodeBuf[off++] = 0xc7;
1349 if (offDisp < 128 && offDisp >= -128)
1350 {
1351 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1352 pbCodeBuf[off++] = (uint8_t)offDisp;
1353 }
1354 else
1355 {
1356 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1357 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1358 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1359 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1360 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1361 }
1362 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1363 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1364 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1365 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1366 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1367 return off;
1368 }
1369#endif
1370
1371 /* Load tmp0, imm64; Store tmp to bp+disp. */
1372 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1373 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1374}
1375
1376
1377/*********************************************************************************************************************************
1378* Subtraction and Additions *
1379*********************************************************************************************************************************/
1380
1381
1382#ifdef RT_ARCH_AMD64
1383/**
1384 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1385 */
1386DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1387{
1388 /* sub gprdst, imm8/imm32 */
1389 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1390 AssertReturn(pbCodeBuf, UINT32_MAX);
1391 if (iGprDst < 8)
1392 pbCodeBuf[off++] = X86_OP_REX_W;
1393 else
1394 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1395 if (iSubtrahend < 128 && iSubtrahend >= -128)
1396 {
1397 pbCodeBuf[off++] = 0x83;
1398 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1399 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1400 }
1401 else
1402 {
1403 pbCodeBuf[off++] = 0x81;
1404 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1405 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1406 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1407 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1408 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1409 }
1410 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1411 return off;
1412}
1413#endif
1414
1415
1416/**
1417 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1418 * @note The AMD64 version sets flags.
1419 */
1420DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1421{
1422#if defined(RT_ARCH_AMD64)
1423 /* add Gv,Ev */
1424 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1425 AssertReturn(pbCodeBuf, UINT32_MAX);
1426 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1427 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1428 pbCodeBuf[off++] = 0x04;
1429 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1430
1431#elif defined(RT_ARCH_ARM64)
1432 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1433 AssertReturn(pu32CodeBuf, UINT32_MAX);
1434 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1435
1436#else
1437# error "Port me"
1438#endif
1439 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1440 return off;
1441}
1442
1443
1444/**
1445 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1446 */
1447DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1448{
1449#if defined(RT_ARCH_AMD64)
1450 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1451 AssertReturn(pbCodeBuf, UINT32_MAX);
1452 /* add or inc */
1453 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1454 if (iImm8 != 1)
1455 {
1456 pbCodeBuf[off++] = 0x83;
1457 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1458 pbCodeBuf[off++] = (uint8_t)iImm8;
1459 }
1460 else
1461 {
1462 pbCodeBuf[off++] = 0xff;
1463 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1464 }
1465
1466#elif defined(RT_ARCH_ARM64)
1467 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1468 AssertReturn(pu32CodeBuf, UINT32_MAX);
1469 if (iImm8 >= 0)
1470 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1471 else
1472 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1473
1474#else
1475# error "Port me"
1476#endif
1477 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1478 return off;
1479}
1480
1481
1482/**
1483 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1484 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1485 */
1486DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1487{
1488#if defined(RT_ARCH_AMD64)
1489 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1490 AssertReturn(pbCodeBuf, UINT32_MAX);
1491 /* add or inc */
1492 if (iGprDst >= 8)
1493 pbCodeBuf[off++] = X86_OP_REX_B;
1494 if (iImm8 != 1)
1495 {
1496 pbCodeBuf[off++] = 0x83;
1497 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1498 pbCodeBuf[off++] = (uint8_t)iImm8;
1499 }
1500 else
1501 {
1502 pbCodeBuf[off++] = 0xff;
1503 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1504 }
1505
1506#elif defined(RT_ARCH_ARM64)
1507 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1508 AssertReturn(pu32CodeBuf, UINT32_MAX);
1509 if (iImm8 >= 0)
1510 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1511 else
1512 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1513
1514#else
1515# error "Port me"
1516#endif
1517 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1518 return off;
1519}
1520
1521
1522/**
1523 * Emits a 64-bit GPR additions with a 64-bit signed addend.
1524 */
1525DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1526{
1527#if defined(RT_ARCH_AMD64)
1528 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1529 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1530
1531 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1532 {
1533 /* add grp, imm32 */
1534 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1535 AssertReturn(pbCodeBuf, UINT32_MAX);
1536 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1537 pbCodeBuf[off++] = 0x81;
1538 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1539 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1540 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1541 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1542 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1543 }
1544 else
1545 {
1546 /* Best to use a temporary register to deal with this in the simplest way: */
1547 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1548 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1549
1550 /* add dst, tmpreg */
1551 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1552 AssertReturn(pbCodeBuf, UINT32_MAX);
1553 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1554 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1555 pbCodeBuf[off++] = 0x03;
1556 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1557
1558 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1559 }
1560
1561#elif defined(RT_ARCH_ARM64)
1562 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1563 {
1564 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1565 AssertReturn(pu32CodeBuf, UINT32_MAX);
1566 if (iAddend >= 0)
1567 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
1568 else
1569 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
1570 }
1571 else
1572 {
1573 /* Use temporary register for the immediate. */
1574 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1575 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1576
1577 /* add gprdst, gprdst, tmpreg */
1578 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1579 AssertReturn(pu32CodeBuf, UINT32_MAX);
1580 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
1581
1582 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1583 }
1584
1585#else
1586# error "Port me"
1587#endif
1588 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1589 return off;
1590}
1591
1592
1593/**
1594 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
1595 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1596 */
1597DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1598{
1599#if defined(RT_ARCH_AMD64)
1600 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1601 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1602
1603 /* add grp, imm32 */
1604 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1605 AssertReturn(pbCodeBuf, UINT32_MAX);
1606 if (iGprDst >= 8)
1607 pbCodeBuf[off++] = X86_OP_REX_B;
1608 pbCodeBuf[off++] = 0x81;
1609 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1610 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1611 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1612 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1613 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1614
1615#elif defined(RT_ARCH_ARM64)
1616 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1617 {
1618 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1619 AssertReturn(pu32CodeBuf, UINT32_MAX);
1620 if (iAddend >= 0)
1621 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
1622 else
1623 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
1624 }
1625 else
1626 {
1627 /* Use temporary register for the immediate. */
1628 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1629 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1630
1631 /* add gprdst, gprdst, tmpreg */
1632 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1633 AssertReturn(pu32CodeBuf, UINT32_MAX);
1634 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
1635
1636 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1637 }
1638
1639#else
1640# error "Port me"
1641#endif
1642 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1643 return off;
1644}
1645
1646
1647
1648/*********************************************************************************************************************************
1649* Bit Operations *
1650*********************************************************************************************************************************/
1651
1652/**
1653 * Emits code for clearing bits 16 thru 63 in the GPR.
1654 */
1655DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1656{
1657#if defined(RT_ARCH_AMD64)
1658 /* movzx reg32, reg16 */
1659 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1660 AssertReturn(pbCodeBuf, UINT32_MAX);
1661 if (iGprDst >= 8)
1662 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
1663 pbCodeBuf[off++] = 0x0f;
1664 pbCodeBuf[off++] = 0xb7;
1665 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1666
1667#elif defined(RT_ARCH_ARM64)
1668 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1669 AssertReturn(pu32CodeBuf, UINT32_MAX);
1670# if 1
1671 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1672# else
1673 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1674 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1675# endif
1676#else
1677# error "Port me"
1678#endif
1679 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1680 return off;
1681}
1682
1683
1684/**
1685 * Emits code for AND'ing two 64-bit GPRs.
1686 */
1687DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1688{
1689#if defined(RT_ARCH_AMD64)
1690 /* and Gv, Ev */
1691 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1692 AssertReturn(pbCodeBuf, UINT32_MAX);
1693 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1694 pbCodeBuf[off++] = 0x23;
1695 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1696
1697#elif defined(RT_ARCH_ARM64)
1698 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1699 AssertReturn(pu32CodeBuf, UINT32_MAX);
1700 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1701
1702#else
1703# error "Port me"
1704#endif
1705 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1706 return off;
1707}
1708
1709
1710/**
1711 * Emits code for AND'ing two 32-bit GPRs.
1712 */
1713DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1714{
1715#if defined(RT_ARCH_AMD64)
1716 /* and Gv, Ev */
1717 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1718 AssertReturn(pbCodeBuf, UINT32_MAX);
1719 if (iGprDst >= 8 || iGprSrc >= 8)
1720 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1721 pbCodeBuf[off++] = 0x23;
1722 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1723
1724#elif defined(RT_ARCH_ARM64)
1725 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1726 AssertReturn(pu32CodeBuf, UINT32_MAX);
1727 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
1728
1729#else
1730# error "Port me"
1731#endif
1732 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1733 return off;
1734}
1735
1736
1737/**
1738 * Emits code for XOR'ing two 64-bit GPRs.
1739 */
1740DECLINLINE(uint32_t ) iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1741{
1742#if defined(RT_ARCH_AMD64)
1743 /* and Gv, Ev */
1744 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1745 AssertReturn(pbCodeBuf, UINT32_MAX);
1746 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1747 pbCodeBuf[off++] = 0x33;
1748 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1749
1750#elif defined(RT_ARCH_ARM64)
1751 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1752 AssertReturn(pu32CodeBuf, UINT32_MAX);
1753 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1754
1755#else
1756# error "Port me"
1757#endif
1758 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1759 return off;
1760}
1761
1762
1763/**
1764 * Emits code for XOR'ing two 32-bit GPRs.
1765 */
1766DECLINLINE(uint32_t ) iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1767{
1768#if defined(RT_ARCH_AMD64)
1769 /* and Gv, Ev */
1770 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1771 AssertReturn(pbCodeBuf, UINT32_MAX);
1772 if (iGprDst >= 8 || iGprSrc >= 8)
1773 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1774 pbCodeBuf[off++] = 0x33;
1775 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1776
1777#elif defined(RT_ARCH_ARM64)
1778 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1779 AssertReturn(pu32CodeBuf, UINT32_MAX);
1780 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
1781
1782#else
1783# error "Port me"
1784#endif
1785 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1786 return off;
1787}
1788
1789
1790/*********************************************************************************************************************************
1791* Shifting *
1792*********************************************************************************************************************************/
1793
1794/**
1795 * Emits code for shifting a GPR a fixed number of bits to the left.
1796 */
1797DECLINLINE(uint32_t ) iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1798{
1799 Assert(cShift > 0 && cShift < 64);
1800
1801#if defined(RT_ARCH_AMD64)
1802 /* shl dst, cShift */
1803 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1804 AssertReturn(pbCodeBuf, UINT32_MAX);
1805 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1806 if (cShift != 1)
1807 {
1808 pbCodeBuf[off++] = 0xc1;
1809 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1810 pbCodeBuf[off++] = cShift;
1811 }
1812 else
1813 {
1814 pbCodeBuf[off++] = 0xd1;
1815 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1816 }
1817
1818#elif defined(RT_ARCH_ARM64)
1819 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1820 AssertReturn(pu32CodeBuf, UINT32_MAX);
1821 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
1822
1823#else
1824# error "Port me"
1825#endif
1826 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1827 return off;
1828}
1829
1830
1831/**
1832 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
1833 */
1834DECLINLINE(uint32_t ) iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1835{
1836 Assert(cShift > 0 && cShift < 32);
1837
1838#if defined(RT_ARCH_AMD64)
1839 /* shl dst, cShift */
1840 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1841 AssertReturn(pbCodeBuf, UINT32_MAX);
1842 if (iGprDst >= 8)
1843 pbCodeBuf[off++] = X86_OP_REX_B;
1844 if (cShift != 1)
1845 {
1846 pbCodeBuf[off++] = 0xc1;
1847 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1848 pbCodeBuf[off++] = cShift;
1849 }
1850 else
1851 {
1852 pbCodeBuf[off++] = 0xd1;
1853 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1854 }
1855
1856#elif defined(RT_ARCH_ARM64)
1857 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1858 AssertReturn(pu32CodeBuf, UINT32_MAX);
1859 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
1860
1861#else
1862# error "Port me"
1863#endif
1864 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1865 return off;
1866}
1867
1868
1869/**
1870 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1871 */
1872DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1873{
1874 Assert(cShift > 0 && cShift < 64);
1875
1876#if defined(RT_ARCH_AMD64)
1877 /* shr dst, cShift */
1878 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1879 AssertReturn(pbCodeBuf, UINT32_MAX);
1880 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1881 if (cShift != 1)
1882 {
1883 pbCodeBuf[off++] = 0xc1;
1884 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1885 pbCodeBuf[off++] = cShift;
1886 }
1887 else
1888 {
1889 pbCodeBuf[off++] = 0xd1;
1890 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1891 }
1892
1893#elif defined(RT_ARCH_ARM64)
1894 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1895 AssertReturn(pu32CodeBuf, UINT32_MAX);
1896 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1897
1898#else
1899# error "Port me"
1900#endif
1901 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1902 return off;
1903}
1904
1905
1906/**
1907 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
1908 * right.
1909 */
1910DECLINLINE(uint32_t ) iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1911{
1912 Assert(cShift > 0 && cShift < 32);
1913
1914#if defined(RT_ARCH_AMD64)
1915 /* shr dst, cShift */
1916 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1917 AssertReturn(pbCodeBuf, UINT32_MAX);
1918 if (iGprDst >= 8)
1919 pbCodeBuf[off++] = X86_OP_REX_B;
1920 if (cShift != 1)
1921 {
1922 pbCodeBuf[off++] = 0xc1;
1923 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1924 pbCodeBuf[off++] = cShift;
1925 }
1926 else
1927 {
1928 pbCodeBuf[off++] = 0xd1;
1929 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1930 }
1931
1932#elif defined(RT_ARCH_ARM64)
1933 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1934 AssertReturn(pu32CodeBuf, UINT32_MAX);
1935 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
1936
1937#else
1938# error "Port me"
1939#endif
1940 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1941 return off;
1942}
1943
1944
1945
1946/*********************************************************************************************************************************
1947* Compare and Testing *
1948*********************************************************************************************************************************/
1949
1950
1951#ifdef RT_ARCH_ARM64
1952/**
1953 * Emits an ARM64 compare instruction.
1954 */
1955DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1956 bool f64Bit = true, uint32_t cShift = 0,
1957 ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1958{
1959 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1960 AssertReturn(pu32CodeBuf, UINT32_MAX);
1961 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
1962 f64Bit, true /*fSetFlags*/, cShift, enmShift);
1963 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1964 return off;
1965}
1966#endif
1967
1968
1969/**
1970 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1971 * with conditional instruction.
1972 */
1973DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1974{
1975#ifdef RT_ARCH_AMD64
1976 /* cmp Gv, Ev */
1977 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1978 AssertReturn(pbCodeBuf, UINT32_MAX);
1979 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1980 pbCodeBuf[off++] = 0x3b;
1981 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1982 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1983
1984#elif defined(RT_ARCH_ARM64)
1985 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1986
1987#else
1988# error "Port me!"
1989#endif
1990 return off;
1991}
1992
1993
1994/**
1995 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1996 * with conditional instruction.
1997 */
1998DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1999 uint8_t iGprLeft, uint8_t iGprRight)
2000{
2001#ifdef RT_ARCH_AMD64
2002 /* cmp Gv, Ev */
2003 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2004 AssertReturn(pbCodeBuf, UINT32_MAX);
2005 if (iGprLeft >= 8 || iGprRight >= 8)
2006 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
2007 pbCodeBuf[off++] = 0x3b;
2008 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2009 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2010
2011#elif defined(RT_ARCH_ARM64)
2012 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
2013
2014#else
2015# error "Port me!"
2016#endif
2017 return off;
2018}
2019
2020
2021
2022/*********************************************************************************************************************************
2023* Branching *
2024*********************************************************************************************************************************/
2025
2026/**
2027 * Emits a JMP rel32 / B imm19 to the given label.
2028 */
2029DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2030{
2031 Assert(idxLabel < pReNative->cLabels);
2032
2033#ifdef RT_ARCH_AMD64
2034 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2035 AssertReturn(pbCodeBuf, UINT32_MAX);
2036 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2037 {
2038 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2039 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2040 {
2041 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2042 pbCodeBuf[off++] = (uint8_t)offRel;
2043 }
2044 else
2045 {
2046 offRel -= 3;
2047 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2048 pbCodeBuf[off++] = RT_BYTE1(offRel);
2049 pbCodeBuf[off++] = RT_BYTE2(offRel);
2050 pbCodeBuf[off++] = RT_BYTE3(offRel);
2051 pbCodeBuf[off++] = RT_BYTE4(offRel);
2052 }
2053 }
2054 else
2055 {
2056 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2057 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
2058 pbCodeBuf[off++] = 0xfe;
2059 pbCodeBuf[off++] = 0xff;
2060 pbCodeBuf[off++] = 0xff;
2061 pbCodeBuf[off++] = 0xff;
2062 }
2063 pbCodeBuf[off++] = 0xcc; /* int3 poison */
2064
2065#elif defined(RT_ARCH_ARM64)
2066 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2067 AssertReturn(pu32CodeBuf, UINT32_MAX);
2068 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2069 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
2070 else
2071 {
2072 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
2073 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2074 }
2075
2076#else
2077# error "Port me!"
2078#endif
2079 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2080 return off;
2081}
2082
2083
2084/**
2085 * Emits a JMP rel32 / B imm19 to a new undefined label.
2086 */
2087DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2088 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2089{
2090 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2091 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
2092 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2093}
2094
2095/** Condition type. */
2096#ifdef RT_ARCH_AMD64
2097typedef enum IEMNATIVEINSTRCOND : uint8_t
2098{
2099 kIemNativeInstrCond_o = 0,
2100 kIemNativeInstrCond_no,
2101 kIemNativeInstrCond_c,
2102 kIemNativeInstrCond_nc,
2103 kIemNativeInstrCond_e,
2104 kIemNativeInstrCond_ne,
2105 kIemNativeInstrCond_be,
2106 kIemNativeInstrCond_nbe,
2107 kIemNativeInstrCond_s,
2108 kIemNativeInstrCond_ns,
2109 kIemNativeInstrCond_p,
2110 kIemNativeInstrCond_np,
2111 kIemNativeInstrCond_l,
2112 kIemNativeInstrCond_nl,
2113 kIemNativeInstrCond_le,
2114 kIemNativeInstrCond_nle
2115} IEMNATIVEINSTRCOND;
2116#elif defined(RT_ARCH_ARM64)
2117typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2118#else
2119# error "Port me!"
2120#endif
2121
2122
2123/**
2124 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2125 */
2126DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2127 uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2128{
2129 Assert(idxLabel < pReNative->cLabels);
2130
2131#ifdef RT_ARCH_AMD64
2132 /* jcc rel32 */
2133 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2134 AssertReturn(pbCodeBuf, UINT32_MAX);
2135 pbCodeBuf[off++] = 0x0f;
2136 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2137 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
2138 pbCodeBuf[off++] = 0x00;
2139 pbCodeBuf[off++] = 0x00;
2140 pbCodeBuf[off++] = 0x00;
2141 pbCodeBuf[off++] = 0x00;
2142
2143#elif defined(RT_ARCH_ARM64)
2144 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2145 AssertReturn(pu32CodeBuf, UINT32_MAX);
2146 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
2147 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2148
2149#else
2150# error "Port me!"
2151#endif
2152 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2153 return off;
2154}
2155
2156
2157/**
2158 * Emits a Jcc rel32 / B.cc imm19 to a new label.
2159 */
2160DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2161 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2162{
2163 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2164 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
2165 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2166}
2167
2168
2169/**
2170 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2171 */
2172DECLINLINE(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2173{
2174#ifdef RT_ARCH_AMD64
2175 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2176#elif defined(RT_ARCH_ARM64)
2177 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2178#else
2179# error "Port me!"
2180#endif
2181}
2182
2183/**
2184 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2185 */
2186DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2187 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2188{
2189#ifdef RT_ARCH_AMD64
2190 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2191#elif defined(RT_ARCH_ARM64)
2192 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2193#else
2194# error "Port me!"
2195#endif
2196}
2197
2198
2199/**
2200 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2201 */
2202DECLINLINE(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2203{
2204#ifdef RT_ARCH_AMD64
2205 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2206#elif defined(RT_ARCH_ARM64)
2207 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2208#else
2209# error "Port me!"
2210#endif
2211}
2212
2213/**
2214 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2215 */
2216DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2217 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2218{
2219#ifdef RT_ARCH_AMD64
2220 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2221#elif defined(RT_ARCH_ARM64)
2222 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2223#else
2224# error "Port me!"
2225#endif
2226}
2227
2228
2229/**
2230 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2231 */
2232DECLINLINE(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2233{
2234#ifdef RT_ARCH_AMD64
2235 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2236#elif defined(RT_ARCH_ARM64)
2237 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2238#else
2239# error "Port me!"
2240#endif
2241}
2242
2243/**
2244 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2245 */
2246DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2247 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2248{
2249#ifdef RT_ARCH_AMD64
2250 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2251#elif defined(RT_ARCH_ARM64)
2252 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2253#else
2254# error "Port me!"
2255#endif
2256}
2257
2258
2259/**
2260 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2261 */
2262DECLINLINE(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2263{
2264#ifdef RT_ARCH_AMD64
2265 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2266#elif defined(RT_ARCH_ARM64)
2267 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2268#else
2269# error "Port me!"
2270#endif
2271}
2272
2273/**
2274 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2275 */
2276DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2277 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2278{
2279#ifdef RT_ARCH_AMD64
2280 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2281#elif defined(RT_ARCH_ARM64)
2282 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2283#else
2284# error "Port me!"
2285#endif
2286}
2287
2288
2289/**
2290 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2291 * How @a offJmp is applied is are target specific.
2292 */
2293DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2294 int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2295{
2296#ifdef RT_ARCH_AMD64
2297 /* jcc rel32 */
2298 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2299 AssertReturn(pbCodeBuf, UINT32_MAX);
2300 if (offTarget < 128 && offTarget >= -128)
2301 {
2302 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
2303 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2304 }
2305 else
2306 {
2307 pbCodeBuf[off++] = 0x0f;
2308 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2309 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2310 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2311 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2312 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2313 }
2314
2315#elif defined(RT_ARCH_ARM64)
2316 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2317 AssertReturn(pu32CodeBuf, UINT32_MAX);
2318 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2319
2320#else
2321# error "Port me!"
2322#endif
2323 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2324 return off;
2325}
2326
2327
2328/**
2329 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2330 * How @a offJmp is applied is are target specific.
2331 */
2332DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2333{
2334#ifdef RT_ARCH_AMD64
2335 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2336#elif defined(RT_ARCH_ARM64)
2337 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2338#else
2339# error "Port me!"
2340#endif
2341}
2342
2343
2344/**
2345 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2346 * How @a offJmp is applied is are target specific.
2347 */
2348DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2349{
2350#ifdef RT_ARCH_AMD64
2351 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2352#elif defined(RT_ARCH_ARM64)
2353 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2354#else
2355# error "Port me!"
2356#endif
2357}
2358
2359
2360/**
2361 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2362 * How @a offJmp is applied is are target specific.
2363 */
2364DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2365{
2366#ifdef RT_ARCH_AMD64
2367 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2368#elif defined(RT_ARCH_ARM64)
2369 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2370#else
2371# error "Port me!"
2372#endif
2373}
2374
2375
2376/**
2377 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2378 * How @a offJmp is applied is are target specific.
2379 */
2380DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2381{
2382#ifdef RT_ARCH_AMD64
2383 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2384#elif defined(RT_ARCH_ARM64)
2385 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2386#else
2387# error "Port me!"
2388#endif
2389}
2390
2391
2392/**
2393 * Fixes up a conditional jump to a fixed label.
2394 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2395 */
2396DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2397{
2398# if defined(RT_ARCH_AMD64)
2399 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2400 if (pbCodeBuf[offFixup] != 0x0f)
2401 {
2402 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2403 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2404 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2405 }
2406 else
2407 {
2408 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2409 uint32_t const offRel32 = offTarget - (offFixup + 6);
2410 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2411 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2412 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2413 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2414 }
2415
2416# elif defined(RT_ARCH_ARM64)
2417 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2418 Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
2419 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
2420 | (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
2421
2422# endif
2423}
2424
2425
2426/**
2427 * Internal helper, don't call directly.
2428 */
2429DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2430 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel,
2431 bool fJmpIfSet)
2432{
2433 Assert(iBitNo < 64);
2434#ifdef RT_ARCH_AMD64
2435 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2436 AssertReturn(pbCodeBuf, UINT32_MAX);
2437 if (iBitNo < 8)
2438 {
2439 /* test Eb, imm8 */
2440 if (iGprSrc >= 8)
2441 pbCodeBuf[off++] = X86_OP_REX_B;
2442 pbCodeBuf[off++] = 0xf6;
2443 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2444 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2445 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2446 }
2447 else
2448 {
2449 /* bt Ev, imm8 */
2450 if (iBitNo >= 32)
2451 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2452 else if (iGprSrc >= 8)
2453 pbCodeBuf[off++] = X86_OP_REX_B;
2454 pbCodeBuf[off++] = 0x0f;
2455 pbCodeBuf[off++] = 0xba;
2456 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2457 pbCodeBuf[off++] = iBitNo;
2458 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2459 }
2460
2461#elif defined(RT_ARCH_ARM64)
2462 /* Use the TBNZ instruction here. */
2463 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2464 AssertReturn(pu32CodeBuf, UINT32_MAX);
2465 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5), UINT32_MAX);
2466 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2467
2468#else
2469# error "Port me!"
2470#endif
2471 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2472 return off;
2473}
2474
2475
2476/**
2477 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2478 * @a iGprSrc.
2479 *
2480 * @note On ARM64 the range is only +/-8191 instructions.
2481 */
2482DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2483 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2484{
2485 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
2486}
2487
2488
2489/**
2490 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2491 * _set_ in @a iGprSrc.
2492 *
2493 * @note On ARM64 the range is only +/-8191 instructions.
2494 */
2495DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2496 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2497{
2498 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
2499}
2500
2501
2502/**
2503 * Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2504 * flags accordingly.
2505 */
2506DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2507{
2508 Assert(fBits != 0);
2509#ifdef RT_ARCH_AMD64
2510
2511 if (fBits >= UINT32_MAX)
2512 {
2513 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2514 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
2515
2516 /* test Ev,Gv */
2517 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2518 AssertReturn(pbCodeBuf, UINT32_MAX);
2519 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R) | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2520 pbCodeBuf[off++] = 0x85;
2521 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2522
2523 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2524 }
2525 else
2526 {
2527 /* test Eb, imm8 or test Ev, imm32 */
2528 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2529 AssertReturn(pbCodeBuf, UINT32_MAX);
2530 if (iGprSrc >= 8)
2531 pbCodeBuf[off++] = X86_OP_REX_B;
2532 if (fBits <= UINT8_MAX)
2533 {
2534 pbCodeBuf[off++] = 0xf6;
2535 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2536 pbCodeBuf[off++] = (uint8_t)fBits;
2537 }
2538 else
2539 {
2540 pbCodeBuf[off++] = 0xf7;
2541 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2542 pbCodeBuf[off++] = RT_BYTE1(fBits);
2543 pbCodeBuf[off++] = RT_BYTE2(fBits);
2544 pbCodeBuf[off++] = RT_BYTE3(fBits);
2545 pbCodeBuf[off++] = RT_BYTE4(fBits);
2546 }
2547 }
2548
2549#elif defined(RT_ARCH_ARM64)
2550
2551 if (false)
2552 {
2553 /** @todo figure out how to work the immr / N:imms constants. */
2554 }
2555 else
2556 {
2557 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2558 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
2559
2560 /* ands Zr, iGprSrc, iTmpReg */
2561 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2562 AssertReturn(pu32CodeBuf, UINT32_MAX);
2563 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2564
2565 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2566 }
2567
2568#else
2569# error "Port me!"
2570#endif
2571 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2572 return off;
2573}
2574
2575
2576/**
2577 * Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2578 * are set in @a iGprSrc.
2579 */
2580DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2581 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2582{
2583 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2584
2585 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2586 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2587
2588 return off;
2589}
2590
2591
2592/**
2593 * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2594 * are set in @a iGprSrc.
2595 */
2596DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2597 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2598{
2599 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2600
2601 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2602 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2603
2604 return off;
2605}
2606
2607
2608/**
2609 * Emits a call to a 64-bit address.
2610 */
2611DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2612{
2613#ifdef RT_ARCH_AMD64
2614 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2615
2616 /* call rax */
2617 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2618 AssertReturn(pbCodeBuf, UINT32_MAX);
2619 pbCodeBuf[off++] = 0xff;
2620 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2621
2622#elif defined(RT_ARCH_ARM64)
2623 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2624
2625 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2626 AssertReturn(pu32CodeBuf, UINT32_MAX);
2627 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2628#else
2629# error "port me"
2630#endif
2631 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2632 return off;
2633}
2634
2635
2636
2637/** @} */
2638
2639#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2640
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette