VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101587

Last change on this file since 101587 was 101584, checked in by vboxsync, 15 months ago

VMM/IEM: Native IEM_MC_IF_ECX_IS_NZ and IEM_MC_IF_RCX_IS_NZ implementations. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 97.3 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101584 2023-10-24 21:19:18Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46
47/** @name Stack Frame Layout
48 *
49 * @{ */
50/** The size of the area for stack variables and spills and stuff.
51 * @note This limit is duplicated in the python script(s). */
52#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
53#ifdef RT_ARCH_AMD64
54/** Number of stack arguments slots for calls made from the frame. */
55# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
56/** An stack alignment adjustment (between non-volatile register pushes and
57 * the stack variable area, so the latter better aligned). */
58# define IEMNATIVE_FRAME_ALIGN_SIZE 8
59/** Number of any shadow arguments (spill area) for calls we make. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
62# else
63# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
64# endif
65
66/** Frame pointer (RBP) relative offset of the last push. */
67# ifdef RT_OS_WINDOWS
68# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
69# else
70# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
71# endif
72/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
73 * address for it). */
74# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
75/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
76# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
77/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
78# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
79/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
80# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
81/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
82# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
83
84# ifdef RT_OS_WINDOWS
85/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
86# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
87/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
88# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
89/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
90# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
91/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
92# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
93# endif
94
95#elif RT_ARCH_ARM64
96/** No stack argument slots, enough got 8 registers for arguments. */
97# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
98/** There are no argument spill area. */
99# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100
101/** Number of saved registers at the top of our stack frame.
102 * This includes the return address and old frame pointer, so x19 thru x30. */
103# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
104/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
105# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
106
107/** Frame pointer (BP) relative offset of the last push. */
108# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
109
110/** Frame pointer (BP) relative offset of the stack variable area (the lowest
111 * address for it). */
112# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
113
114#else
115# error "port me"
116#endif
117/** @} */
118
119
120/** @name Fixed Register Allocation(s)
121 * @{ */
122/** @def IEMNATIVE_REG_FIXED_PVMCPU
123 * The number of the register holding the pVCpu pointer. */
124/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
125 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
126 * @note This not available on AMD64, only ARM64. */
127/** @def IEMNATIVE_REG_FIXED_TMP0
128 * Dedicated temporary register.
129 * @todo replace this by a register allocator and content tracker. */
130/** @def IEMNATIVE_REG_FIXED_MASK
131 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
132 * architecture. */
133#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
134# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
135# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
136# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
137 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
138 | RT_BIT_32(X86_GREG_xSP) \
139 | RT_BIT_32(X86_GREG_xBP) )
140
141#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
142# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
143# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
144# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
145# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
146 | RT_BIT_32(ARMV8_A64_REG_LR) \
147 | RT_BIT_32(ARMV8_A64_REG_BP) \
148 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
149 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
150 | RT_BIT_32(ARMV8_A64_REG_X18) \
151 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
152
153#else
154# error "port me"
155#endif
156/** @} */
157
158/** @name Call related registers.
159 * @{ */
160/** @def IEMNATIVE_CALL_RET_GREG
161 * The return value register. */
162/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
163 * Number of arguments in registers. */
164/** @def IEMNATIVE_CALL_ARG0_GREG
165 * The general purpose register carrying argument \#0. */
166/** @def IEMNATIVE_CALL_ARG1_GREG
167 * The general purpose register carrying argument \#1. */
168/** @def IEMNATIVE_CALL_ARG2_GREG
169 * The general purpose register carrying argument \#2. */
170/** @def IEMNATIVE_CALL_ARG3_GREG
171 * The general purpose register carrying argument \#3. */
172/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
173 * Mask of registers the callee will not save and may trash. */
174#ifdef RT_ARCH_AMD64
175# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
176
177# ifdef RT_OS_WINDOWS
178# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
179# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
180# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
181# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
182# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
183# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
184 | RT_BIT_32(X86_GREG_xCX) \
185 | RT_BIT_32(X86_GREG_xDX) \
186 | RT_BIT_32(X86_GREG_x8) \
187 | RT_BIT_32(X86_GREG_x9) \
188 | RT_BIT_32(X86_GREG_x10) \
189 | RT_BIT_32(X86_GREG_x11) )
190# else
191# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
192# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
193# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
194# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
195# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
196# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
197# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
198# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
199 | RT_BIT_32(X86_GREG_xCX) \
200 | RT_BIT_32(X86_GREG_xDX) \
201 | RT_BIT_32(X86_GREG_xDI) \
202 | RT_BIT_32(X86_GREG_xSI) \
203 | RT_BIT_32(X86_GREG_x8) \
204 | RT_BIT_32(X86_GREG_x9) \
205 | RT_BIT_32(X86_GREG_x10) \
206 | RT_BIT_32(X86_GREG_x11) )
207# endif
208
209#elif defined(RT_ARCH_ARM64)
210# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
211# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
212# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
213# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
214# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
215# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
216# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
217# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
218# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
219# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
220# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
221 | RT_BIT_32(ARMV8_A64_REG_X1) \
222 | RT_BIT_32(ARMV8_A64_REG_X2) \
223 | RT_BIT_32(ARMV8_A64_REG_X3) \
224 | RT_BIT_32(ARMV8_A64_REG_X4) \
225 | RT_BIT_32(ARMV8_A64_REG_X5) \
226 | RT_BIT_32(ARMV8_A64_REG_X6) \
227 | RT_BIT_32(ARMV8_A64_REG_X7) \
228 | RT_BIT_32(ARMV8_A64_REG_X8) \
229 | RT_BIT_32(ARMV8_A64_REG_X9) \
230 | RT_BIT_32(ARMV8_A64_REG_X10) \
231 | RT_BIT_32(ARMV8_A64_REG_X11) \
232 | RT_BIT_32(ARMV8_A64_REG_X12) \
233 | RT_BIT_32(ARMV8_A64_REG_X13) \
234 | RT_BIT_32(ARMV8_A64_REG_X14) \
235 | RT_BIT_32(ARMV8_A64_REG_X15) \
236 | RT_BIT_32(ARMV8_A64_REG_X16) \
237 | RT_BIT_32(ARMV8_A64_REG_X17) )
238
239#endif
240
241/** @} */
242
243
244/** @def IEMNATIVE_HST_GREG_COUNT
245 * Number of host general purpose registers we tracker. */
246/** @def IEMNATIVE_HST_GREG_MASK
247 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
248 * inverted register masks and such to get down to a correct set of regs. */
249#ifdef RT_ARCH_AMD64
250# define IEMNATIVE_HST_GREG_COUNT 16
251# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
252
253#elif defined(RT_ARCH_ARM64)
254# define IEMNATIVE_HST_GREG_COUNT 32
255# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
256#else
257# error "Port me!"
258#endif
259
260
261/** Native code generator label types. */
262typedef enum
263{
264 kIemNativeLabelType_Invalid = 0,
265 kIemNativeLabelType_Return,
266 kIemNativeLabelType_If,
267 kIemNativeLabelType_Else,
268 kIemNativeLabelType_Endif,
269 kIemNativeLabelType_NonZeroRetOrPassUp,
270 kIemNativeLabelType_RaiseGp0,
271 kIemNativeLabelType_End
272} IEMNATIVELABELTYPE;
273
274/** Native code generator label definition. */
275typedef struct IEMNATIVELABEL
276{
277 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
278 * the epilog. */
279 uint32_t off;
280 /** The type of label (IEMNATIVELABELTYPE). */
281 uint16_t enmType;
282 /** Additional label data, type specific. */
283 uint16_t uData;
284} IEMNATIVELABEL;
285/** Pointer to a label. */
286typedef IEMNATIVELABEL *PIEMNATIVELABEL;
287
288
289/** Native code generator fixup types. */
290typedef enum
291{
292 kIemNativeFixupType_Invalid = 0,
293#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
294 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
295 kIemNativeFixupType_Rel32,
296#elif defined(RT_ARCH_ARM64)
297 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
298 kIemNativeFixupType_RelImm19At5,
299 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
300 kIemNativeFixupType_RelImm14At5,
301#endif
302 kIemNativeFixupType_End
303} IEMNATIVEFIXUPTYPE;
304
305/** Native code generator fixup. */
306typedef struct IEMNATIVEFIXUP
307{
308 /** Code offset of the fixup location. */
309 uint32_t off;
310 /** The IEMNATIVELABEL this is a fixup for. */
311 uint16_t idxLabel;
312 /** The fixup type (IEMNATIVEFIXUPTYPE). */
313 uint8_t enmType;
314 /** Addend or other data. */
315 int8_t offAddend;
316} IEMNATIVEFIXUP;
317/** Pointer to a native code generator fixup. */
318typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
319
320
321/**
322 * Guest registers that can be shadowed in GPRs.
323 */
324typedef enum IEMNATIVEGSTREG : uint8_t
325{
326 kIemNativeGstReg_GprFirst = 0,
327 kIemNativeGstReg_GprLast = 15,
328 kIemNativeGstReg_Pc,
329 kIemNativeGstReg_EFlags, /**< This one is problematic since the higher bits are used internally. */
330 /* gap: 18..23 */
331 kIemNativeGstReg_SegSelFirst = 24,
332 kIemNativeGstReg_SegSelLast = 29,
333 kIemNativeGstReg_SegBaseFirst = 30,
334 kIemNativeGstReg_SegBaseLast = 35,
335 kIemNativeGstReg_SegLimitFirst = 36,
336 kIemNativeGstReg_SegLimitLast = 41,
337 kIemNativeGstReg_End
338} IEMNATIVEGSTREG;
339
340/**
341 * Guest registers (classes) that can be referenced.
342 */
343typedef enum IEMNATIVEGSTREGREF : uint8_t
344{
345 kIemNativeGstRegRef_Invalid = 0,
346 kIemNativeGstRegRef_Gpr,
347 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
348 kIemNativeGstRegRef_EFlags,
349 kIemNativeGstRegRef_MxCsr,
350 kIemNativeGstRegRef_FpuReg,
351 kIemNativeGstRegRef_MReg,
352 kIemNativeGstRegRef_XReg,
353 kIemNativeGstRegRef_YReg,
354 kIemNativeGstRegRef_End
355} IEMNATIVEGSTREGREF;
356
357
358/** Variable kinds. */
359typedef enum IEMNATIVEVARKIND : uint8_t
360{
361 /** Customary invalid zero value. */
362 kIemNativeVarKind_Invalid = 0,
363 /** This is either in a register or on the stack. */
364 kIemNativeVarKind_Stack,
365 /** Immediate value - loaded into register when needed, or can live on the
366 * stack if referenced (in theory). */
367 kIemNativeVarKind_Immediate,
368 /** Variable reference - loaded into register when needed, never stack. */
369 kIemNativeVarKind_VarRef,
370 /** Guest register reference - loaded into register when needed, never stack. */
371 kIemNativeVarKind_GstRegRef,
372 /** End of valid values. */
373 kIemNativeVarKind_End
374} IEMNATIVEVARKIND;
375
376
377/** Variable or argument. */
378typedef struct IEMNATIVEVAR
379{
380 /** The kind of variable. */
381 IEMNATIVEVARKIND enmKind;
382 /** The variable size in bytes. */
383 uint8_t cbVar;
384 /** The first stack slot (uint64_t), except for immediate and references
385 * where it usually is UINT8_MAX. */
386 uint8_t idxStackSlot;
387 /** The host register allocated for the variable, UINT8_MAX if not. */
388 uint8_t idxReg;
389 /** The argument number if argument, UINT8_MAX if regular variable. */
390 uint8_t uArgNo;
391 /** If referenced, the index of the variable referencing this one, otherwise
392 * UINT8_MAX. A referenced variable must only be placed on the stack and
393 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
394 uint8_t idxReferrerVar;
395 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
396 IEMNATIVEGSTREG enmGstReg;
397 uint8_t bAlign;
398
399 union
400 {
401 /** kIemNativeVarKind_Immediate: The immediate value. */
402 uint64_t uValue;
403 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
404 uint8_t idxRefVar;
405 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
406 struct
407 {
408 /** The class of register. */
409 IEMNATIVEGSTREGREF enmClass;
410 /** Index within the class. */
411 uint8_t idx;
412 } GstRegRef;
413 } u;
414} IEMNATIVEVAR;
415
416/** What is being kept in a host register. */
417typedef enum IEMNATIVEWHAT : uint8_t
418{
419 /** The traditional invalid zero value. */
420 kIemNativeWhat_Invalid = 0,
421 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
422 kIemNativeWhat_Var,
423 /** Temporary register, this is typically freed when a MC completes. */
424 kIemNativeWhat_Tmp,
425 /** Call argument w/o a variable mapping. This is free (via
426 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
427 kIemNativeWhat_Arg,
428 /** Return status code.
429 * @todo not sure if we need this... */
430 kIemNativeWhat_rc,
431 /** The fixed pVCpu (PVMCPUCC) register.
432 * @todo consider offsetting this on amd64 to use negative offsets to access
433 * more members using 8-byte disp. */
434 kIemNativeWhat_pVCpuFixed,
435 /** The fixed pCtx (PCPUMCTX) register.
436 * @todo consider offsetting this on amd64 to use negative offsets to access
437 * more members using 8-byte disp. */
438 kIemNativeWhat_pCtxFixed,
439 /** Fixed temporary register. */
440 kIemNativeWhat_FixedTmp,
441 /** Register reserved by the CPU or OS architecture. */
442 kIemNativeWhat_FixedReserved,
443 /** End of valid values. */
444 kIemNativeWhat_End
445} IEMNATIVEWHAT;
446
447/**
448 * Host general register entry.
449 *
450 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
451 *
452 * @todo Track immediate values in host registers similarlly to how we track the
453 * guest register shadow copies. For it to be real helpful, though,
454 * we probably need to know which will be reused and put them into
455 * non-volatile registers, otherwise it's going to be more or less
456 * restricted to an instruction or two.
457 */
458typedef struct IEMNATIVEHSTREG
459{
460 /** Set of guest registers this one shadows.
461 *
462 * Using a bitmap here so we can designate the same host register as a copy
463 * for more than one guest register. This is expected to be useful in
464 * situations where one value is copied to several registers in a sequence.
465 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
466 * sequence we'd want to let this register follow to be a copy of and there
467 * will always be places where we'd be picking the wrong one.
468 */
469 uint64_t fGstRegShadows;
470 /** What is being kept in this register. */
471 IEMNATIVEWHAT enmWhat;
472 /** Variable index if holding a variable, otherwise UINT8_MAX. */
473 uint8_t idxVar;
474 /** Alignment padding. */
475 uint8_t abAlign[6];
476} IEMNATIVEHSTREG;
477
478
479/**
480 * Core state for the native recompiler, that is, things that needs careful
481 * handling when dealing with branches.
482 */
483typedef struct IEMNATIVECORESTATE
484{
485 /** Allocation bitmap for aHstRegs. */
486 uint32_t bmHstRegs;
487
488 /** Bitmap marking which host register contains guest register shadow copies.
489 * This is used during register allocation to try preserve copies. */
490 uint32_t bmHstRegsWithGstShadow;
491 /** Bitmap marking valid entries in aidxGstRegShadows. */
492 uint64_t bmGstRegShadows;
493
494 union
495 {
496 /** Index of variable arguments, UINT8_MAX if not valid. */
497 uint8_t aidxArgVars[8];
498 /** For more efficient resetting. */
499 uint64_t u64ArgVars;
500 };
501
502 /** Allocation bitmap for aVars. */
503 uint32_t bmVars;
504
505 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
506 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
507 * (A shadow copy of a guest register can only be held in a one host register,
508 * there are no duplicate copies or ambiguities like that). */
509 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
510
511 /** Host register allocation tracking. */
512 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
513
514 /** Variables and arguments. */
515 IEMNATIVEVAR aVars[9];
516} IEMNATIVECORESTATE;
517/** Pointer to core state. */
518typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
519/** Pointer to const core state. */
520typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
521
522
523/**
524 * Conditional stack entry.
525 */
526typedef struct IEMNATIVECOND
527{
528 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
529 bool fInElse;
530 /** The label for the IEM_MC_ELSE. */
531 uint32_t idxLabelElse;
532 /** The label for the IEM_MC_ENDIF. */
533 uint32_t idxLabelEndIf;
534 /** The initial state snapshot as the if-block starts executing. */
535 IEMNATIVECORESTATE InitialState;
536 /** The state snapshot at the end of the if-block. */
537 IEMNATIVECORESTATE IfFinalState;
538} IEMNATIVECOND;
539/** Pointer to a condition stack entry. */
540typedef IEMNATIVECOND *PIEMNATIVECOND;
541
542
543/**
544 * Native recompiler state.
545 */
546typedef struct IEMRECOMPILERSTATE
547{
548 /** Size of the buffer that pbNativeRecompileBufR3 points to in
549 * IEMNATIVEINSTR units. */
550 uint32_t cInstrBufAlloc;
551#ifdef VBOX_STRICT
552 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
553 uint32_t offInstrBufChecked;
554#else
555 uint32_t uPadding1; /* We don't keep track of the size here... */
556#endif
557 /** Fixed temporary code buffer for native recompilation. */
558 PIEMNATIVEINSTR pInstrBuf;
559
560 /** Bitmaps with the label types used. */
561 uint64_t bmLabelTypes;
562 /** Actual number of labels in paLabels. */
563 uint32_t cLabels;
564 /** Max number of entries allowed in paLabels before reallocating it. */
565 uint32_t cLabelsAlloc;
566 /** Labels defined while recompiling (referenced by fixups). */
567 PIEMNATIVELABEL paLabels;
568
569 /** Actual number of fixups paFixups. */
570 uint32_t cFixups;
571 /** Max number of entries allowed in paFixups before reallocating it. */
572 uint32_t cFixupsAlloc;
573 /** Buffer used by the recompiler for recording fixups when generating code. */
574 PIEMNATIVEFIXUP paFixups;
575
576#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
577 /** Number of debug info entries allocated for pDbgInfo. */
578 uint32_t cDbgInfoAlloc;
579 uint32_t uPadding;
580 /** Debug info. */
581 PIEMTBDBG pDbgInfo;
582#endif
583
584 /** The translation block being recompiled. */
585 PCIEMTB pTbOrg;
586
587 /** The current condition stack depth (aCondStack). */
588 uint8_t cCondDepth;
589 uint8_t bPadding2;
590 /** Condition sequence number (for generating unique labels). */
591 uint16_t uCondSeqNo;
592 uint32_t uPadding3;
593
594 /** Core state requiring care with branches. */
595 IEMNATIVECORESTATE Core;
596
597 /** The condition nesting stack. */
598 IEMNATIVECOND aCondStack[2];
599} IEMRECOMPILERSTATE;
600/** Pointer to a native recompiler state. */
601typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
602
603
604/**
605 * Native recompiler worker for a threaded function.
606 *
607 * @returns New code buffer offset, UINT32_MAX in case of failure.
608 * @param pReNative The native recompiler state.
609 * @param off The current code buffer offset.
610 * @param pCallEntry The threaded call entry.
611 *
612 * @note This is not allowed to throw anything atm.
613 */
614typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
615 PCIEMTHRDEDCALLENTRY pCallEntry));
616/** Pointer to a native recompiler worker for a threaded function. */
617typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
618
619/** Defines a native recompiler worker for a threaded function. */
620#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
621 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
622/** Prototypes a native recompiler function for a threaded function. */
623#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
624
625
626DECLHIDDEN(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
627 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
628DECLHIDDEN(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere) RT_NOEXCEPT;
629DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
630 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
631DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
632 uint32_t cInstrReq) RT_NOEXCEPT;
633
634DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
635 bool fPreferVolatile = true) RT_NOEXCEPT;
636DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
637 bool fPreferVolatile = true) RT_NOEXCEPT;
638DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
639 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
640DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
641DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
642DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
643DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
644DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
645DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
646DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
647DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
648
649DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
650 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
651DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
652 uint8_t idxInstr) RT_NOEXCEPT;
653
654
655/**
656 * Ensures that there is sufficient space in the instruction output buffer.
657 *
658 * This will reallocate the buffer if needed and allowed.
659 *
660 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
661 * allocation size.
662 *
663 * @returns Pointer to the instruction output buffer on success, NULL on
664 * failure.
665 * @param pReNative The native recompile state.
666 * @param off Current instruction offset. Works safely for UINT32_MAX
667 * as well.
668 * @param cInstrReq Number of instruction about to be added. It's okay to
669 * overestimate this a bit.
670 */
671DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
672{
673 uint64_t const offChecked = off + (uint64_t)cInstrReq;
674 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
675 {
676#ifdef VBOX_STRICT
677 pReNative->offInstrBufChecked = offChecked;
678#endif
679 return pReNative->pInstrBuf;
680 }
681 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
682}
683
684/**
685 * Checks that we didn't exceed the space requested in the last
686 * iemNativeInstrBufEnsure() call. */
687#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
688 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
689 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
690
691
692/**
693 * Emit a simple marker instruction to more easily tell where something starts
694 * in the disassembly.
695 */
696DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
697{
698#ifdef RT_ARCH_AMD64
699 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
700 AssertReturn(pbCodeBuf, UINT32_MAX);
701 if (uInfo == 0)
702 {
703 /* nop */
704 pbCodeBuf[off++] = 0x90;
705 }
706 else
707 {
708 /* nop [disp32] */
709 pbCodeBuf[off++] = 0x0f;
710 pbCodeBuf[off++] = 0x1f;
711 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
712 pbCodeBuf[off++] = RT_BYTE1(uInfo);
713 pbCodeBuf[off++] = RT_BYTE2(uInfo);
714 pbCodeBuf[off++] = RT_BYTE3(uInfo);
715 pbCodeBuf[off++] = RT_BYTE4(uInfo);
716 }
717#elif RT_ARCH_ARM64
718 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
719 AssertReturn(pu32CodeBuf, UINT32_MAX);
720 /* nop */
721 pu32CodeBuf[off++] = 0xd503201f;
722
723 RT_NOREF(uInfo);
724#else
725# error "port me"
726#endif
727 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
728 return off;
729}
730
731
732/*********************************************************************************************************************************
733* Loads, Stores and Related Stuff. *
734*********************************************************************************************************************************/
735
736/**
737 * Emits setting a GPR to zero.
738 */
739DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
740{
741#ifdef RT_ARCH_AMD64
742 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
743 AssertReturn(pbCodeBuf, UINT32_MAX);
744 /* xor gpr32, gpr32 */
745 if (iGpr >= 8)
746 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
747 pbCodeBuf[off++] = 0x33;
748 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
749
750#elif RT_ARCH_ARM64
751 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
752 AssertReturn(pu32CodeBuf, UINT32_MAX);
753 /* mov gpr, #0x0 */
754 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
755
756#else
757# error "port me"
758#endif
759 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
760 return off;
761}
762
763
764/**
765 * Emits loading a constant into a 64-bit GPR
766 */
767DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
768{
769 if (!uImm64)
770 return iemNativeEmitGprZero(pReNative, off, iGpr);
771
772#ifdef RT_ARCH_AMD64
773 if (uImm64 <= UINT32_MAX)
774 {
775 /* mov gpr, imm32 */
776 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
777 AssertReturn(pbCodeBuf, UINT32_MAX);
778 if (iGpr >= 8)
779 pbCodeBuf[off++] = X86_OP_REX_B;
780 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
781 pbCodeBuf[off++] = RT_BYTE1(uImm64);
782 pbCodeBuf[off++] = RT_BYTE2(uImm64);
783 pbCodeBuf[off++] = RT_BYTE3(uImm64);
784 pbCodeBuf[off++] = RT_BYTE4(uImm64);
785 }
786 else
787 {
788 /* mov gpr, imm64 */
789 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
790 AssertReturn(pbCodeBuf, UINT32_MAX);
791 if (iGpr < 8)
792 pbCodeBuf[off++] = X86_OP_REX_W;
793 else
794 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
795 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
796 pbCodeBuf[off++] = RT_BYTE1(uImm64);
797 pbCodeBuf[off++] = RT_BYTE2(uImm64);
798 pbCodeBuf[off++] = RT_BYTE3(uImm64);
799 pbCodeBuf[off++] = RT_BYTE4(uImm64);
800 pbCodeBuf[off++] = RT_BYTE5(uImm64);
801 pbCodeBuf[off++] = RT_BYTE6(uImm64);
802 pbCodeBuf[off++] = RT_BYTE7(uImm64);
803 pbCodeBuf[off++] = RT_BYTE8(uImm64);
804 }
805
806#elif RT_ARCH_ARM64
807 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
808 AssertReturn(pu32CodeBuf, UINT32_MAX);
809
810 /*
811 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
812 * supply remaining bits using 'movk grp, imm16, lsl #x'.
813 *
814 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
815 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
816 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
817 * after the first non-zero immediate component so we switch to movk for
818 * the remainder.
819 */
820 uint32_t fMovK = 0;
821 /* mov gpr, imm16 */
822 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
823 if (uImmPart)
824 {
825 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
826 fMovK |= RT_BIT_32(29);
827 }
828 /* mov[k] gpr, imm16, lsl #16 */
829 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
830 if (uImmPart)
831 {
832 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
833 fMovK |= RT_BIT_32(29);
834 }
835 /* mov[k] gpr, imm16, lsl #32 */
836 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
837 if (uImmPart)
838 {
839 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
840 fMovK |= RT_BIT_32(29);
841 }
842 /* mov[k] gpr, imm16, lsl #48 */
843 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
844 if (uImmPart)
845 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
846
847 /** @todo there is an inverted mask variant we might want to explore if it
848 * reduces the number of instructions... */
849 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
850 * clang 12.x does that, only to use the 'x' version for the
851 * addressing in the following ldr). */
852
853#else
854# error "port me"
855#endif
856 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
857 return off;
858}
859
860
861/**
862 * Emits loading a constant into a 8-bit GPR
863 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
864 * only the ARM64 version does that.
865 */
866DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
867{
868#ifdef RT_ARCH_AMD64
869 /* mov gpr, imm8 */
870 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
871 AssertReturn(pbCodeBuf, UINT32_MAX);
872 if (iGpr >= 8)
873 pbCodeBuf[off++] = X86_OP_REX_B;
874 else if (iGpr >= 4)
875 pbCodeBuf[off++] = X86_OP_REX;
876 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
877 pbCodeBuf[off++] = RT_BYTE1(uImm8);
878
879#elif RT_ARCH_ARM64
880 /* movz gpr, imm16, lsl #0 */
881 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
882 AssertReturn(pu32CodeBuf, UINT32_MAX);
883 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
884
885#else
886# error "port me"
887#endif
888 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
889 return off;
890}
891
892
893#ifdef RT_ARCH_AMD64
894/**
895 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
896 */
897DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
898{
899 if (offVCpu < 128)
900 {
901 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
902 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
903 }
904 else
905 {
906 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
907 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
908 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
909 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
910 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
911 }
912 return off;
913}
914#elif RT_ARCH_ARM64
915/**
916 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
917 */
918DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
919 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
920{
921 /*
922 * There are a couple of ldr variants that takes an immediate offset, so
923 * try use those if we can, otherwise we have to use the temporary register
924 * help with the addressing.
925 */
926 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
927 {
928 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
929 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
930 AssertReturn(pu32CodeBuf, UINT32_MAX);
931 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
932 }
933 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
934 {
935 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
936 AssertReturn(pu32CodeBuf, UINT32_MAX);
937 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
938 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
939 }
940 else
941 {
942 /* The offset is too large, so we must load it into a register and use
943 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
944 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
945 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
946
947 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
948 AssertReturn(pu32CodeBuf, UINT32_MAX);
949 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
950 }
951 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
952 return off;
953}
954#endif
955
956
957/**
958 * Emits a 64-bit GPR load of a VCpu value.
959 */
960DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
961{
962#ifdef RT_ARCH_AMD64
963 /* mov reg64, mem64 */
964 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
965 AssertReturn(pbCodeBuf, UINT32_MAX);
966 if (iGpr < 8)
967 pbCodeBuf[off++] = X86_OP_REX_W;
968 else
969 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
970 pbCodeBuf[off++] = 0x8b;
971 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
972 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
973
974#elif RT_ARCH_ARM64
975 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
976
977#else
978# error "port me"
979#endif
980 return off;
981}
982
983
984/**
985 * Emits a 32-bit GPR load of a VCpu value.
986 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
987 */
988DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
989{
990#ifdef RT_ARCH_AMD64
991 /* mov reg32, mem32 */
992 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
993 AssertReturn(pbCodeBuf, UINT32_MAX);
994 if (iGpr >= 8)
995 pbCodeBuf[off++] = X86_OP_REX_R;
996 pbCodeBuf[off++] = 0x8b;
997 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
998 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
999
1000#elif RT_ARCH_ARM64
1001 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
1002
1003#else
1004# error "port me"
1005#endif
1006 return off;
1007}
1008
1009
1010/**
1011 * Emits a 16-bit GPR load of a VCpu value.
1012 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
1013 */
1014DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1015{
1016#ifdef RT_ARCH_AMD64
1017 /* movzx reg32, mem16 */
1018 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1019 AssertReturn(pbCodeBuf, UINT32_MAX);
1020 if (iGpr >= 8)
1021 pbCodeBuf[off++] = X86_OP_REX_R;
1022 pbCodeBuf[off++] = 0x0f;
1023 pbCodeBuf[off++] = 0xb7;
1024 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1025 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1026
1027#elif RT_ARCH_ARM64
1028 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1029
1030#else
1031# error "port me"
1032#endif
1033 return off;
1034}
1035
1036
1037/**
1038 * Emits a 8-bit GPR load of a VCpu value.
1039 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
1040 */
1041DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1042{
1043#ifdef RT_ARCH_AMD64
1044 /* movzx reg32, mem8 */
1045 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1046 AssertReturn(pbCodeBuf, UINT32_MAX);
1047 if (iGpr >= 8)
1048 pbCodeBuf[off++] = X86_OP_REX_R;
1049 pbCodeBuf[off++] = 0x0f;
1050 pbCodeBuf[off++] = 0xb6;
1051 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1052 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1053
1054#elif RT_ARCH_ARM64
1055 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1056
1057#else
1058# error "port me"
1059#endif
1060 return off;
1061}
1062
1063
1064/**
1065 * Emits a store of a GPR value to a 64-bit VCpu field.
1066 */
1067DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1068{
1069#ifdef RT_ARCH_AMD64
1070 /* mov mem64, reg64 */
1071 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1072 AssertReturn(pbCodeBuf, UINT32_MAX);
1073 if (iGpr < 8)
1074 pbCodeBuf[off++] = X86_OP_REX_W;
1075 else
1076 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1077 pbCodeBuf[off++] = 0x89;
1078 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1079 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1080
1081#elif RT_ARCH_ARM64
1082 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1083
1084#else
1085# error "port me"
1086#endif
1087 return off;
1088}
1089
1090
1091/**
1092 * Emits a store of a GPR value to a 32-bit VCpu field.
1093 */
1094DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1095{
1096#ifdef RT_ARCH_AMD64
1097 /* mov mem32, reg32 */
1098 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1099 AssertReturn(pbCodeBuf, UINT32_MAX);
1100 if (iGpr >= 8)
1101 pbCodeBuf[off++] = X86_OP_REX_R;
1102 pbCodeBuf[off++] = 0x89;
1103 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1104 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1105
1106#elif RT_ARCH_ARM64
1107 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1108
1109#else
1110# error "port me"
1111#endif
1112 return off;
1113}
1114
1115
1116/**
1117 * Emits a store of a GPR value to a 16-bit VCpu field.
1118 */
1119DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1120{
1121#ifdef RT_ARCH_AMD64
1122 /* mov mem16, reg16 */
1123 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1124 AssertReturn(pbCodeBuf, UINT32_MAX);
1125 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1126 if (iGpr >= 8)
1127 pbCodeBuf[off++] = X86_OP_REX_R;
1128 pbCodeBuf[off++] = 0x89;
1129 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1130 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1131
1132#elif RT_ARCH_ARM64
1133 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1134
1135#else
1136# error "port me"
1137#endif
1138 return off;
1139}
1140
1141
1142/**
1143 * Emits a store of a GPR value to a 8-bit VCpu field.
1144 */
1145DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1146{
1147#ifdef RT_ARCH_AMD64
1148 /* mov mem8, reg8 */
1149 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1150 AssertReturn(pbCodeBuf, UINT32_MAX);
1151 if (iGpr >= 8)
1152 pbCodeBuf[off++] = X86_OP_REX_R;
1153 pbCodeBuf[off++] = 0x88;
1154 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1155 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1156
1157#elif RT_ARCH_ARM64
1158 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1159
1160#else
1161# error "port me"
1162#endif
1163 return off;
1164}
1165
1166
1167/**
1168 * Emits a gprdst = gprsrc load.
1169 */
1170DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1171{
1172#ifdef RT_ARCH_AMD64
1173 /* mov gprdst, gprsrc */
1174 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1175 AssertReturn(pbCodeBuf, UINT32_MAX);
1176 if ((iGprDst | iGprSrc) >= 8)
1177 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1178 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1179 : X86_OP_REX_W | X86_OP_REX_R;
1180 else
1181 pbCodeBuf[off++] = X86_OP_REX_W;
1182 pbCodeBuf[off++] = 0x8b;
1183 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1184
1185#elif RT_ARCH_ARM64
1186 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1187 AssertReturn(pu32CodeBuf, UINT32_MAX);
1188 /* mov dst, src; alias for: orr dst, xzr, src */
1189 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1190
1191#else
1192# error "port me"
1193#endif
1194 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1195 return off;
1196}
1197
1198#ifdef RT_ARCH_AMD64
1199/**
1200 * Common bit of iemNativeEmitLoadGprByBp and friends.
1201 */
1202DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1203 PIEMRECOMPILERSTATE pReNativeAssert)
1204{
1205 if (offDisp < 128 && offDisp >= -128)
1206 {
1207 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1208 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1209 }
1210 else
1211 {
1212 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1213 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1214 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1215 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1216 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1217 }
1218 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1219 return off;
1220}
1221#endif
1222
1223
1224#ifdef RT_ARCH_AMD64
1225/**
1226 * Emits a 64-bit GRP load instruction with an BP relative source address.
1227 */
1228DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1229{
1230 /* mov gprdst, qword [rbp + offDisp] */
1231 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1232 AssertReturn(pbCodeBuf, UINT32_MAX);
1233 if (iGprDst < 8)
1234 pbCodeBuf[off++] = X86_OP_REX_W;
1235 else
1236 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1237 pbCodeBuf[off++] = 0x8b;
1238 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1239}
1240#endif
1241
1242
1243#ifdef RT_ARCH_AMD64
1244/**
1245 * Emits a 32-bit GRP load instruction with an BP relative source address.
1246 */
1247DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1248{
1249 /* mov gprdst, dword [rbp + offDisp] */
1250 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1251 AssertReturn(pbCodeBuf, UINT32_MAX);
1252 if (iGprDst >= 8)
1253 pbCodeBuf[off++] = X86_OP_REX_R;
1254 pbCodeBuf[off++] = 0x8b;
1255 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1256}
1257#endif
1258
1259
1260#ifdef RT_ARCH_AMD64
1261/**
1262 * Emits a load effective address to a GRP with an BP relative source address.
1263 */
1264DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1265{
1266 /* lea gprdst, [rbp + offDisp] */
1267 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1268 AssertReturn(pbCodeBuf, UINT32_MAX);
1269 if (iGprDst < 8)
1270 pbCodeBuf[off++] = X86_OP_REX_W;
1271 else
1272 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1273 pbCodeBuf[off++] = 0x8d;
1274 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1275}
1276#endif
1277
1278
1279/**
1280 * Emits a 64-bit GPR store with an BP relative destination address.
1281 *
1282 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1283 */
1284DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1285{
1286#ifdef RT_ARCH_AMD64
1287 /* mov qword [rbp + offDisp], gprdst */
1288 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1289 AssertReturn(pbCodeBuf, UINT32_MAX);
1290 if (iGprSrc < 8)
1291 pbCodeBuf[off++] = X86_OP_REX_W;
1292 else
1293 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1294 pbCodeBuf[off++] = 0x89;
1295 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1296
1297#elif defined(RT_ARCH_ARM64)
1298 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1299 {
1300 /* str w/ unsigned imm12 (scaled) */
1301 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1302 AssertReturn(pu32CodeBuf, UINT32_MAX);
1303 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1304 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1305 }
1306 else if (offDisp >= -256 && offDisp <= 256)
1307 {
1308 /* stur w/ signed imm9 (unscaled) */
1309 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1310 AssertReturn(pu32CodeBuf, UINT32_MAX);
1311 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1312 }
1313 else
1314 {
1315 /* Use temporary indexing register. */
1316 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1317 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1318 AssertReturn(pu32CodeBuf, UINT32_MAX);
1319 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1320 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1321 }
1322 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1323 return off;
1324
1325#else
1326# error "Port me!"
1327#endif
1328}
1329
1330
1331/**
1332 * Emits a 64-bit immediate store with an BP relative destination address.
1333 *
1334 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1335 */
1336DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1337{
1338#ifdef RT_ARCH_AMD64
1339 if ((int64_t)uImm64 == (int32_t)uImm64)
1340 {
1341 /* mov qword [rbp + offDisp], imm32 - sign extended */
1342 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1343 AssertReturn(pbCodeBuf, UINT32_MAX);
1344
1345 pbCodeBuf[off++] = X86_OP_REX_W;
1346 pbCodeBuf[off++] = 0xc7;
1347 if (offDisp < 128 && offDisp >= -128)
1348 {
1349 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1350 pbCodeBuf[off++] = (uint8_t)offDisp;
1351 }
1352 else
1353 {
1354 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1355 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1356 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1357 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1358 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1359 }
1360 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1361 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1362 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1363 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1364 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1365 return off;
1366 }
1367#endif
1368
1369 /* Load tmp0, imm64; Store tmp to bp+disp. */
1370 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1371 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1372}
1373
1374
1375/*********************************************************************************************************************************
1376* Subtraction and Additions *
1377*********************************************************************************************************************************/
1378
1379
1380#ifdef RT_ARCH_AMD64
1381/**
1382 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1383 */
1384DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1385{
1386 /* sub gprdst, imm8/imm32 */
1387 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1388 AssertReturn(pbCodeBuf, UINT32_MAX);
1389 if (iGprDst < 8)
1390 pbCodeBuf[off++] = X86_OP_REX_W;
1391 else
1392 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1393 if (iSubtrahend < 128 && iSubtrahend >= -128)
1394 {
1395 pbCodeBuf[off++] = 0x83;
1396 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1397 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1398 }
1399 else
1400 {
1401 pbCodeBuf[off++] = 0x81;
1402 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1403 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1404 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1405 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1406 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1407 }
1408 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1409 return off;
1410}
1411#endif
1412
1413
1414/**
1415 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1416 * @note The AMD64 version sets flags.
1417 */
1418DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1419{
1420#if defined(RT_ARCH_AMD64)
1421 /* add Gv,Ev */
1422 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1423 AssertReturn(pbCodeBuf, UINT32_MAX);
1424 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1425 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1426 pbCodeBuf[off++] = 0x04;
1427 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1428
1429#elif defined(RT_ARCH_ARM64)
1430 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1431 AssertReturn(pu32CodeBuf, UINT32_MAX);
1432 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1433
1434#else
1435# error "Port me"
1436#endif
1437 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1438 return off;
1439}
1440
1441
1442/**
1443 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1444 */
1445DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1446{
1447#if defined(RT_ARCH_AMD64)
1448 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1449 AssertReturn(pbCodeBuf, UINT32_MAX);
1450 /* add or inc */
1451 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1452 if (iImm8 != 1)
1453 {
1454 pbCodeBuf[off++] = 0x83;
1455 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1456 pbCodeBuf[off++] = (uint8_t)iImm8;
1457 }
1458 else
1459 {
1460 pbCodeBuf[off++] = 0xff;
1461 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1462 }
1463
1464#elif defined(RT_ARCH_ARM64)
1465 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1466 AssertReturn(pu32CodeBuf, UINT32_MAX);
1467 if (iImm8 >= 0)
1468 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1469 else
1470 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1471
1472#else
1473# error "Port me"
1474#endif
1475 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1476 return off;
1477}
1478
1479
1480/**
1481 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1482 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1483 */
1484DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1485{
1486#if defined(RT_ARCH_AMD64)
1487 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1488 AssertReturn(pbCodeBuf, UINT32_MAX);
1489 /* add or inc */
1490 if (iGprDst >= 8)
1491 pbCodeBuf[off++] = X86_OP_REX_B;
1492 if (iImm8 != 1)
1493 {
1494 pbCodeBuf[off++] = 0x83;
1495 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1496 pbCodeBuf[off++] = (uint8_t)iImm8;
1497 }
1498 else
1499 {
1500 pbCodeBuf[off++] = 0xff;
1501 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1502 }
1503
1504#elif defined(RT_ARCH_ARM64)
1505 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1506 AssertReturn(pu32CodeBuf, UINT32_MAX);
1507 if (iImm8 >= 0)
1508 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1509 else
1510 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1511
1512#else
1513# error "Port me"
1514#endif
1515 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1516 return off;
1517}
1518
1519
1520/**
1521 * Emits a 64-bit GPR additions with a 64-bit signed addend.
1522 */
1523DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1524{
1525#if defined(RT_ARCH_AMD64)
1526 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1527 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1528
1529 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1530 {
1531 /* add grp, imm32 */
1532 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1533 AssertReturn(pbCodeBuf, UINT32_MAX);
1534 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1535 pbCodeBuf[off++] = 0x81;
1536 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1537 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1538 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1539 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1540 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1541 }
1542 else
1543 {
1544 /* Best to use a temporary register to deal with this in the simplest way: */
1545 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1546 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1547
1548 /* add dst, tmpreg */
1549 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1550 AssertReturn(pbCodeBuf, UINT32_MAX);
1551 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1552 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1553 pbCodeBuf[off++] = 0x03;
1554 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1555
1556 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1557 }
1558
1559#elif defined(RT_ARCH_ARM64)
1560 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1561 {
1562 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1563 AssertReturn(pu32CodeBuf, UINT32_MAX);
1564 if (iAddend >= 0)
1565 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
1566 else
1567 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
1568 }
1569 else
1570 {
1571 /* Use temporary register for the immediate. */
1572 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1573 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1574
1575 /* add gprdst, gprdst, tmpreg */
1576 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1577 AssertReturn(pu32CodeBuf, UINT32_MAX);
1578 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
1579
1580 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1581 }
1582
1583#else
1584# error "Port me"
1585#endif
1586 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1587 return off;
1588}
1589
1590
1591/**
1592 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
1593 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1594 */
1595DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1596{
1597#if defined(RT_ARCH_AMD64)
1598 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1599 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1600
1601 /* add grp, imm32 */
1602 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1603 AssertReturn(pbCodeBuf, UINT32_MAX);
1604 if (iGprDst >= 8)
1605 pbCodeBuf[off++] = X86_OP_REX_B;
1606 pbCodeBuf[off++] = 0x81;
1607 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1608 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1609 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1610 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1611 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1612
1613#elif defined(RT_ARCH_ARM64)
1614 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1615 {
1616 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1617 AssertReturn(pu32CodeBuf, UINT32_MAX);
1618 if (iAddend >= 0)
1619 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
1620 else
1621 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
1622 }
1623 else
1624 {
1625 /* Use temporary register for the immediate. */
1626 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1627 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
1628
1629 /* add gprdst, gprdst, tmpreg */
1630 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1631 AssertReturn(pu32CodeBuf, UINT32_MAX);
1632 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
1633
1634 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1635 }
1636
1637#else
1638# error "Port me"
1639#endif
1640 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1641 return off;
1642}
1643
1644
1645
1646/*********************************************************************************************************************************
1647* Bit Operations *
1648*********************************************************************************************************************************/
1649
1650/**
1651 * Emits code for clearing bits 16 thru 63 in the GPR.
1652 */
1653DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1654{
1655#if defined(RT_ARCH_AMD64)
1656 /* movzx reg32, reg16 */
1657 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1658 AssertReturn(pbCodeBuf, UINT32_MAX);
1659 if (iGprDst >= 8)
1660 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
1661 pbCodeBuf[off++] = 0x0f;
1662 pbCodeBuf[off++] = 0xb7;
1663 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1664
1665#elif defined(RT_ARCH_ARM64)
1666 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1667 AssertReturn(pu32CodeBuf, UINT32_MAX);
1668# if 1
1669 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1670# else
1671 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1672 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1673# endif
1674#else
1675# error "Port me"
1676#endif
1677 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1678 return off;
1679}
1680
1681
1682/**
1683 * Emits code for AND'ing two 64-bit GPRs.
1684 */
1685DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1686{
1687#if defined(RT_ARCH_AMD64)
1688 /* and Gv, Ev */
1689 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1690 AssertReturn(pbCodeBuf, UINT32_MAX);
1691 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1692 pbCodeBuf[off++] = 0x23;
1693 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1694
1695#elif defined(RT_ARCH_ARM64)
1696 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1697 AssertReturn(pu32CodeBuf, UINT32_MAX);
1698 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
1699
1700#else
1701# error "Port me"
1702#endif
1703 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1704 return off;
1705}
1706
1707
1708/**
1709 * Emits code for AND'ing two 32-bit GPRs.
1710 */
1711DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1712{
1713#if defined(RT_ARCH_AMD64)
1714 /* and Gv, Ev */
1715 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1716 AssertReturn(pbCodeBuf, UINT32_MAX);
1717 if (iGprDst >= 8 || iGprSrc >= 8)
1718 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1719 pbCodeBuf[off++] = 0x23;
1720 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1721
1722#elif defined(RT_ARCH_ARM64)
1723 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1724 AssertReturn(pu32CodeBuf, UINT32_MAX);
1725 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
1726
1727#else
1728# error "Port me"
1729#endif
1730 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1731 return off;
1732}
1733
1734
1735/**
1736 * Emits code for XOR'ing two 64-bit GPRs.
1737 */
1738DECLINLINE(uint32_t ) iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1739{
1740#if defined(RT_ARCH_AMD64)
1741 /* and Gv, Ev */
1742 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1743 AssertReturn(pbCodeBuf, UINT32_MAX);
1744 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1745 pbCodeBuf[off++] = 0x33;
1746 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1747
1748#elif defined(RT_ARCH_ARM64)
1749 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1750 AssertReturn(pu32CodeBuf, UINT32_MAX);
1751 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);
1752
1753#else
1754# error "Port me"
1755#endif
1756 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1757 return off;
1758}
1759
1760
1761/**
1762 * Emits code for XOR'ing two 32-bit GPRs.
1763 */
1764DECLINLINE(uint32_t ) iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1765{
1766#if defined(RT_ARCH_AMD64)
1767 /* and Gv, Ev */
1768 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1769 AssertReturn(pbCodeBuf, UINT32_MAX);
1770 if (iGprDst >= 8 || iGprSrc >= 8)
1771 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
1772 pbCodeBuf[off++] = 0x33;
1773 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1774
1775#elif defined(RT_ARCH_ARM64)
1776 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1777 AssertReturn(pu32CodeBuf, UINT32_MAX);
1778 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
1779
1780#else
1781# error "Port me"
1782#endif
1783 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1784 return off;
1785}
1786
1787
1788/*********************************************************************************************************************************
1789* Shifting *
1790*********************************************************************************************************************************/
1791
1792/**
1793 * Emits code for shifting a GPR a fixed number of bits to the left.
1794 */
1795DECLINLINE(uint32_t ) iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1796{
1797 Assert(cShift > 0 && cShift < 64);
1798
1799#if defined(RT_ARCH_AMD64)
1800 /* shl dst, cShift */
1801 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1802 AssertReturn(pbCodeBuf, UINT32_MAX);
1803 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1804 if (cShift != 1)
1805 {
1806 pbCodeBuf[off++] = 0xc1;
1807 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1808 pbCodeBuf[off++] = cShift;
1809 }
1810 else
1811 {
1812 pbCodeBuf[off++] = 0xd1;
1813 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1814 }
1815
1816#elif defined(RT_ARCH_ARM64)
1817 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1818 AssertReturn(pu32CodeBuf, UINT32_MAX);
1819 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
1820
1821#else
1822# error "Port me"
1823#endif
1824 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1825 return off;
1826}
1827
1828
1829/**
1830 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
1831 */
1832DECLINLINE(uint32_t ) iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1833{
1834 Assert(cShift > 0 && cShift < 32);
1835
1836#if defined(RT_ARCH_AMD64)
1837 /* shl dst, cShift */
1838 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1839 AssertReturn(pbCodeBuf, UINT32_MAX);
1840 if (iGprDst >= 8)
1841 pbCodeBuf[off++] = X86_OP_REX_B;
1842 if (cShift != 1)
1843 {
1844 pbCodeBuf[off++] = 0xc1;
1845 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1846 pbCodeBuf[off++] = cShift;
1847 }
1848 else
1849 {
1850 pbCodeBuf[off++] = 0xd1;
1851 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
1852 }
1853
1854#elif defined(RT_ARCH_ARM64)
1855 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1856 AssertReturn(pu32CodeBuf, UINT32_MAX);
1857 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
1858
1859#else
1860# error "Port me"
1861#endif
1862 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1863 return off;
1864}
1865
1866
1867/**
1868 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1869 */
1870DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1871{
1872 Assert(cShift > 0 && cShift < 64);
1873
1874#if defined(RT_ARCH_AMD64)
1875 /* shr dst, cShift */
1876 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1877 AssertReturn(pbCodeBuf, UINT32_MAX);
1878 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1879 if (cShift != 1)
1880 {
1881 pbCodeBuf[off++] = 0xc1;
1882 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1883 pbCodeBuf[off++] = cShift;
1884 }
1885 else
1886 {
1887 pbCodeBuf[off++] = 0xd1;
1888 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1889 }
1890
1891#elif defined(RT_ARCH_ARM64)
1892 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1893 AssertReturn(pu32CodeBuf, UINT32_MAX);
1894 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1895
1896#else
1897# error "Port me"
1898#endif
1899 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1900 return off;
1901}
1902
1903
1904/**
1905 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
1906 * right.
1907 */
1908DECLINLINE(uint32_t ) iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1909{
1910 Assert(cShift > 0 && cShift < 32);
1911
1912#if defined(RT_ARCH_AMD64)
1913 /* shr dst, cShift */
1914 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1915 AssertReturn(pbCodeBuf, UINT32_MAX);
1916 if (iGprDst >= 8)
1917 pbCodeBuf[off++] = X86_OP_REX_B;
1918 if (cShift != 1)
1919 {
1920 pbCodeBuf[off++] = 0xc1;
1921 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1922 pbCodeBuf[off++] = cShift;
1923 }
1924 else
1925 {
1926 pbCodeBuf[off++] = 0xd1;
1927 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1928 }
1929
1930#elif defined(RT_ARCH_ARM64)
1931 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1932 AssertReturn(pu32CodeBuf, UINT32_MAX);
1933 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
1934
1935#else
1936# error "Port me"
1937#endif
1938 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1939 return off;
1940}
1941
1942
1943
1944/*********************************************************************************************************************************
1945* Compare and Testing *
1946*********************************************************************************************************************************/
1947
1948
1949#ifdef RT_ARCH_ARM64
1950/**
1951 * Emits an ARM64 compare instruction.
1952 */
1953DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1954 bool f64Bit = true, uint32_t cShift = 0,
1955 ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1956{
1957 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1958 AssertReturn(pu32CodeBuf, UINT32_MAX);
1959 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
1960 f64Bit, true /*fSetFlags*/, cShift, enmShift);
1961 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1962 return off;
1963}
1964#endif
1965
1966
1967/**
1968 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1969 * with conditional instruction.
1970 */
1971DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1972{
1973#ifdef RT_ARCH_AMD64
1974 /* cmp Gv, Ev */
1975 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1976 AssertReturn(pbCodeBuf, UINT32_MAX);
1977 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1978 pbCodeBuf[off++] = 0x3b;
1979 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1980 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1981
1982#elif defined(RT_ARCH_ARM64)
1983 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1984
1985#else
1986# error "Port me!"
1987#endif
1988 return off;
1989}
1990
1991
1992/**
1993 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1994 * with conditional instruction.
1995 */
1996DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1997 uint8_t iGprLeft, uint8_t iGprRight)
1998{
1999#ifdef RT_ARCH_AMD64
2000 /* cmp Gv, Ev */
2001 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2002 AssertReturn(pbCodeBuf, UINT32_MAX);
2003 if (iGprLeft >= 8 || iGprRight >= 8)
2004 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
2005 pbCodeBuf[off++] = 0x3b;
2006 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
2007 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2008
2009#elif defined(RT_ARCH_ARM64)
2010 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
2011
2012#else
2013# error "Port me!"
2014#endif
2015 return off;
2016}
2017
2018
2019
2020/*********************************************************************************************************************************
2021* Branching *
2022*********************************************************************************************************************************/
2023
2024/**
2025 * Emits a JMP rel32 / B imm19 to the given label.
2026 */
2027DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2028{
2029 Assert(idxLabel < pReNative->cLabels);
2030
2031#ifdef RT_ARCH_AMD64
2032 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2033 AssertReturn(pbCodeBuf, UINT32_MAX);
2034 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2035 {
2036 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
2037 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
2038 {
2039 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
2040 pbCodeBuf[off++] = (uint8_t)offRel;
2041 }
2042 else
2043 {
2044 offRel -= 3;
2045 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2046 pbCodeBuf[off++] = RT_BYTE1(offRel);
2047 pbCodeBuf[off++] = RT_BYTE2(offRel);
2048 pbCodeBuf[off++] = RT_BYTE3(offRel);
2049 pbCodeBuf[off++] = RT_BYTE4(offRel);
2050 }
2051 }
2052 else
2053 {
2054 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
2055 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
2056 pbCodeBuf[off++] = 0xfe;
2057 pbCodeBuf[off++] = 0xff;
2058 pbCodeBuf[off++] = 0xff;
2059 pbCodeBuf[off++] = 0xff;
2060 }
2061 pbCodeBuf[off++] = 0xcc; /* int3 poison */
2062
2063#elif defined(RT_ARCH_ARM64)
2064 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2065 AssertReturn(pu32CodeBuf, UINT32_MAX);
2066 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
2067 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
2068 else
2069 {
2070 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
2071 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
2072 }
2073
2074#else
2075# error "Port me!"
2076#endif
2077 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2078 return off;
2079}
2080
2081
2082/**
2083 * Emits a JMP rel32 / B imm19 to a new undefined label.
2084 */
2085DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2086 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2087{
2088 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2089 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
2090 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2091}
2092
2093/** Condition type. */
2094#ifdef RT_ARCH_AMD64
2095typedef enum IEMNATIVEINSTRCOND : uint8_t
2096{
2097 kIemNativeInstrCond_o = 0,
2098 kIemNativeInstrCond_no,
2099 kIemNativeInstrCond_c,
2100 kIemNativeInstrCond_nc,
2101 kIemNativeInstrCond_e,
2102 kIemNativeInstrCond_ne,
2103 kIemNativeInstrCond_be,
2104 kIemNativeInstrCond_nbe,
2105 kIemNativeInstrCond_s,
2106 kIemNativeInstrCond_ns,
2107 kIemNativeInstrCond_p,
2108 kIemNativeInstrCond_np,
2109 kIemNativeInstrCond_l,
2110 kIemNativeInstrCond_nl,
2111 kIemNativeInstrCond_le,
2112 kIemNativeInstrCond_nle
2113} IEMNATIVEINSTRCOND;
2114#elif defined(RT_ARCH_ARM64)
2115typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
2116#else
2117# error "Port me!"
2118#endif
2119
2120
2121/**
2122 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
2123 */
2124DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2125 uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
2126{
2127 Assert(idxLabel < pReNative->cLabels);
2128
2129#ifdef RT_ARCH_AMD64
2130 /* jcc rel32 */
2131 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2132 AssertReturn(pbCodeBuf, UINT32_MAX);
2133 pbCodeBuf[off++] = 0x0f;
2134 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2135 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
2136 pbCodeBuf[off++] = 0x00;
2137 pbCodeBuf[off++] = 0x00;
2138 pbCodeBuf[off++] = 0x00;
2139 pbCodeBuf[off++] = 0x00;
2140
2141#elif defined(RT_ARCH_ARM64)
2142 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2143 AssertReturn(pu32CodeBuf, UINT32_MAX);
2144 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
2145 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
2146
2147#else
2148# error "Port me!"
2149#endif
2150 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2151 return off;
2152}
2153
2154
2155/**
2156 * Emits a Jcc rel32 / B.cc imm19 to a new label.
2157 */
2158DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2159 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
2160{
2161 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2162 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
2163 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
2164}
2165
2166
2167/**
2168 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
2169 */
2170DECLINLINE(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2171{
2172#ifdef RT_ARCH_AMD64
2173 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
2174#elif defined(RT_ARCH_ARM64)
2175 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
2176#else
2177# error "Port me!"
2178#endif
2179}
2180
2181/**
2182 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
2183 */
2184DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2185 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2186{
2187#ifdef RT_ARCH_AMD64
2188 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
2189#elif defined(RT_ARCH_ARM64)
2190 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
2191#else
2192# error "Port me!"
2193#endif
2194}
2195
2196
2197/**
2198 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
2199 */
2200DECLINLINE(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2201{
2202#ifdef RT_ARCH_AMD64
2203 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
2204#elif defined(RT_ARCH_ARM64)
2205 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
2206#else
2207# error "Port me!"
2208#endif
2209}
2210
2211/**
2212 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
2213 */
2214DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2215 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2216{
2217#ifdef RT_ARCH_AMD64
2218 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
2219#elif defined(RT_ARCH_ARM64)
2220 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
2221#else
2222# error "Port me!"
2223#endif
2224}
2225
2226
2227/**
2228 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
2229 */
2230DECLINLINE(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2231{
2232#ifdef RT_ARCH_AMD64
2233 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
2234#elif defined(RT_ARCH_ARM64)
2235 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
2236#else
2237# error "Port me!"
2238#endif
2239}
2240
2241/**
2242 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
2243 */
2244DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2245 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2246{
2247#ifdef RT_ARCH_AMD64
2248 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
2249#elif defined(RT_ARCH_ARM64)
2250 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
2251#else
2252# error "Port me!"
2253#endif
2254}
2255
2256
2257/**
2258 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2259 */
2260DECLINLINE(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2261{
2262#ifdef RT_ARCH_AMD64
2263 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2264#elif defined(RT_ARCH_ARM64)
2265 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2266#else
2267# error "Port me!"
2268#endif
2269}
2270
2271/**
2272 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2273 */
2274DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2275 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2276{
2277#ifdef RT_ARCH_AMD64
2278 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2279#elif defined(RT_ARCH_ARM64)
2280 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2281#else
2282# error "Port me!"
2283#endif
2284}
2285
2286
2287/**
2288 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2289 * How @a offJmp is applied is are target specific.
2290 */
2291DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2292 int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2293{
2294#ifdef RT_ARCH_AMD64
2295 /* jcc rel32 */
2296 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2297 AssertReturn(pbCodeBuf, UINT32_MAX);
2298 if (offTarget < 128 && offTarget >= -128)
2299 {
2300 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
2301 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2302 }
2303 else
2304 {
2305 pbCodeBuf[off++] = 0x0f;
2306 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2307 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2308 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2309 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2310 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2311 }
2312
2313#elif defined(RT_ARCH_ARM64)
2314 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2315 AssertReturn(pu32CodeBuf, UINT32_MAX);
2316 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2317
2318#else
2319# error "Port me!"
2320#endif
2321 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2322 return off;
2323}
2324
2325
2326/**
2327 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2328 * How @a offJmp is applied is are target specific.
2329 */
2330DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2331{
2332#ifdef RT_ARCH_AMD64
2333 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2334#elif defined(RT_ARCH_ARM64)
2335 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2336#else
2337# error "Port me!"
2338#endif
2339}
2340
2341
2342/**
2343 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2344 * How @a offJmp is applied is are target specific.
2345 */
2346DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2347{
2348#ifdef RT_ARCH_AMD64
2349 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2350#elif defined(RT_ARCH_ARM64)
2351 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2352#else
2353# error "Port me!"
2354#endif
2355}
2356
2357
2358/**
2359 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2360 * How @a offJmp is applied is are target specific.
2361 */
2362DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2363{
2364#ifdef RT_ARCH_AMD64
2365 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2366#elif defined(RT_ARCH_ARM64)
2367 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2368#else
2369# error "Port me!"
2370#endif
2371}
2372
2373
2374/**
2375 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2376 * How @a offJmp is applied is are target specific.
2377 */
2378DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2379{
2380#ifdef RT_ARCH_AMD64
2381 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2382#elif defined(RT_ARCH_ARM64)
2383 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2384#else
2385# error "Port me!"
2386#endif
2387}
2388
2389
2390/**
2391 * Fixes up a conditional jump to a fixed label.
2392 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2393 */
2394DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2395{
2396# if defined(RT_ARCH_AMD64)
2397 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2398 if (pbCodeBuf[offFixup] != 0x0f)
2399 {
2400 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2401 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2402 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2403 }
2404 else
2405 {
2406 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2407 uint32_t const offRel32 = offTarget - (offFixup + 6);
2408 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2409 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2410 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2411 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2412 }
2413
2414# elif defined(RT_ARCH_ARM64)
2415 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2416 Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
2417 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
2418 | (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
2419
2420# endif
2421}
2422
2423
2424/**
2425 * Internal helper, don't call directly.
2426 */
2427DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2428 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel,
2429 bool fJmpIfSet)
2430{
2431 Assert(iBitNo < 64);
2432#ifdef RT_ARCH_AMD64
2433 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2434 AssertReturn(pbCodeBuf, UINT32_MAX);
2435 if (iBitNo < 8)
2436 {
2437 /* test Eb, imm8 */
2438 if (iGprSrc >= 8)
2439 pbCodeBuf[off++] = X86_OP_REX_B;
2440 pbCodeBuf[off++] = 0xf6;
2441 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2442 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2443 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2444 }
2445 else
2446 {
2447 /* bt Ev, imm8 */
2448 if (iBitNo >= 32)
2449 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2450 else if (iGprSrc >= 8)
2451 pbCodeBuf[off++] = X86_OP_REX_B;
2452 pbCodeBuf[off++] = 0x0f;
2453 pbCodeBuf[off++] = 0xba;
2454 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2455 pbCodeBuf[off++] = iBitNo;
2456 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2457 }
2458
2459#elif defined(RT_ARCH_ARM64)
2460 /* Use the TBNZ instruction here. */
2461 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2462 AssertReturn(pu32CodeBuf, UINT32_MAX);
2463 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5), UINT32_MAX);
2464 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2465
2466#else
2467# error "Port me!"
2468#endif
2469 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2470 return off;
2471}
2472
2473
2474/**
2475 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
2476 * @a iGprSrc.
2477 *
2478 * @note On ARM64 the range is only +/-8191 instructions.
2479 */
2480DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2481 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2482{
2483 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
2484}
2485
2486
2487/**
2488 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
2489 * _set_ in @a iGprSrc.
2490 *
2491 * @note On ARM64 the range is only +/-8191 instructions.
2492 */
2493DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2494 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2495{
2496 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
2497}
2498
2499
2500/**
2501 * Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU
2502 * flags accordingly.
2503 */
2504DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
2505{
2506 Assert(fBits != 0);
2507#ifdef RT_ARCH_AMD64
2508
2509 if (fBits >= UINT32_MAX)
2510 {
2511 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2512 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
2513
2514 /* test Ev,Gv */
2515 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2516 AssertReturn(pbCodeBuf, UINT32_MAX);
2517 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R) | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
2518 pbCodeBuf[off++] = 0x85;
2519 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);
2520
2521 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2522 }
2523 else
2524 {
2525 /* test Eb, imm8 or test Ev, imm32 */
2526 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
2527 AssertReturn(pbCodeBuf, UINT32_MAX);
2528 if (iGprSrc >= 8)
2529 pbCodeBuf[off++] = X86_OP_REX_B;
2530 if (fBits <= UINT8_MAX)
2531 {
2532 pbCodeBuf[off++] = 0xf6;
2533 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2534 pbCodeBuf[off++] = (uint8_t)fBits;
2535 }
2536 else
2537 {
2538 pbCodeBuf[off++] = 0xf7;
2539 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2540 pbCodeBuf[off++] = RT_BYTE1(fBits);
2541 pbCodeBuf[off++] = RT_BYTE2(fBits);
2542 pbCodeBuf[off++] = RT_BYTE3(fBits);
2543 pbCodeBuf[off++] = RT_BYTE4(fBits);
2544 }
2545 }
2546
2547#elif defined(RT_ARCH_ARM64)
2548
2549 if (false)
2550 {
2551 /** @todo figure out how to work the immr / N:imms constants. */
2552 }
2553 else
2554 {
2555 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);
2556 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
2557
2558 /* ands Zr, iGprSrc, iTmpReg */
2559 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2560 AssertReturn(pu32CodeBuf, UINT32_MAX);
2561 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);
2562
2563 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
2564 }
2565
2566#else
2567# error "Port me!"
2568#endif
2569 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2570 return off;
2571}
2572
2573
2574/**
2575 * Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
2576 * are set in @a iGprSrc.
2577 */
2578DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2579 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2580{
2581 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2582
2583 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2584 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
2585
2586 return off;
2587}
2588
2589
2590/**
2591 * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
2592 * are set in @a iGprSrc.
2593 */
2594DECLINLINE(uint32_t) iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2595 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)
2596{
2597 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
2598
2599 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
2600 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2601
2602 return off;
2603}
2604
2605
2606/**
2607 * Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
2608 *
2609 * The operand size is given by @a f64Bit.
2610 */
2611DECLINLINE(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2612 uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
2613{
2614 Assert(idxLabel < pReNative->cLabels);
2615
2616#ifdef RT_ARCH_AMD64
2617 /* test reg32,reg32 / test reg64,reg64 */
2618 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
2619 AssertReturn(pbCodeBuf, UINT32_MAX);
2620 if (f64Bit)
2621 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
2622 else if (iGprSrc >= 8)
2623 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
2624 pbCodeBuf[off++] = 0x85;
2625 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
2626 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2627
2628 /* jz idxLabel */
2629 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
2630
2631#elif defined(RT_ARCH_ARM64)
2632 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2633 AssertReturn(pu32CodeBuf, UINT32_MAX);
2634 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
2635 pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 0, f64Bit);
2636 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2637
2638#else
2639# error "Port me!"
2640#endif
2641 return off;
2642}
2643
2644
2645/**
2646 * Emits code that jumps to a new label if @a iGprSrc is zero.
2647 *
2648 * The operand size is given by @a f64Bit.
2649 */
2650DECLINLINE(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
2651 bool f64Bit, IEMNATIVELABELTYPE enmLabelType, uint16_t uData)
2652{
2653 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
2654 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
2655 return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
2656}
2657
2658
2659
2660/**
2661 * Emits a call to a 64-bit address.
2662 */
2663DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2664{
2665#ifdef RT_ARCH_AMD64
2666 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2667
2668 /* call rax */
2669 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2670 AssertReturn(pbCodeBuf, UINT32_MAX);
2671 pbCodeBuf[off++] = 0xff;
2672 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2673
2674#elif defined(RT_ARCH_ARM64)
2675 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2676
2677 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2678 AssertReturn(pu32CodeBuf, UINT32_MAX);
2679 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2680#else
2681# error "port me"
2682#endif
2683 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2684 return off;
2685}
2686
2687
2688/** @} */
2689
2690#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2691
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette