VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101547

Last change on this file since 101547 was 101547, checked in by vboxsync, 18 months ago

VMM/IEM: More TB disassembly and TB debuginfo. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 83.6 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101547 2023-10-23 00:50:37Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46
47/** @name Stack Frame Layout
48 *
49 * @{ */
50/** The size of the area for stack variables and spills and stuff.
51 * @note This limit is duplicated in the python script(s). */
52#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
53#ifdef RT_ARCH_AMD64
54/** Number of stack arguments slots for calls made from the frame. */
55# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
56/** An stack alignment adjustment (between non-volatile register pushes and
57 * the stack variable area, so the latter better aligned). */
58# define IEMNATIVE_FRAME_ALIGN_SIZE 8
59/** Number of any shadow arguments (spill area) for calls we make. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
62# else
63# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
64# endif
65
66/** Frame pointer (RBP) relative offset of the last push. */
67# ifdef RT_OS_WINDOWS
68# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
69# else
70# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
71# endif
72/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
73 * address for it). */
74# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
75/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
76# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
77/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
78# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
79/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
80# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
81/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
82# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
83
84# ifdef RT_OS_WINDOWS
85/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
86# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
87/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
88# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
89/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
90# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
91/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
92# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
93# endif
94
95#elif RT_ARCH_ARM64
96/** No stack argument slots, enough got 8 registers for arguments. */
97# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
98/** There are no argument spill area. */
99# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100
101/** Number of saved registers at the top of our stack frame.
102 * This includes the return address and old frame pointer, so x19 thru x30. */
103# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
104/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
105# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
106
107/** Frame pointer (BP) relative offset of the last push. */
108# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
109
110/** Frame pointer (BP) relative offset of the stack variable area (the lowest
111 * address for it). */
112# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
113
114#else
115# error "port me"
116#endif
117/** @} */
118
119
120/** @name Fixed Register Allocation(s)
121 * @{ */
122/** @def IEMNATIVE_REG_FIXED_PVMCPU
123 * The number of the register holding the pVCpu pointer. */
124/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
125 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
126 * @note This not available on AMD64, only ARM64. */
127/** @def IEMNATIVE_REG_FIXED_TMP0
128 * Dedicated temporary register.
129 * @todo replace this by a register allocator and content tracker. */
130/** @def IEMNATIVE_REG_FIXED_MASK
131 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
132 * architecture. */
133#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
134# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
135# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
136# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
137 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
138 | RT_BIT_32(X86_GREG_xSP) \
139 | RT_BIT_32(X86_GREG_xBP) )
140
141#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
142# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
143# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
144# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
145# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
146 | RT_BIT_32(ARMV8_A64_REG_LR) \
147 | RT_BIT_32(ARMV8_A64_REG_BP) \
148 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
149 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
150 | RT_BIT_32(ARMV8_A64_REG_X18) \
151 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
152
153#else
154# error "port me"
155#endif
156/** @} */
157
158/** @name Call related registers.
159 * @{ */
160/** @def IEMNATIVE_CALL_RET_GREG
161 * The return value register. */
162/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
163 * Number of arguments in registers. */
164/** @def IEMNATIVE_CALL_ARG0_GREG
165 * The general purpose register carrying argument \#0. */
166/** @def IEMNATIVE_CALL_ARG1_GREG
167 * The general purpose register carrying argument \#1. */
168/** @def IEMNATIVE_CALL_ARG2_GREG
169 * The general purpose register carrying argument \#2. */
170/** @def IEMNATIVE_CALL_ARG3_GREG
171 * The general purpose register carrying argument \#3. */
172/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
173 * Mask of registers the callee will not save and may trash. */
174#ifdef RT_ARCH_AMD64
175# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
176
177# ifdef RT_OS_WINDOWS
178# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
179# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
180# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
181# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
182# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
183# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
184 | RT_BIT_32(X86_GREG_xCX) \
185 | RT_BIT_32(X86_GREG_xDX) \
186 | RT_BIT_32(X86_GREG_x8) \
187 | RT_BIT_32(X86_GREG_x9) \
188 | RT_BIT_32(X86_GREG_x10) \
189 | RT_BIT_32(X86_GREG_x11) )
190# else
191# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
192# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
193# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
194# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
195# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
196# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
197# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
198# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
199 | RT_BIT_32(X86_GREG_xCX) \
200 | RT_BIT_32(X86_GREG_xDX) \
201 | RT_BIT_32(X86_GREG_xDI) \
202 | RT_BIT_32(X86_GREG_xSI) \
203 | RT_BIT_32(X86_GREG_x8) \
204 | RT_BIT_32(X86_GREG_x9) \
205 | RT_BIT_32(X86_GREG_x10) \
206 | RT_BIT_32(X86_GREG_x11) )
207# endif
208
209#elif defined(RT_ARCH_ARM64)
210# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
211# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
212# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
213# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
214# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
215# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
216# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
217# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
218# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
219# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
220# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
221 | RT_BIT_32(ARMV8_A64_REG_X1) \
222 | RT_BIT_32(ARMV8_A64_REG_X2) \
223 | RT_BIT_32(ARMV8_A64_REG_X3) \
224 | RT_BIT_32(ARMV8_A64_REG_X4) \
225 | RT_BIT_32(ARMV8_A64_REG_X5) \
226 | RT_BIT_32(ARMV8_A64_REG_X6) \
227 | RT_BIT_32(ARMV8_A64_REG_X7) \
228 | RT_BIT_32(ARMV8_A64_REG_X8) \
229 | RT_BIT_32(ARMV8_A64_REG_X9) \
230 | RT_BIT_32(ARMV8_A64_REG_X10) \
231 | RT_BIT_32(ARMV8_A64_REG_X11) \
232 | RT_BIT_32(ARMV8_A64_REG_X12) \
233 | RT_BIT_32(ARMV8_A64_REG_X13) \
234 | RT_BIT_32(ARMV8_A64_REG_X14) \
235 | RT_BIT_32(ARMV8_A64_REG_X15) \
236 | RT_BIT_32(ARMV8_A64_REG_X16) \
237 | RT_BIT_32(ARMV8_A64_REG_X17) )
238
239#endif
240
241/** @} */
242
243
244/** @def IEMNATIVE_HST_GREG_COUNT
245 * Number of host general purpose registers we tracker. */
246/** @def IEMNATIVE_HST_GREG_MASK
247 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
248 * inverted register masks and such to get down to a correct set of regs. */
249#ifdef RT_ARCH_AMD64
250# define IEMNATIVE_HST_GREG_COUNT 16
251# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
252
253#elif defined(RT_ARCH_ARM64)
254# define IEMNATIVE_HST_GREG_COUNT 32
255# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
256#else
257# error "Port me!"
258#endif
259
260
261/** Native code generator label types. */
262typedef enum
263{
264 kIemNativeLabelType_Invalid = 0,
265 kIemNativeLabelType_Return,
266 kIemNativeLabelType_Else,
267 kIemNativeLabelType_Endif,
268 kIemNativeLabelType_NonZeroRetOrPassUp,
269 kIemNativeLabelType_RaiseGp0,
270 kIemNativeLabelType_End
271} IEMNATIVELABELTYPE;
272
273/** Native code generator label definition. */
274typedef struct IEMNATIVELABEL
275{
276 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
277 * the epilog. */
278 uint32_t off;
279 /** The type of label (IEMNATIVELABELTYPE). */
280 uint16_t enmType;
281 /** Additional label data, type specific. */
282 uint16_t uData;
283} IEMNATIVELABEL;
284/** Pointer to a label. */
285typedef IEMNATIVELABEL *PIEMNATIVELABEL;
286
287
288/** Native code generator fixup types. */
289typedef enum
290{
291 kIemNativeFixupType_Invalid = 0,
292#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
293 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
294 kIemNativeFixupType_Rel32,
295#elif defined(RT_ARCH_ARM64)
296 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
297 kIemNativeFixupType_RelImm19At5,
298 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
299 kIemNativeFixupType_RelImm14At5,
300#endif
301 kIemNativeFixupType_End
302} IEMNATIVEFIXUPTYPE;
303
304/** Native code generator fixup. */
305typedef struct IEMNATIVEFIXUP
306{
307 /** Code offset of the fixup location. */
308 uint32_t off;
309 /** The IEMNATIVELABEL this is a fixup for. */
310 uint16_t idxLabel;
311 /** The fixup type (IEMNATIVEFIXUPTYPE). */
312 uint8_t enmType;
313 /** Addend or other data. */
314 int8_t offAddend;
315} IEMNATIVEFIXUP;
316/** Pointer to a native code generator fixup. */
317typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
318
319
320/**
321 * Guest registers that can be shadowed in GPRs.
322 */
323typedef enum IEMNATIVEGSTREG : uint8_t
324{
325 kIemNativeGstReg_GprFirst = 0,
326 kIemNativeGstReg_GprLast = 15,
327 kIemNativeGstReg_Pc,
328 kIemNativeGstReg_EFlags, /**< This one is problematic since the higher bits are used internally. */
329 /* gap: 18..23 */
330 kIemNativeGstReg_SegSelFirst = 24,
331 kIemNativeGstReg_SegSelLast = 29,
332 kIemNativeGstReg_SegBaseFirst = 30,
333 kIemNativeGstReg_SegBaseLast = 35,
334 kIemNativeGstReg_SegLimitFirst = 36,
335 kIemNativeGstReg_SegLimitLast = 41,
336 kIemNativeGstReg_End
337} IEMNATIVEGSTREG;
338
339/**
340 * Guest registers (classes) that can be referenced.
341 */
342typedef enum IEMNATIVEGSTREGREF : uint8_t
343{
344 kIemNativeGstRegRef_Invalid = 0,
345 kIemNativeGstRegRef_Gpr,
346 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
347 kIemNativeGstRegRef_EFlags,
348 kIemNativeGstRegRef_MxCsr,
349 kIemNativeGstRegRef_FpuReg,
350 kIemNativeGstRegRef_MReg,
351 kIemNativeGstRegRef_XReg,
352 kIemNativeGstRegRef_YReg,
353 kIemNativeGstRegRef_End
354} IEMNATIVEGSTREGREF;
355
356
357/** Variable kinds. */
358typedef enum IEMNATIVEVARKIND : uint8_t
359{
360 /** Customary invalid zero value. */
361 kIemNativeVarKind_Invalid = 0,
362 /** This is either in a register or on the stack. */
363 kIemNativeVarKind_Stack,
364 /** Immediate value - loaded into register when needed, or can live on the
365 * stack if referenced (in theory). */
366 kIemNativeVarKind_Immediate,
367 /** Variable reference - loaded into register when needed, never stack. */
368 kIemNativeVarKind_VarRef,
369 /** Guest register reference - loaded into register when needed, never stack. */
370 kIemNativeVarKind_GstRegRef,
371 /** End of valid values. */
372 kIemNativeVarKind_End
373} IEMNATIVEVARKIND;
374
375
376/** Variable or argument. */
377typedef struct IEMNATIVEVAR
378{
379 /** The kind of variable. */
380 IEMNATIVEVARKIND enmKind;
381 /** The variable size in bytes. */
382 uint8_t cbVar;
383 /** The first stack slot (uint64_t), except for immediate and references
384 * where it usually is UINT8_MAX. */
385 uint8_t idxStackSlot;
386 /** The host register allocated for the variable, UINT8_MAX if not. */
387 uint8_t idxReg;
388 /** The argument number if argument, UINT8_MAX if regular variable. */
389 uint8_t uArgNo;
390 /** If referenced, the index of the variable referencing this one, otherwise
391 * UINT8_MAX. A referenced variable must only be placed on the stack and
392 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
393 uint8_t idxReferrerVar;
394 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
395 IEMNATIVEGSTREG enmGstReg;
396 uint8_t bAlign;
397
398 union
399 {
400 /** kIemNativeVarKind_Immediate: The immediate value. */
401 uint64_t uValue;
402 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
403 uint8_t idxRefVar;
404 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
405 struct
406 {
407 /** The class of register. */
408 IEMNATIVEGSTREGREF enmClass;
409 /** Index within the class. */
410 uint8_t idx;
411 } GstRegRef;
412 } u;
413} IEMNATIVEVAR;
414
415/** What is being kept in a host register. */
416typedef enum IEMNATIVEWHAT : uint8_t
417{
418 /** The traditional invalid zero value. */
419 kIemNativeWhat_Invalid = 0,
420 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
421 kIemNativeWhat_Var,
422 /** Temporary register, this is typically freed when a MC completes. */
423 kIemNativeWhat_Tmp,
424 /** Call argument w/o a variable mapping. This is free (via
425 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
426 kIemNativeWhat_Arg,
427 /** Return status code.
428 * @todo not sure if we need this... */
429 kIemNativeWhat_rc,
430 /** The fixed pVCpu (PVMCPUCC) register.
431 * @todo consider offsetting this on amd64 to use negative offsets to access
432 * more members using 8-byte disp. */
433 kIemNativeWhat_pVCpuFixed,
434 /** The fixed pCtx (PCPUMCTX) register.
435 * @todo consider offsetting this on amd64 to use negative offsets to access
436 * more members using 8-byte disp. */
437 kIemNativeWhat_pCtxFixed,
438 /** Fixed temporary register. */
439 kIemNativeWhat_FixedTmp,
440 /** Register reserved by the CPU or OS architecture. */
441 kIemNativeWhat_FixedReserved,
442 /** End of valid values. */
443 kIemNativeWhat_End
444} IEMNATIVEWHAT;
445
446/**
447 * Host general register entry.
448 *
449 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
450 *
451 * @todo Track immediate values in host registers similarlly to how we track the
452 * guest register shadow copies. For it to be real helpful, though,
453 * we probably need to know which will be reused and put them into
454 * non-volatile registers, otherwise it's going to be more or less
455 * restricted to an instruction or two.
456 */
457typedef struct IEMNATIVEHSTREG
458{
459 /** Set of guest registers this one shadows.
460 *
461 * Using a bitmap here so we can designate the same host register as a copy
462 * for more than one guest register. This is expected to be useful in
463 * situations where one value is copied to several registers in a sequence.
464 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
465 * sequence we'd want to let this register follow to be a copy of and there
466 * will always be places where we'd be picking the wrong one.
467 */
468 uint64_t fGstRegShadows;
469 /** What is being kept in this register. */
470 IEMNATIVEWHAT enmWhat;
471 /** Variable index if holding a variable, otherwise UINT8_MAX. */
472 uint8_t idxVar;
473 /** Alignment padding. */
474 uint8_t abAlign[6];
475} IEMNATIVEHSTREG;
476
477
478/**
479 * Conditional stack entry.
480 */
481typedef struct IEMNATIVECOND
482{
483 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
484 bool fInElse;
485 /** The label for the IEM_MC_ELSE. */
486 uint32_t idxLabelElse;
487 /** The label for the IEM_MC_ENDIF. */
488 uint32_t idxLabelEndIf;
489} IEMNATIVECOND;
490/** Pointer to a condition stack entry. */
491typedef IEMNATIVECOND *PIEMNATIVECOND;
492
493
494/**
495 * Native recompiler state.
496 */
497typedef struct IEMRECOMPILERSTATE
498{
499 /** Size of the buffer that pbNativeRecompileBufR3 points to in
500 * IEMNATIVEINSTR units. */
501 uint32_t cInstrBufAlloc;
502#ifdef VBOX_STRICT
503 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
504 uint32_t offInstrBufChecked;
505#else
506 uint32_t uPadding; /* We don't keep track of the size here... */
507#endif
508 /** Fixed temporary code buffer for native recompilation. */
509 PIEMNATIVEINSTR pInstrBuf;
510
511 /** Bitmaps with the label types used. */
512 uint64_t bmLabelTypes;
513 /** Actual number of labels in paLabels. */
514 uint32_t cLabels;
515 /** Max number of entries allowed in paLabels before reallocating it. */
516 uint32_t cLabelsAlloc;
517 /** Labels defined while recompiling (referenced by fixups). */
518 PIEMNATIVELABEL paLabels;
519
520 /** Actual number of fixups paFixups. */
521 uint32_t cFixups;
522 /** Max number of entries allowed in paFixups before reallocating it. */
523 uint32_t cFixupsAlloc;
524 /** Buffer used by the recompiler for recording fixups when generating code. */
525 PIEMNATIVEFIXUP paFixups;
526
527#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
528 /** Number of debug info entries allocated for pDbgInfo. */
529 uint32_t cDbgInfoAlloc;
530 uint32_t uPadding;
531 /** Debug info. */
532 PIEMTBDBG pDbgInfo;
533#else
534 uint32_t abPadding1[2];
535 uintptr_t uPtrPadding2;
536#endif
537
538 /** The translation block being recompiled. */
539 PCIEMTB pTbOrg;
540
541 /** Allocation bitmap fro aHstRegs. */
542 uint32_t bmHstRegs;
543
544 /** Bitmap marking which host register contains guest register shadow copies.
545 * This is used during register allocation to try preserve copies. */
546 uint32_t bmHstRegsWithGstShadow;
547 /** Bitmap marking valid entries in aidxGstRegShadows. */
548 uint64_t bmGstRegShadows;
549
550 /** The current condition stack depth (aCondStack). */
551 uint8_t cCondDepth;
552 uint8_t bPadding;
553 /** Condition sequence number (for generating unique labels). */
554 uint16_t uCondSeqNo;
555
556 /** Allocation bitmap for aVars. */
557 uint32_t bmVars;
558 union
559 {
560 /** Index of variable arguments, UINT8_MAX if not valid. */
561 uint8_t aidxArgVars[8];
562 /** For more efficient resetting. */
563 uint64_t u64ArgVars;
564 };
565
566 /** Host register allocation tracking. */
567 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
568 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
569 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
570 * (A shadow copy of a guest register can only be held in a one host register,
571 * there are no duplicate copies or ambiguities like that). */
572 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
573
574 /** The condition nesting stack. */
575 IEMNATIVECOND aCondStack[4];
576
577 /** Variables and arguments. */
578 IEMNATIVEVAR aVars[16];
579} IEMRECOMPILERSTATE;
580/** Pointer to a native recompiler state. */
581typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
582
583
584/**
585 * Native recompiler worker for a threaded function.
586 *
587 * @returns New code buffer offset, UINT32_MAX in case of failure.
588 * @param pReNative The native recompiler state.
589 * @param off The current code buffer offset.
590 * @param pCallEntry The threaded call entry.
591 *
592 * @note This is not allowed to throw anything atm.
593 */
594typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
595 PCIEMTHRDEDCALLENTRY pCallEntry));
596/** Pointer to a native recompiler worker for a threaded function. */
597typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
598
599/** Defines a native recompiler worker for a threaded function. */
600#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
601 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
602/** Prototypes a native recompiler function for a threaded function. */
603#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
604
605
606DECLHIDDEN(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
607 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
608DECLHIDDEN(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere) RT_NOEXCEPT;
609DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
610 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
611DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
612 uint32_t cInstrReq) RT_NOEXCEPT;
613
614DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
615 bool fPreferVolatile = true) RT_NOEXCEPT;
616DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
617 bool fPreferVolatile = true) RT_NOEXCEPT;
618DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
619 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
620DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
621DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
622DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
623DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
624DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
625DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
626DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
627DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
628
629DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
630 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
631DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
632 uint8_t idxInstr) RT_NOEXCEPT;
633
634
635/**
636 * Ensures that there is sufficient space in the instruction output buffer.
637 *
638 * This will reallocate the buffer if needed and allowed.
639 *
640 * @returns Pointer to the instruction output buffer on success, NULL on
641 * failure.
642 * @param pReNative The native recompile state.
643 * @param off Current instruction offset. Works safely for UINT32_MAX
644 * as well.
645 * @param cInstrReq Number of instruction about to be added. It's okay to
646 * overestimate this a bit.
647 */
648DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
649{
650 uint64_t const offChecked = off + (uint64_t)cInstrReq;
651 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
652 {
653#ifdef VBOX_STRICT
654 pReNative->offInstrBufChecked = offChecked;
655#endif
656 return pReNative->pInstrBuf;
657 }
658 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
659}
660
661/**
662 * Checks that we didn't exceed the space requested in the last
663 * iemNativeInstrBufEnsure() call. */
664#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
665 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
666 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
667
668
669/**
670 * Emit a simple marker instruction to more easily tell where something starts
671 * in the disassembly.
672 */
673DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
674{
675#ifdef RT_ARCH_AMD64
676 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
677 AssertReturn(pbCodeBuf, UINT32_MAX);
678 if (uInfo == 0)
679 {
680 /* nop */
681 pbCodeBuf[off++] = 0x90;
682 }
683 else
684 {
685 /* nop [disp32] */
686 pbCodeBuf[off++] = 0x0f;
687 pbCodeBuf[off++] = 0x1f;
688 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
689 pbCodeBuf[off++] = RT_BYTE1(uInfo);
690 pbCodeBuf[off++] = RT_BYTE2(uInfo);
691 pbCodeBuf[off++] = RT_BYTE3(uInfo);
692 pbCodeBuf[off++] = RT_BYTE4(uInfo);
693 }
694#elif RT_ARCH_ARM64
695 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
696 AssertReturn(pu32CodeBuf, UINT32_MAX);
697 /* nop */
698 pu32CodeBuf[off++] = 0xd503201f;
699
700 RT_NOREF(uInfo);
701#else
702# error "port me"
703#endif
704 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
705 return off;
706}
707
708
709/*********************************************************************************************************************************
710* Loads, Stores and Related Stuff. *
711*********************************************************************************************************************************/
712
713/**
714 * Emits setting a GPR to zero.
715 */
716DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
717{
718#ifdef RT_ARCH_AMD64
719 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
720 AssertReturn(pbCodeBuf, UINT32_MAX);
721 /* xor gpr32, gpr32 */
722 if (iGpr >= 8)
723 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
724 pbCodeBuf[off++] = 0x33;
725 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
726
727#elif RT_ARCH_ARM64
728 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
729 AssertReturn(pu32CodeBuf, UINT32_MAX);
730 /* mov gpr, #0x0 */
731 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
732
733#else
734# error "port me"
735#endif
736 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
737 return off;
738}
739
740
741/**
742 * Emits loading a constant into a 64-bit GPR
743 */
744DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
745{
746 if (!uImm64)
747 return iemNativeEmitGprZero(pReNative, off, iGpr);
748
749#ifdef RT_ARCH_AMD64
750 if (uImm64 <= UINT32_MAX)
751 {
752 /* mov gpr, imm32 */
753 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
754 AssertReturn(pbCodeBuf, UINT32_MAX);
755 if (iGpr >= 8)
756 pbCodeBuf[off++] = X86_OP_REX_B;
757 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
758 pbCodeBuf[off++] = RT_BYTE1(uImm64);
759 pbCodeBuf[off++] = RT_BYTE2(uImm64);
760 pbCodeBuf[off++] = RT_BYTE3(uImm64);
761 pbCodeBuf[off++] = RT_BYTE4(uImm64);
762 }
763 else
764 {
765 /* mov gpr, imm64 */
766 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
767 AssertReturn(pbCodeBuf, UINT32_MAX);
768 if (iGpr < 8)
769 pbCodeBuf[off++] = X86_OP_REX_W;
770 else
771 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
772 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
773 pbCodeBuf[off++] = RT_BYTE1(uImm64);
774 pbCodeBuf[off++] = RT_BYTE2(uImm64);
775 pbCodeBuf[off++] = RT_BYTE3(uImm64);
776 pbCodeBuf[off++] = RT_BYTE4(uImm64);
777 pbCodeBuf[off++] = RT_BYTE5(uImm64);
778 pbCodeBuf[off++] = RT_BYTE6(uImm64);
779 pbCodeBuf[off++] = RT_BYTE7(uImm64);
780 pbCodeBuf[off++] = RT_BYTE8(uImm64);
781 }
782
783#elif RT_ARCH_ARM64
784 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
785 AssertReturn(pu32CodeBuf, UINT32_MAX);
786
787 /*
788 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
789 * supply remaining bits using 'movk grp, imm16, lsl #x'.
790 *
791 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
792 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
793 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
794 * after the first non-zero immediate component so we switch to movk for
795 * the remainder.
796 */
797 uint32_t fMovK = 0;
798 /* mov gpr, imm16 */
799 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
800 if (uImmPart)
801 {
802 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
803 fMovK |= RT_BIT_32(29);
804 }
805 /* mov[k] gpr, imm16, lsl #16 */
806 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
807 if (uImmPart)
808 {
809 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
810 fMovK |= RT_BIT_32(29);
811 }
812 /* mov[k] gpr, imm16, lsl #32 */
813 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
814 if (uImmPart)
815 {
816 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
817 fMovK |= RT_BIT_32(29);
818 }
819 /* mov[k] gpr, imm16, lsl #48 */
820 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
821 if (uImmPart)
822 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
823
824 /** @todo there is an inverted mask variant we might want to explore if it
825 * reduces the number of instructions... */
826 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
827 * clang 12.x does that, only to use the 'x' version for the
828 * addressing in the following ldr). */
829
830#else
831# error "port me"
832#endif
833 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
834 return off;
835}
836
837
838/**
839 * Emits loading a constant into a 8-bit GPR
840 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
841 * only the ARM64 version does that.
842 */
843DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
844{
845#ifdef RT_ARCH_AMD64
846 /* mov gpr, imm8 */
847 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
848 AssertReturn(pbCodeBuf, UINT32_MAX);
849 if (iGpr >= 8)
850 pbCodeBuf[off++] = X86_OP_REX_B;
851 else if (iGpr >= 4)
852 pbCodeBuf[off++] = X86_OP_REX;
853 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
854 pbCodeBuf[off++] = RT_BYTE1(uImm8);
855
856#elif RT_ARCH_ARM64
857 /* movz gpr, imm16, lsl #0 */
858 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
859 AssertReturn(pu32CodeBuf, UINT32_MAX);
860 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
861
862#else
863# error "port me"
864#endif
865 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
866 return off;
867}
868
869
870#ifdef RT_ARCH_AMD64
871/**
872 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
873 */
874DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
875{
876 if (offVCpu < 128)
877 {
878 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
879 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
880 }
881 else
882 {
883 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
884 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
885 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
886 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
887 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
888 }
889 return off;
890}
891#elif RT_ARCH_ARM64
892/**
893 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
894 */
895DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
896 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
897{
898 /*
899 * There are a couple of ldr variants that takes an immediate offset, so
900 * try use those if we can, otherwise we have to use the temporary register
901 * help with the addressing.
902 */
903 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
904 {
905 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
906 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
907 AssertReturn(pu32CodeBuf, UINT32_MAX);
908 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
909 }
910 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
911 {
912 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
913 AssertReturn(pu32CodeBuf, UINT32_MAX);
914 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
915 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
916 }
917 else
918 {
919 /* The offset is too large, so we must load it into a register and use
920 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
921 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
922 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
923
924 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
925 AssertReturn(pu32CodeBuf, UINT32_MAX);
926 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
927 }
928 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
929 return off;
930}
931#endif
932
933
934/**
935 * Emits a 64-bit GPR load of a VCpu value.
936 */
937DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
938{
939#ifdef RT_ARCH_AMD64
940 /* mov reg64, mem64 */
941 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
942 AssertReturn(pbCodeBuf, UINT32_MAX);
943 if (iGpr < 8)
944 pbCodeBuf[off++] = X86_OP_REX_W;
945 else
946 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
947 pbCodeBuf[off++] = 0x8b;
948 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
949 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
950
951#elif RT_ARCH_ARM64
952 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
953
954#else
955# error "port me"
956#endif
957 return off;
958}
959
960
961/**
962 * Emits a 32-bit GPR load of a VCpu value.
963 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
964 */
965DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
966{
967#ifdef RT_ARCH_AMD64
968 /* mov reg32, mem32 */
969 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
970 AssertReturn(pbCodeBuf, UINT32_MAX);
971 if (iGpr >= 8)
972 pbCodeBuf[off++] = X86_OP_REX_R;
973 pbCodeBuf[off++] = 0x8b;
974 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
975 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
976
977#elif RT_ARCH_ARM64
978 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
979
980#else
981# error "port me"
982#endif
983 return off;
984}
985
986
987/**
988 * Emits a 16-bit GPR load of a VCpu value.
989 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
990 */
991DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
992{
993#ifdef RT_ARCH_AMD64
994 /* movzx reg32, mem16 */
995 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
996 AssertReturn(pbCodeBuf, UINT32_MAX);
997 if (iGpr >= 8)
998 pbCodeBuf[off++] = X86_OP_REX_R;
999 pbCodeBuf[off++] = 0x0f;
1000 pbCodeBuf[off++] = 0xb7;
1001 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1002 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1003
1004#elif RT_ARCH_ARM64
1005 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
1006
1007#else
1008# error "port me"
1009#endif
1010 return off;
1011}
1012
1013
1014/**
1015 * Emits a 8-bit GPR load of a VCpu value.
1016 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
1017 */
1018DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1019{
1020#ifdef RT_ARCH_AMD64
1021 /* movzx reg32, mem8 */
1022 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1023 AssertReturn(pbCodeBuf, UINT32_MAX);
1024 if (iGpr >= 8)
1025 pbCodeBuf[off++] = X86_OP_REX_R;
1026 pbCodeBuf[off++] = 0x0f;
1027 pbCodeBuf[off++] = 0xb6;
1028 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1029 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1030
1031#elif RT_ARCH_ARM64
1032 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1033
1034#else
1035# error "port me"
1036#endif
1037 return off;
1038}
1039
1040
1041/**
1042 * Emits a store of a GPR value to a 64-bit VCpu field.
1043 */
1044DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1045{
1046#ifdef RT_ARCH_AMD64
1047 /* mov mem64, reg64 */
1048 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1049 AssertReturn(pbCodeBuf, UINT32_MAX);
1050 if (iGpr < 8)
1051 pbCodeBuf[off++] = X86_OP_REX_W;
1052 else
1053 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1054 pbCodeBuf[off++] = 0x89;
1055 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1056 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1057
1058#elif RT_ARCH_ARM64
1059 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1060
1061#else
1062# error "port me"
1063#endif
1064 return off;
1065}
1066
1067
1068/**
1069 * Emits a store of a GPR value to a 32-bit VCpu field.
1070 */
1071DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1072{
1073#ifdef RT_ARCH_AMD64
1074 /* mov mem32, reg32 */
1075 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1076 AssertReturn(pbCodeBuf, UINT32_MAX);
1077 if (iGpr >= 8)
1078 pbCodeBuf[off++] = X86_OP_REX_R;
1079 pbCodeBuf[off++] = 0x89;
1080 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1081 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1082
1083#elif RT_ARCH_ARM64
1084 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1085
1086#else
1087# error "port me"
1088#endif
1089 return off;
1090}
1091
1092
1093/**
1094 * Emits a store of a GPR value to a 16-bit VCpu field.
1095 */
1096DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1097{
1098#ifdef RT_ARCH_AMD64
1099 /* mov mem16, reg16 */
1100 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1101 AssertReturn(pbCodeBuf, UINT32_MAX);
1102 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1103 if (iGpr >= 8)
1104 pbCodeBuf[off++] = X86_OP_REX_R;
1105 pbCodeBuf[off++] = 0x89;
1106 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1107 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1108
1109#elif RT_ARCH_ARM64
1110 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1111
1112#else
1113# error "port me"
1114#endif
1115 return off;
1116}
1117
1118
1119/**
1120 * Emits a store of a GPR value to a 8-bit VCpu field.
1121 */
1122DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1123{
1124#ifdef RT_ARCH_AMD64
1125 /* mov mem8, reg8 */
1126 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1127 AssertReturn(pbCodeBuf, UINT32_MAX);
1128 if (iGpr >= 8)
1129 pbCodeBuf[off++] = X86_OP_REX_R;
1130 pbCodeBuf[off++] = 0x88;
1131 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1132 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1133
1134#elif RT_ARCH_ARM64
1135 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1136
1137#else
1138# error "port me"
1139#endif
1140 return off;
1141}
1142
1143
1144/**
1145 * Emits a gprdst = gprsrc load.
1146 */
1147DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1148{
1149#ifdef RT_ARCH_AMD64
1150 /* mov gprdst, gprsrc */
1151 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1152 AssertReturn(pbCodeBuf, UINT32_MAX);
1153 if ((iGprDst | iGprSrc) >= 8)
1154 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1155 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1156 : X86_OP_REX_W | X86_OP_REX_R;
1157 else
1158 pbCodeBuf[off++] = X86_OP_REX_W;
1159 pbCodeBuf[off++] = 0x8b;
1160 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1161
1162#elif RT_ARCH_ARM64
1163 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1164 AssertReturn(pu32CodeBuf, UINT32_MAX);
1165 /* mov dst, src; alias for: orr dst, xzr, src */
1166 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1167
1168#else
1169# error "port me"
1170#endif
1171 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1172 return off;
1173}
1174
1175#ifdef RT_ARCH_AMD64
1176/**
1177 * Common bit of iemNativeEmitLoadGprByBp and friends.
1178 */
1179DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1180 PIEMRECOMPILERSTATE pReNativeAssert)
1181{
1182 if (offDisp < 128 && offDisp >= -128)
1183 {
1184 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1185 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1186 }
1187 else
1188 {
1189 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1190 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1191 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1192 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1193 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1194 }
1195 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1196 return off;
1197}
1198#endif
1199
1200
1201#ifdef RT_ARCH_AMD64
1202/**
1203 * Emits a 64-bit GRP load instruction with an BP relative source address.
1204 */
1205DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1206{
1207 /* mov gprdst, qword [rbp + offDisp] */
1208 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1209 AssertReturn(pbCodeBuf, UINT32_MAX);
1210 if (iGprDst < 8)
1211 pbCodeBuf[off++] = X86_OP_REX_W;
1212 else
1213 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1214 pbCodeBuf[off++] = 0x8b;
1215 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1216}
1217#endif
1218
1219
1220#ifdef RT_ARCH_AMD64
1221/**
1222 * Emits a 32-bit GRP load instruction with an BP relative source address.
1223 */
1224DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1225{
1226 /* mov gprdst, dword [rbp + offDisp] */
1227 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1228 AssertReturn(pbCodeBuf, UINT32_MAX);
1229 if (iGprDst >= 8)
1230 pbCodeBuf[off++] = X86_OP_REX_R;
1231 pbCodeBuf[off++] = 0x8b;
1232 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1233}
1234#endif
1235
1236
1237#ifdef RT_ARCH_AMD64
1238/**
1239 * Emits a load effective address to a GRP with an BP relative source address.
1240 */
1241DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1242{
1243 /* lea gprdst, [rbp + offDisp] */
1244 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1245 AssertReturn(pbCodeBuf, UINT32_MAX);
1246 if (iGprDst < 8)
1247 pbCodeBuf[off++] = X86_OP_REX_W;
1248 else
1249 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1250 pbCodeBuf[off++] = 0x8d;
1251 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1252}
1253#endif
1254
1255
1256/**
1257 * Emits a 64-bit GPR store with an BP relative destination address.
1258 *
1259 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1260 */
1261DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1262{
1263#ifdef RT_ARCH_AMD64
1264 /* mov qword [rbp + offDisp], gprdst */
1265 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1266 AssertReturn(pbCodeBuf, UINT32_MAX);
1267 if (iGprSrc < 8)
1268 pbCodeBuf[off++] = X86_OP_REX_W;
1269 else
1270 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1271 pbCodeBuf[off++] = 0x89;
1272 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1273
1274#elif defined(RT_ARCH_ARM64)
1275 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1276 {
1277 /* str w/ unsigned imm12 (scaled) */
1278 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1279 AssertReturn(pu32CodeBuf, UINT32_MAX);
1280 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1281 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1282 }
1283 else if (offDisp >= -256 && offDisp <= 256)
1284 {
1285 /* stur w/ signed imm9 (unscaled) */
1286 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1287 AssertReturn(pu32CodeBuf, UINT32_MAX);
1288 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1289 }
1290 else
1291 {
1292 /* Use temporary indexing register. */
1293 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1294 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1295 AssertReturn(pu32CodeBuf, UINT32_MAX);
1296 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1297 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1298 }
1299 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1300 return off;
1301
1302#else
1303# error "Port me!"
1304#endif
1305}
1306
1307
1308/**
1309 * Emits a 64-bit immediate store with an BP relative destination address.
1310 *
1311 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1312 */
1313DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1314{
1315#ifdef RT_ARCH_AMD64
1316 if ((int64_t)uImm64 == (int32_t)uImm64)
1317 {
1318 /* mov qword [rbp + offDisp], imm32 - sign extended */
1319 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1320 AssertReturn(pbCodeBuf, UINT32_MAX);
1321
1322 pbCodeBuf[off++] = X86_OP_REX_W;
1323 pbCodeBuf[off++] = 0xc7;
1324 if (offDisp < 128 && offDisp >= -128)
1325 {
1326 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1327 pbCodeBuf[off++] = (uint8_t)offDisp;
1328 }
1329 else
1330 {
1331 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1332 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1333 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1334 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1335 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1336 }
1337 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1338 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1339 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1340 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1341 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1342 return off;
1343 }
1344#endif
1345
1346 /* Load tmp0, imm64; Store tmp to bp+disp. */
1347 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1348 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1349}
1350
1351
1352/*********************************************************************************************************************************
1353* Subtraction and Additions *
1354*********************************************************************************************************************************/
1355
1356
1357#ifdef RT_ARCH_AMD64
1358/**
1359 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1360 */
1361DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1362{
1363 /* sub gprdst, imm8/imm32 */
1364 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1365 AssertReturn(pbCodeBuf, UINT32_MAX);
1366 if (iGprDst < 8)
1367 pbCodeBuf[off++] = X86_OP_REX_W;
1368 else
1369 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1370 if (iSubtrahend < 128 && iSubtrahend >= -128)
1371 {
1372 pbCodeBuf[off++] = 0x83;
1373 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1374 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1375 }
1376 else
1377 {
1378 pbCodeBuf[off++] = 0x81;
1379 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1380 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1381 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1382 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1383 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1384 }
1385 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1386 return off;
1387}
1388#endif
1389
1390
1391/**
1392 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1393 * @note The AMD64 version sets flags.
1394 */
1395DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1396{
1397#if defined(RT_ARCH_AMD64)
1398 /* add Gv,Ev */
1399 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1400 AssertReturn(pbCodeBuf, UINT32_MAX);
1401 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1402 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1403 pbCodeBuf[off++] = 0x04;
1404 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1405
1406#elif defined(RT_ARCH_ARM64)
1407 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1408 AssertReturn(pu32CodeBuf, UINT32_MAX);
1409 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1410
1411#else
1412# error "Port me"
1413#endif
1414 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1415 return off;
1416}
1417
1418
1419/**
1420 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1421 */
1422DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1423{
1424#if defined(RT_ARCH_AMD64)
1425 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1426 AssertReturn(pbCodeBuf, UINT32_MAX);
1427 /* add or inc */
1428 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1429 if (iImm8 != 1)
1430 {
1431 pbCodeBuf[off++] = 0x83;
1432 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1433 pbCodeBuf[off++] = (uint8_t)iImm8;
1434 }
1435 else
1436 {
1437 pbCodeBuf[off++] = 0xff;
1438 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1439 }
1440
1441#elif defined(RT_ARCH_ARM64)
1442 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1443 AssertReturn(pu32CodeBuf, UINT32_MAX);
1444 if (iImm8 >= 0)
1445 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1446 else
1447 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1448
1449#else
1450# error "Port me"
1451#endif
1452 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1453 return off;
1454}
1455
1456
1457/**
1458 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1459 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1460 */
1461DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1462{
1463#if defined(RT_ARCH_AMD64)
1464 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1465 AssertReturn(pbCodeBuf, UINT32_MAX);
1466 /* add or inc */
1467 if (iGprDst >= 8)
1468 pbCodeBuf[off++] = X86_OP_REX_B;
1469 if (iImm8 != 1)
1470 {
1471 pbCodeBuf[off++] = 0x83;
1472 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1473 pbCodeBuf[off++] = (uint8_t)iImm8;
1474 }
1475 else
1476 {
1477 pbCodeBuf[off++] = 0xff;
1478 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1479 }
1480
1481#elif defined(RT_ARCH_ARM64)
1482 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1483 AssertReturn(pu32CodeBuf, UINT32_MAX);
1484 if (iImm8 >= 0)
1485 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1486 else
1487 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1488
1489#else
1490# error "Port me"
1491#endif
1492 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1493 return off;
1494}
1495
1496
1497/**
1498 * Emits a 64-bit GPR additions with a 64-bit signed addend.
1499 */
1500DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1501{
1502#if defined(RT_ARCH_AMD64)
1503 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1504 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1505
1506 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1507 {
1508 /* add grp, imm32 */
1509 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1510 AssertReturn(pbCodeBuf, UINT32_MAX);
1511 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1512 pbCodeBuf[off++] = 0x81;
1513 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1514 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1515 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1516 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1517 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1518 }
1519 else
1520 {
1521 /* Best to use a temporary register to deal with this in the simplest way: */
1522 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1523 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1524
1525 /* add dst, tmpreg */
1526 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1527 AssertReturn(pbCodeBuf, UINT32_MAX);
1528 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1529 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1530 pbCodeBuf[off++] = 0x03;
1531 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1532
1533 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1534 }
1535
1536#elif defined(RT_ARCH_ARM64)
1537 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1538 {
1539 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1540 AssertReturn(pu32CodeBuf, UINT32_MAX);
1541 if (iAddend >= 0)
1542 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
1543 else
1544 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
1545 }
1546 else
1547 {
1548 /* Use temporary register for the immediate. */
1549 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1550 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1551
1552 /* add gprdst, gprdst, tmpreg */
1553 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1554 AssertReturn(pu32CodeBuf, UINT32_MAX);
1555 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
1556
1557 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1558 }
1559
1560#else
1561# error "Port me"
1562#endif
1563 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1564 return off;
1565}
1566
1567
1568/**
1569 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
1570 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1571 */
1572DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1573{
1574#if defined(RT_ARCH_AMD64)
1575 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1576 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1577
1578 /* add grp, imm32 */
1579 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1580 AssertReturn(pbCodeBuf, UINT32_MAX);
1581 if (iGprDst >= 8)
1582 pbCodeBuf[off++] = X86_OP_REX_B;
1583 pbCodeBuf[off++] = 0x81;
1584 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1585 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1586 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1587 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1588 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1589
1590#elif defined(RT_ARCH_ARM64)
1591 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1592 {
1593 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1594 AssertReturn(pu32CodeBuf, UINT32_MAX);
1595 if (iAddend >= 0)
1596 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
1597 else
1598 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
1599 }
1600 else
1601 {
1602 /* Use temporary register for the immediate. */
1603 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1604 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1605
1606 /* add gprdst, gprdst, tmpreg */
1607 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1608 AssertReturn(pu32CodeBuf, UINT32_MAX);
1609 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
1610
1611 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1612 }
1613
1614#else
1615# error "Port me"
1616#endif
1617 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1618 return off;
1619}
1620
1621
1622
1623/*********************************************************************************************************************************
1624* Bit Operations *
1625*********************************************************************************************************************************/
1626
1627/**
1628 * Emits code for clearing bits 16 thru 63 in the GPR.
1629 */
1630DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1631{
1632#if defined(RT_ARCH_AMD64)
1633 /* movzx reg32, reg16 */
1634 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1635 AssertReturn(pbCodeBuf, UINT32_MAX);
1636 if (iGprDst >= 8)
1637 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
1638 pbCodeBuf[off++] = 0x0f;
1639 pbCodeBuf[off++] = 0xb7;
1640 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1641
1642#elif defined(RT_ARCH_ARM64)
1643 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1644 AssertReturn(pu32CodeBuf, UINT32_MAX);
1645# if 1
1646 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1647# else
1648 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1649 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1650# endif
1651#else
1652# error "Port me"
1653#endif
1654 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1655 return off;
1656}
1657
1658
1659/**
1660 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1661 */
1662DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1663{
1664#if defined(RT_ARCH_AMD64)
1665 /* shr dst, cShift */
1666 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1667 AssertReturn(pbCodeBuf, UINT32_MAX);
1668 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1669 pbCodeBuf[off++] = 0xc0;
1670 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1671 pbCodeBuf[off++] = cShift;
1672 Assert(cShift > 0 && cShift < 64);
1673
1674#elif defined(RT_ARCH_ARM64)
1675 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1676 AssertReturn(pu32CodeBuf, UINT32_MAX);
1677 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1678#else
1679# error "Port me"
1680#endif
1681 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1682 return off;
1683}
1684
1685
1686/*********************************************************************************************************************************
1687* Compare and Testing *
1688*********************************************************************************************************************************/
1689
1690
1691#ifdef RT_ARCH_ARM64
1692/**
1693 * Emits an ARM64 compare instruction.
1694 */
1695DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1696 bool f64Bit = true, uint32_t cShift = 0,
1697 ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1698{
1699 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1700 AssertReturn(pu32CodeBuf, UINT32_MAX);
1701 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
1702 f64Bit, true /*fSetFlags*/, cShift, enmShift);
1703 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1704 return off;
1705}
1706#endif
1707
1708
1709/**
1710 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1711 * with conditional instruction.
1712 */
1713DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1714{
1715#ifdef RT_ARCH_AMD64
1716 /* cmp Gv, Ev */
1717 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1718 AssertReturn(pbCodeBuf, UINT32_MAX);
1719 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1720 pbCodeBuf[off++] = 0x3b;
1721 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1722 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1723
1724#elif defined(RT_ARCH_ARM64)
1725 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1726
1727#else
1728# error "Port me!"
1729#endif
1730 return off;
1731}
1732
1733
1734/**
1735 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1736 * with conditional instruction.
1737 */
1738DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1739 uint8_t iGprLeft, uint8_t iGprRight)
1740{
1741#ifdef RT_ARCH_AMD64
1742 /* cmp Gv, Ev */
1743 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1744 AssertReturn(pbCodeBuf, UINT32_MAX);
1745 if (iGprLeft >= 8 || iGprRight >= 8)
1746 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1747 pbCodeBuf[off++] = 0x3b;
1748 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1749 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1750
1751#elif defined(RT_ARCH_ARM64)
1752 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1753
1754#else
1755# error "Port me!"
1756#endif
1757 return off;
1758}
1759
1760
1761
1762/*********************************************************************************************************************************
1763* Branching *
1764*********************************************************************************************************************************/
1765
1766/**
1767 * Emits a JMP rel32 / B imm19 to the given label.
1768 */
1769DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1770{
1771 Assert(idxLabel < pReNative->cLabels);
1772
1773#ifdef RT_ARCH_AMD64
1774 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1775 AssertReturn(pbCodeBuf, UINT32_MAX);
1776 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1777 {
1778 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
1779 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
1780 {
1781 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
1782 pbCodeBuf[off++] = (uint8_t)offRel;
1783 }
1784 else
1785 {
1786 offRel -= 3;
1787 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1788 pbCodeBuf[off++] = RT_BYTE1(offRel);
1789 pbCodeBuf[off++] = RT_BYTE2(offRel);
1790 pbCodeBuf[off++] = RT_BYTE3(offRel);
1791 pbCodeBuf[off++] = RT_BYTE4(offRel);
1792 }
1793 }
1794 else
1795 {
1796 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1797 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1798 pbCodeBuf[off++] = 0xfe;
1799 pbCodeBuf[off++] = 0xff;
1800 pbCodeBuf[off++] = 0xff;
1801 pbCodeBuf[off++] = 0xff;
1802 }
1803 pbCodeBuf[off++] = 0xcc; /* int3 poison */
1804
1805#elif defined(RT_ARCH_ARM64)
1806 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1807 AssertReturn(pu32CodeBuf, UINT32_MAX);
1808 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1809 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
1810 else
1811 {
1812 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1813 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
1814 }
1815
1816#else
1817# error "Port me!"
1818#endif
1819 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1820 return off;
1821}
1822
1823
1824/**
1825 * Emits a JMP rel32 / B imm19 to a new undefined label.
1826 */
1827DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1828 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1829{
1830 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1831 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1832 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
1833}
1834
1835/** Condition type. */
1836#ifdef RT_ARCH_AMD64
1837typedef enum IEMNATIVEINSTRCOND : uint8_t
1838{
1839 kIemNativeInstrCond_o = 0,
1840 kIemNativeInstrCond_no,
1841 kIemNativeInstrCond_c,
1842 kIemNativeInstrCond_nc,
1843 kIemNativeInstrCond_e,
1844 kIemNativeInstrCond_ne,
1845 kIemNativeInstrCond_be,
1846 kIemNativeInstrCond_nbe,
1847 kIemNativeInstrCond_s,
1848 kIemNativeInstrCond_ns,
1849 kIemNativeInstrCond_p,
1850 kIemNativeInstrCond_np,
1851 kIemNativeInstrCond_l,
1852 kIemNativeInstrCond_nl,
1853 kIemNativeInstrCond_le,
1854 kIemNativeInstrCond_nle
1855} IEMNATIVEINSTRCOND;
1856#elif defined(RT_ARCH_ARM64)
1857typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
1858#else
1859# error "Port me!"
1860#endif
1861
1862
1863/**
1864 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
1865 */
1866DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1867 uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
1868{
1869 Assert(idxLabel < pReNative->cLabels);
1870
1871#ifdef RT_ARCH_AMD64
1872 /* jcc rel32 */
1873 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1874 AssertReturn(pbCodeBuf, UINT32_MAX);
1875 pbCodeBuf[off++] = 0x0f;
1876 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
1877 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1878 pbCodeBuf[off++] = 0x00;
1879 pbCodeBuf[off++] = 0x00;
1880 pbCodeBuf[off++] = 0x00;
1881 pbCodeBuf[off++] = 0x00;
1882
1883#elif defined(RT_ARCH_ARM64)
1884 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1885 AssertReturn(pu32CodeBuf, UINT32_MAX);
1886 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1887 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
1888
1889#else
1890# error "Port me!"
1891#endif
1892 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1893 return off;
1894}
1895
1896
1897/**
1898 * Emits a Jcc rel32 / B.cc imm19 to a new label.
1899 */
1900DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1901 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
1902{
1903 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1904 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1905 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
1906}
1907
1908
1909/**
1910 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
1911 */
1912DECLINLINE(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1913{
1914#ifdef RT_ARCH_AMD64
1915 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
1916#elif defined(RT_ARCH_ARM64)
1917 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
1918#else
1919# error "Port me!"
1920#endif
1921}
1922
1923/**
1924 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
1925 */
1926DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1927 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1928{
1929#ifdef RT_ARCH_AMD64
1930 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
1931#elif defined(RT_ARCH_ARM64)
1932 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
1933#else
1934# error "Port me!"
1935#endif
1936}
1937
1938
1939/**
1940 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
1941 */
1942DECLINLINE(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1943{
1944#ifdef RT_ARCH_AMD64
1945 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
1946#elif defined(RT_ARCH_ARM64)
1947 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
1948#else
1949# error "Port me!"
1950#endif
1951}
1952
1953/**
1954 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
1955 */
1956DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1957 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1958{
1959#ifdef RT_ARCH_AMD64
1960 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
1961#elif defined(RT_ARCH_ARM64)
1962 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
1963#else
1964# error "Port me!"
1965#endif
1966}
1967
1968
1969/**
1970 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
1971 */
1972DECLINLINE(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1973{
1974#ifdef RT_ARCH_AMD64
1975 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
1976#elif defined(RT_ARCH_ARM64)
1977 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
1978#else
1979# error "Port me!"
1980#endif
1981}
1982
1983/**
1984 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
1985 */
1986DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1987 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1988{
1989#ifdef RT_ARCH_AMD64
1990 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
1991#elif defined(RT_ARCH_ARM64)
1992 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
1993#else
1994# error "Port me!"
1995#endif
1996}
1997
1998
1999/**
2000 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
2001 */
2002DECLINLINE(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
2003{
2004#ifdef RT_ARCH_AMD64
2005 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
2006#elif defined(RT_ARCH_ARM64)
2007 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
2008#else
2009# error "Port me!"
2010#endif
2011}
2012
2013/**
2014 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
2015 */
2016DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2017 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2018{
2019#ifdef RT_ARCH_AMD64
2020 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2021#elif defined(RT_ARCH_ARM64)
2022 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2023#else
2024# error "Port me!"
2025#endif
2026}
2027
2028
2029/**
2030 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2031 * How @a offJmp is applied is are target specific.
2032 */
2033DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2034 int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2035{
2036#ifdef RT_ARCH_AMD64
2037 /* jcc rel32 */
2038 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2039 AssertReturn(pbCodeBuf, UINT32_MAX);
2040 if (offTarget < 128 && offTarget >= -128)
2041 {
2042 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
2043 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2044 }
2045 else
2046 {
2047 pbCodeBuf[off++] = 0x0f;
2048 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2049 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2050 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2051 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2052 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2053 }
2054
2055#elif defined(RT_ARCH_ARM64)
2056 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2057 AssertReturn(pu32CodeBuf, UINT32_MAX);
2058 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2059
2060#else
2061# error "Port me!"
2062#endif
2063 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2064 return off;
2065}
2066
2067
2068/**
2069 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2070 * How @a offJmp is applied is are target specific.
2071 */
2072DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2073{
2074#ifdef RT_ARCH_AMD64
2075 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2076#elif defined(RT_ARCH_ARM64)
2077 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2078#else
2079# error "Port me!"
2080#endif
2081}
2082
2083
2084/**
2085 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2086 * How @a offJmp is applied is are target specific.
2087 */
2088DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2089{
2090#ifdef RT_ARCH_AMD64
2091 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2092#elif defined(RT_ARCH_ARM64)
2093 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2094#else
2095# error "Port me!"
2096#endif
2097}
2098
2099
2100/**
2101 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2102 * How @a offJmp is applied is are target specific.
2103 */
2104DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2105{
2106#ifdef RT_ARCH_AMD64
2107 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2108#elif defined(RT_ARCH_ARM64)
2109 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2110#else
2111# error "Port me!"
2112#endif
2113}
2114
2115
2116/**
2117 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2118 * How @a offJmp is applied is are target specific.
2119 */
2120DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2121{
2122#ifdef RT_ARCH_AMD64
2123 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2124#elif defined(RT_ARCH_ARM64)
2125 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2126#else
2127# error "Port me!"
2128#endif
2129}
2130
2131
2132/**
2133 * Fixes up a conditional jump to a fixed label.
2134 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2135 */
2136DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2137{
2138# if defined(RT_ARCH_AMD64)
2139 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2140 if (pbCodeBuf[offFixup] != 0x0f)
2141 {
2142 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2143 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2144 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2145 }
2146 else
2147 {
2148 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2149 uint32_t const offRel32 = offTarget - (offFixup + 6);
2150 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2151 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2152 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2153 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2154 }
2155
2156# elif defined(RT_ARCH_ARM64)
2157 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2158 Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
2159 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
2160 | (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
2161
2162# endif
2163}
2164
2165
2166/**
2167 * Internal helper, don't call directly.
2168 */
2169DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2170 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel,
2171 bool fJmpIfSet)
2172{
2173 Assert(iBitNo < 64);
2174#ifdef RT_ARCH_AMD64
2175 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2176 AssertReturn(pbCodeBuf, UINT32_MAX);
2177 if (iBitNo < 8)
2178 {
2179 /* test Eb, imm8 */
2180 if (iGprSrc >= 8)
2181 pbCodeBuf[off++] = X86_OP_REX_B;
2182 pbCodeBuf[off++] = 0xf6;
2183 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2184 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2185 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2186 }
2187 else
2188 {
2189 /* bt Ev, imm8 */
2190 if (iBitNo >= 32)
2191 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2192 else if (iGprSrc >= 8)
2193 pbCodeBuf[off++] = X86_OP_REX_B;
2194 pbCodeBuf[off++] = 0x0f;
2195 pbCodeBuf[off++] = 0xba;
2196 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2197 pbCodeBuf[off++] = iBitNo;
2198 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2199 }
2200
2201#elif defined(RT_ARCH_ARM64)
2202 /* Use the TBNZ instruction here. */
2203 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2204 AssertReturn(pu32CodeBuf, UINT32_MAX);
2205 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5), UINT32_MAX);
2206 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2207
2208#else
2209# error "Port me!"
2210#endif
2211 return off;
2212}
2213
2214
2215/**
2216 * Emits a jump to @a idxLabel con the condition that bit@a iBitNo _is_ _set_ in
2217 * @a iGprSrc.
2218 *
2219 * @note On ARM64 the range is only +/-8191 instructions.
2220 */
2221DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2222 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2223{
2224 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
2225}
2226
2227
2228/**
2229 * Emits a jump to @a idxLabel con the condition that bit@a iBitNo _is_ _not_
2230 * _set_ in @a iGprSrc.
2231 *
2232 * @note On ARM64 the range is only +/-8191 instructions.
2233 */
2234DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2235 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2236{
2237 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
2238}
2239
2240
2241/**
2242 * Emits a call to a 64-bit address.
2243 */
2244DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2245{
2246#ifdef RT_ARCH_AMD64
2247 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2248
2249 /* call rax */
2250 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2251 AssertReturn(pbCodeBuf, UINT32_MAX);
2252 pbCodeBuf[off++] = 0xff;
2253 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2254
2255#elif defined(RT_ARCH_ARM64)
2256 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2257
2258 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2259 AssertReturn(pu32CodeBuf, UINT32_MAX);
2260 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2261#else
2262# error "port me"
2263#endif
2264 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2265 return off;
2266}
2267
2268
2269
2270/** @} */
2271
2272#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2273
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette