VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101537

Last change on this file since 101537 was 101537, checked in by vboxsync, 18 months ago

VMM/IEM: Undebugged code for IEM_MC_IF_EFL_BIT_SET/IEM_MC_ELSE/IEM_MC_ENDIF support (disabled). bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 82.9 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101537 2023-10-21 02:16:05Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @name Stack Frame Layout
41 *
42 * @{ */
43/** The size of the area for stack variables and spills and stuff.
44 * @note This limit is duplicated in the python script(s). */
45#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
46#ifdef RT_ARCH_AMD64
47/** Number of stack arguments slots for calls made from the frame. */
48# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
49/** An stack alignment adjustment (between non-volatile register pushes and
50 * the stack variable area, so the latter better aligned). */
51# define IEMNATIVE_FRAME_ALIGN_SIZE 8
52/** Number of any shadow arguments (spill area) for calls we make. */
53# ifdef RT_OS_WINDOWS
54# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
55# else
56# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
57# endif
58
59/** Frame pointer (RBP) relative offset of the last push. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
62# else
63# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
64# endif
65/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
66 * address for it). */
67# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
68/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
69# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
70/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
71# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
72/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
73# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
74/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
75# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
76
77# ifdef RT_OS_WINDOWS
78/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
79# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
80/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
81# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
82/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
83# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
84/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
85# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
86# endif
87
88#elif RT_ARCH_ARM64
89/** No stack argument slots, enough got 8 registers for arguments. */
90# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
91/** There are no argument spill area. */
92# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
93
94/** Number of saved registers at the top of our stack frame.
95 * This includes the return address and old frame pointer, so x19 thru x30. */
96# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
97/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
98# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
99
100/** Frame pointer (BP) relative offset of the last push. */
101# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
102
103/** Frame pointer (BP) relative offset of the stack variable area (the lowest
104 * address for it). */
105# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
106
107#else
108# error "port me"
109#endif
110/** @} */
111
112
113/** @name Fixed Register Allocation(s)
114 * @{ */
115/** @def IEMNATIVE_REG_FIXED_PVMCPU
116 * The number of the register holding the pVCpu pointer. */
117/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
118 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
119 * @note This not available on AMD64, only ARM64. */
120/** @def IEMNATIVE_REG_FIXED_TMP0
121 * Dedicated temporary register.
122 * @todo replace this by a register allocator and content tracker. */
123/** @def IEMNATIVE_REG_FIXED_MASK
124 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
125 * architecture. */
126#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
127# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
128# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
129# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
130 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
131 | RT_BIT_32(X86_GREG_xSP) \
132 | RT_BIT_32(X86_GREG_xBP) )
133
134#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
135# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
136# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
137# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
138# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
139 | RT_BIT_32(ARMV8_A64_REG_LR) \
140 | RT_BIT_32(ARMV8_A64_REG_BP) \
141 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
142 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
143 | RT_BIT_32(ARMV8_A64_REG_X18) \
144 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
145
146#else
147# error "port me"
148#endif
149/** @} */
150
151/** @name Call related registers.
152 * @{ */
153/** @def IEMNATIVE_CALL_RET_GREG
154 * The return value register. */
155/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
156 * Number of arguments in registers. */
157/** @def IEMNATIVE_CALL_ARG0_GREG
158 * The general purpose register carrying argument \#0. */
159/** @def IEMNATIVE_CALL_ARG1_GREG
160 * The general purpose register carrying argument \#1. */
161/** @def IEMNATIVE_CALL_ARG2_GREG
162 * The general purpose register carrying argument \#2. */
163/** @def IEMNATIVE_CALL_ARG3_GREG
164 * The general purpose register carrying argument \#3. */
165/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
166 * Mask of registers the callee will not save and may trash. */
167#ifdef RT_ARCH_AMD64
168# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
169
170# ifdef RT_OS_WINDOWS
171# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
172# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
173# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
174# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
175# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
176# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
177 | RT_BIT_32(X86_GREG_xCX) \
178 | RT_BIT_32(X86_GREG_xDX) \
179 | RT_BIT_32(X86_GREG_x8) \
180 | RT_BIT_32(X86_GREG_x9) \
181 | RT_BIT_32(X86_GREG_x10) \
182 | RT_BIT_32(X86_GREG_x11) )
183# else
184# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
185# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
186# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
187# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
188# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
189# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
190# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
191# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
192 | RT_BIT_32(X86_GREG_xCX) \
193 | RT_BIT_32(X86_GREG_xDX) \
194 | RT_BIT_32(X86_GREG_xDI) \
195 | RT_BIT_32(X86_GREG_xSI) \
196 | RT_BIT_32(X86_GREG_x8) \
197 | RT_BIT_32(X86_GREG_x9) \
198 | RT_BIT_32(X86_GREG_x10) \
199 | RT_BIT_32(X86_GREG_x11) )
200# endif
201
202#elif defined(RT_ARCH_ARM64)
203# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
204# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
205# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
206# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
207# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
208# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
209# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
210# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
211# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
212# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
213# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
214 | RT_BIT_32(ARMV8_A64_REG_X1) \
215 | RT_BIT_32(ARMV8_A64_REG_X2) \
216 | RT_BIT_32(ARMV8_A64_REG_X3) \
217 | RT_BIT_32(ARMV8_A64_REG_X4) \
218 | RT_BIT_32(ARMV8_A64_REG_X5) \
219 | RT_BIT_32(ARMV8_A64_REG_X6) \
220 | RT_BIT_32(ARMV8_A64_REG_X7) \
221 | RT_BIT_32(ARMV8_A64_REG_X8) \
222 | RT_BIT_32(ARMV8_A64_REG_X9) \
223 | RT_BIT_32(ARMV8_A64_REG_X10) \
224 | RT_BIT_32(ARMV8_A64_REG_X11) \
225 | RT_BIT_32(ARMV8_A64_REG_X12) \
226 | RT_BIT_32(ARMV8_A64_REG_X13) \
227 | RT_BIT_32(ARMV8_A64_REG_X14) \
228 | RT_BIT_32(ARMV8_A64_REG_X15) \
229 | RT_BIT_32(ARMV8_A64_REG_X16) \
230 | RT_BIT_32(ARMV8_A64_REG_X17) )
231
232#endif
233
234/** @} */
235
236
237/** @def IEMNATIVE_HST_GREG_COUNT
238 * Number of host general purpose registers we tracker. */
239/** @def IEMNATIVE_HST_GREG_MASK
240 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
241 * inverted register masks and such to get down to a correct set of regs. */
242#ifdef RT_ARCH_AMD64
243# define IEMNATIVE_HST_GREG_COUNT 16
244# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
245
246#elif defined(RT_ARCH_ARM64)
247# define IEMNATIVE_HST_GREG_COUNT 32
248# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
249#else
250# error "Port me!"
251#endif
252
253
254/** Native code generator label types. */
255typedef enum
256{
257 kIemNativeLabelType_Invalid = 0,
258 kIemNativeLabelType_Return,
259 kIemNativeLabelType_Else,
260 kIemNativeLabelType_Endif,
261 kIemNativeLabelType_NonZeroRetOrPassUp,
262 kIemNativeLabelType_RaiseGp0,
263 kIemNativeLabelType_End
264} IEMNATIVELABELTYPE;
265
266/** Native code generator label definition. */
267typedef struct IEMNATIVELABEL
268{
269 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
270 * the epilog. */
271 uint32_t off;
272 /** The type of label (IEMNATIVELABELTYPE). */
273 uint16_t enmType;
274 /** Additional label data, type specific. */
275 uint16_t uData;
276} IEMNATIVELABEL;
277/** Pointer to a label. */
278typedef IEMNATIVELABEL *PIEMNATIVELABEL;
279
280
281/** Native code generator fixup types. */
282typedef enum
283{
284 kIemNativeFixupType_Invalid = 0,
285#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
286 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
287 kIemNativeFixupType_Rel32,
288#elif defined(RT_ARCH_ARM64)
289 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B, B.CC). */
290 kIemNativeFixupType_RelImm19At5,
291 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
292 kIemNativeFixupType_RelImm14At5,
293#endif
294 kIemNativeFixupType_End
295} IEMNATIVEFIXUPTYPE;
296
297/** Native code generator fixup. */
298typedef struct IEMNATIVEFIXUP
299{
300 /** Code offset of the fixup location. */
301 uint32_t off;
302 /** The IEMNATIVELABEL this is a fixup for. */
303 uint16_t idxLabel;
304 /** The fixup type (IEMNATIVEFIXUPTYPE). */
305 uint8_t enmType;
306 /** Addend or other data. */
307 int8_t offAddend;
308} IEMNATIVEFIXUP;
309/** Pointer to a native code generator fixup. */
310typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
311
312
313/**
314 * Guest registers that can be shadowed in GPRs.
315 */
316typedef enum IEMNATIVEGSTREG : uint8_t
317{
318 kIemNativeGstReg_GprFirst = 0,
319 kIemNativeGstReg_GprLast = 15,
320 kIemNativeGstReg_Pc,
321 kIemNativeGstReg_EFlags, /**< This one is problematic since the higher bits are used internally. */
322 /* gap: 18..23 */
323 kIemNativeGstReg_SegSelFirst = 24,
324 kIemNativeGstReg_SegSelLast = 29,
325 kIemNativeGstReg_SegBaseFirst = 30,
326 kIemNativeGstReg_SegBaseLast = 35,
327 kIemNativeGstReg_SegLimitFirst = 36,
328 kIemNativeGstReg_SegLimitLast = 41,
329 kIemNativeGstReg_End
330} IEMNATIVEGSTREG;
331
332/**
333 * Guest registers (classes) that can be referenced.
334 */
335typedef enum IEMNATIVEGSTREGREF : uint8_t
336{
337 kIemNativeGstRegRef_Invalid = 0,
338 kIemNativeGstRegRef_Gpr,
339 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
340 kIemNativeGstRegRef_EFlags,
341 kIemNativeGstRegRef_MxCsr,
342 kIemNativeGstRegRef_FpuReg,
343 kIemNativeGstRegRef_MReg,
344 kIemNativeGstRegRef_XReg,
345 kIemNativeGstRegRef_YReg,
346 kIemNativeGstRegRef_End
347} IEMNATIVEGSTREGREF;
348
349
350/** Variable kinds. */
351typedef enum IEMNATIVEVARKIND : uint8_t
352{
353 /** Customary invalid zero value. */
354 kIemNativeVarKind_Invalid = 0,
355 /** This is either in a register or on the stack. */
356 kIemNativeVarKind_Stack,
357 /** Immediate value - loaded into register when needed, or can live on the
358 * stack if referenced (in theory). */
359 kIemNativeVarKind_Immediate,
360 /** Variable reference - loaded into register when needed, never stack. */
361 kIemNativeVarKind_VarRef,
362 /** Guest register reference - loaded into register when needed, never stack. */
363 kIemNativeVarKind_GstRegRef,
364 /** End of valid values. */
365 kIemNativeVarKind_End
366} IEMNATIVEVARKIND;
367
368
369/** Variable or argument. */
370typedef struct IEMNATIVEVAR
371{
372 /** The kind of variable. */
373 IEMNATIVEVARKIND enmKind;
374 /** The variable size in bytes. */
375 uint8_t cbVar;
376 /** The first stack slot (uint64_t), except for immediate and references
377 * where it usually is UINT8_MAX. */
378 uint8_t idxStackSlot;
379 /** The host register allocated for the variable, UINT8_MAX if not. */
380 uint8_t idxReg;
381 /** The argument number if argument, UINT8_MAX if regular variable. */
382 uint8_t uArgNo;
383 /** If referenced, the index of the variable referencing this one, otherwise
384 * UINT8_MAX. A referenced variable must only be placed on the stack and
385 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
386 uint8_t idxReferrerVar;
387 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
388 IEMNATIVEGSTREG enmGstReg;
389 uint8_t bAlign;
390
391 union
392 {
393 /** kIemNativeVarKind_Immediate: The immediate value. */
394 uint64_t uValue;
395 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
396 uint8_t idxRefVar;
397 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
398 struct
399 {
400 /** The class of register. */
401 IEMNATIVEGSTREGREF enmClass;
402 /** Index within the class. */
403 uint8_t idx;
404 } GstRegRef;
405 } u;
406} IEMNATIVEVAR;
407
408/** What is being kept in a host register. */
409typedef enum IEMNATIVEWHAT : uint8_t
410{
411 /** The traditional invalid zero value. */
412 kIemNativeWhat_Invalid = 0,
413 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
414 kIemNativeWhat_Var,
415 /** Temporary register, this is typically freed when a MC completes. */
416 kIemNativeWhat_Tmp,
417 /** Call argument w/o a variable mapping. This is free (via
418 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
419 kIemNativeWhat_Arg,
420 /** Return status code.
421 * @todo not sure if we need this... */
422 kIemNativeWhat_rc,
423 /** The fixed pVCpu (PVMCPUCC) register.
424 * @todo consider offsetting this on amd64 to use negative offsets to access
425 * more members using 8-byte disp. */
426 kIemNativeWhat_pVCpuFixed,
427 /** The fixed pCtx (PCPUMCTX) register.
428 * @todo consider offsetting this on amd64 to use negative offsets to access
429 * more members using 8-byte disp. */
430 kIemNativeWhat_pCtxFixed,
431 /** Fixed temporary register. */
432 kIemNativeWhat_FixedTmp,
433 /** Register reserved by the CPU or OS architecture. */
434 kIemNativeWhat_FixedReserved,
435 /** End of valid values. */
436 kIemNativeWhat_End
437} IEMNATIVEWHAT;
438
439/**
440 * Host general register entry.
441 *
442 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
443 *
444 * @todo Track immediate values in host registers similarlly to how we track the
445 * guest register shadow copies. For it to be real helpful, though,
446 * we probably need to know which will be reused and put them into
447 * non-volatile registers, otherwise it's going to be more or less
448 * restricted to an instruction or two.
449 */
450typedef struct IEMNATIVEHSTREG
451{
452 /** Set of guest registers this one shadows.
453 *
454 * Using a bitmap here so we can designate the same host register as a copy
455 * for more than one guest register. This is expected to be useful in
456 * situations where one value is copied to several registers in a sequence.
457 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
458 * sequence we'd want to let this register follow to be a copy of and there
459 * will always be places where we'd be picking the wrong one.
460 */
461 uint64_t fGstRegShadows;
462 /** What is being kept in this register. */
463 IEMNATIVEWHAT enmWhat;
464 /** Variable index if holding a variable, otherwise UINT8_MAX. */
465 uint8_t idxVar;
466 /** Alignment padding. */
467 uint8_t abAlign[6];
468} IEMNATIVEHSTREG;
469
470
471/**
472 * Conditional stack entry.
473 */
474typedef struct IEMNATIVECOND
475{
476 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
477 bool fInElse;
478 /** The label for the IEM_MC_ELSE. */
479 uint32_t idxLabelElse;
480 /** The label for the IEM_MC_ENDIF. */
481 uint32_t idxLabelEndIf;
482} IEMNATIVECOND;
483/** Pointer to a condition stack entry. */
484typedef IEMNATIVECOND *PIEMNATIVECOND;
485
486
487/**
488 * Native recompiler state.
489 */
490typedef struct IEMRECOMPILERSTATE
491{
492 /** Size of the buffer that pbNativeRecompileBufR3 points to in
493 * IEMNATIVEINSTR units. */
494 uint32_t cInstrBufAlloc;
495#ifdef VBOX_STRICT
496 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
497 uint32_t offInstrBufChecked;
498#else
499 uint32_t uPadding; /* We don't keep track of the size here... */
500#endif
501 /** Fixed temporary code buffer for native recompilation. */
502 PIEMNATIVEINSTR pInstrBuf;
503
504 /** Bitmaps with the label types used. */
505 uint64_t bmLabelTypes;
506 /** Actual number of labels in paLabels. */
507 uint32_t cLabels;
508 /** Max number of entries allowed in paLabels before reallocating it. */
509 uint32_t cLabelsAlloc;
510 /** Labels defined while recompiling (referenced by fixups). */
511 PIEMNATIVELABEL paLabels;
512
513 /** Actual number of fixups paFixups. */
514 uint32_t cFixups;
515 /** Max number of entries allowed in paFixups before reallocating it. */
516 uint32_t cFixupsAlloc;
517 /** Buffer used by the recompiler for recording fixups when generating code. */
518 PIEMNATIVEFIXUP paFixups;
519
520 /** The translation block being recompiled. */
521 PCIEMTB pTbOrg;
522
523 /** Allocation bitmap fro aHstRegs. */
524 uint32_t bmHstRegs;
525
526 /** Bitmap marking which host register contains guest register shadow copies.
527 * This is used during register allocation to try preserve copies. */
528 uint32_t bmHstRegsWithGstShadow;
529 /** Bitmap marking valid entries in aidxGstRegShadows. */
530 uint64_t bmGstRegShadows;
531
532 /** The current condition stack depth (aCondStack). */
533 uint8_t cCondDepth;
534 uint8_t bPadding;
535 /** Condition sequence number (for generating unique labels). */
536 uint16_t uCondSeqNo;
537
538 /** Allocation bitmap for aVars. */
539 uint32_t bmVars;
540 union
541 {
542 /** Index of variable arguments, UINT8_MAX if not valid. */
543 uint8_t aidxArgVars[8];
544 /** For more efficient resetting. */
545 uint64_t u64ArgVars;
546 };
547
548 /** Host register allocation tracking. */
549 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
550 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
551 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
552 * (A shadow copy of a guest register can only be held in a one host register,
553 * there are no duplicate copies or ambiguities like that). */
554 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
555
556 /** The condition nesting stack. */
557 IEMNATIVECOND aCondStack[4];
558
559 /** Variables and arguments. */
560 IEMNATIVEVAR aVars[16];
561} IEMRECOMPILERSTATE;
562/** Pointer to a native recompiler state. */
563typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
564
565
566/**
567 * Native recompiler worker for a threaded function.
568 *
569 * @returns New code buffer offset, UINT32_MAX in case of failure.
570 * @param pReNative The native recompiler state.
571 * @param off The current code buffer offset.
572 * @param pCallEntry The threaded call entry.
573 *
574 * @note This is not allowed to throw anything atm.
575 */
576typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
577 PCIEMTHRDEDCALLENTRY pCallEntry));
578/** Pointer to a native recompiler worker for a threaded function. */
579typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
580
581/** Defines a native recompiler worker for a threaded function. */
582#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
583 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
584/** Prototypes a native recompiler function for a threaded function. */
585#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
586
587
588DECLHIDDEN(uint32_t) iemNativeMakeLabel(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
589 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
590DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
591 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
592DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
593 uint32_t cInstrReq) RT_NOEXCEPT;
594
595DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
596 bool fPreferVolatile = true) RT_NOEXCEPT;
597DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
598 bool fPreferVolatile = true) RT_NOEXCEPT;
599DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
600 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
601DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
602DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
603DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
604DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
605DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
606DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
607DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
608DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
609
610DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
611 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
612DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
613 uint8_t idxInstr) RT_NOEXCEPT;
614
615
616/**
617 * Ensures that there is sufficient space in the instruction output buffer.
618 *
619 * This will reallocate the buffer if needed and allowed.
620 *
621 * @returns Pointer to the instruction output buffer on success, NULL on
622 * failure.
623 * @param pReNative The native recompile state.
624 * @param off Current instruction offset. Works safely for UINT32_MAX
625 * as well.
626 * @param cInstrReq Number of instruction about to be added. It's okay to
627 * overestimate this a bit.
628 */
629DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
630{
631 uint64_t const offChecked = off + (uint64_t)cInstrReq;
632 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
633 {
634#ifdef VBOX_STRICT
635 pReNative->offInstrBufChecked = offChecked;
636#endif
637 return pReNative->pInstrBuf;
638 }
639 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
640}
641
642/**
643 * Checks that we didn't exceed the space requested in the last
644 * iemNativeInstrBufEnsure() call. */
645#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
646 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
647 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
648
649
650/**
651 * Emit a simple marker instruction to more easily tell where something starts
652 * in the disassembly.
653 */
654DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
655{
656#ifdef RT_ARCH_AMD64
657 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
658 AssertReturn(pbCodeBuf, UINT32_MAX);
659 if (uInfo == 0)
660 {
661 /* nop */
662 pbCodeBuf[off++] = 0x90;
663 }
664 else
665 {
666 /* nop [disp32] */
667 pbCodeBuf[off++] = 0x0f;
668 pbCodeBuf[off++] = 0x1f;
669 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
670 pbCodeBuf[off++] = RT_BYTE1(uInfo);
671 pbCodeBuf[off++] = RT_BYTE2(uInfo);
672 pbCodeBuf[off++] = RT_BYTE3(uInfo);
673 pbCodeBuf[off++] = RT_BYTE4(uInfo);
674 }
675#elif RT_ARCH_ARM64
676 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
677 AssertReturn(pu32CodeBuf, UINT32_MAX);
678 /* nop */
679 pu32CodeBuf[off++] = 0xd503201f;
680
681 RT_NOREF(uInfo);
682#else
683# error "port me"
684#endif
685 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
686 return off;
687}
688
689
690/*********************************************************************************************************************************
691* Loads, Stores and Related Stuff. *
692*********************************************************************************************************************************/
693
694/**
695 * Emits setting a GPR to zero.
696 */
697DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
698{
699#ifdef RT_ARCH_AMD64
700 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
701 AssertReturn(pbCodeBuf, UINT32_MAX);
702 /* xor gpr32, gpr32 */
703 if (iGpr >= 8)
704 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
705 pbCodeBuf[off++] = 0x33;
706 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
707
708#elif RT_ARCH_ARM64
709 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
710 AssertReturn(pu32CodeBuf, UINT32_MAX);
711 /* mov gpr, #0x0 */
712 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
713
714#else
715# error "port me"
716#endif
717 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
718 return off;
719}
720
721
722/**
723 * Emits loading a constant into a 64-bit GPR
724 */
725DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
726{
727 if (!uImm64)
728 return iemNativeEmitGprZero(pReNative, off, iGpr);
729
730#ifdef RT_ARCH_AMD64
731 if (uImm64 <= UINT32_MAX)
732 {
733 /* mov gpr, imm32 */
734 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
735 AssertReturn(pbCodeBuf, UINT32_MAX);
736 if (iGpr >= 8)
737 pbCodeBuf[off++] = X86_OP_REX_B;
738 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
739 pbCodeBuf[off++] = RT_BYTE1(uImm64);
740 pbCodeBuf[off++] = RT_BYTE2(uImm64);
741 pbCodeBuf[off++] = RT_BYTE3(uImm64);
742 pbCodeBuf[off++] = RT_BYTE4(uImm64);
743 }
744 else
745 {
746 /* mov gpr, imm64 */
747 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
748 AssertReturn(pbCodeBuf, UINT32_MAX);
749 if (iGpr < 8)
750 pbCodeBuf[off++] = X86_OP_REX_W;
751 else
752 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
753 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
754 pbCodeBuf[off++] = RT_BYTE1(uImm64);
755 pbCodeBuf[off++] = RT_BYTE2(uImm64);
756 pbCodeBuf[off++] = RT_BYTE3(uImm64);
757 pbCodeBuf[off++] = RT_BYTE4(uImm64);
758 pbCodeBuf[off++] = RT_BYTE5(uImm64);
759 pbCodeBuf[off++] = RT_BYTE6(uImm64);
760 pbCodeBuf[off++] = RT_BYTE7(uImm64);
761 pbCodeBuf[off++] = RT_BYTE8(uImm64);
762 }
763
764#elif RT_ARCH_ARM64
765 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
766 AssertReturn(pu32CodeBuf, UINT32_MAX);
767
768 /*
769 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
770 * supply remaining bits using 'movk grp, imm16, lsl #x'.
771 *
772 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
773 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
774 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
775 * after the first non-zero immediate component so we switch to movk for
776 * the remainder.
777 */
778 uint32_t fMovK = 0;
779 /* mov gpr, imm16 */
780 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
781 if (uImmPart)
782 {
783 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
784 fMovK |= RT_BIT_32(29);
785 }
786 /* mov[k] gpr, imm16, lsl #16 */
787 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
788 if (uImmPart)
789 {
790 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
791 fMovK |= RT_BIT_32(29);
792 }
793 /* mov[k] gpr, imm16, lsl #32 */
794 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
795 if (uImmPart)
796 {
797 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
798 fMovK |= RT_BIT_32(29);
799 }
800 /* mov[k] gpr, imm16, lsl #48 */
801 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
802 if (uImmPart)
803 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
804
805 /** @todo there is an inverted mask variant we might want to explore if it
806 * reduces the number of instructions... */
807 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
808 * clang 12.x does that, only to use the 'x' version for the
809 * addressing in the following ldr). */
810
811#else
812# error "port me"
813#endif
814 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
815 return off;
816}
817
818
819/**
820 * Emits loading a constant into a 8-bit GPR
821 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
822 * only the ARM64 version does that.
823 */
824DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
825{
826#ifdef RT_ARCH_AMD64
827 /* mov gpr, imm8 */
828 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
829 AssertReturn(pbCodeBuf, UINT32_MAX);
830 if (iGpr >= 8)
831 pbCodeBuf[off++] = X86_OP_REX_B;
832 else if (iGpr >= 4)
833 pbCodeBuf[off++] = X86_OP_REX;
834 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
835 pbCodeBuf[off++] = RT_BYTE1(uImm8);
836
837#elif RT_ARCH_ARM64
838 /* movz gpr, imm16, lsl #0 */
839 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
840 AssertReturn(pu32CodeBuf, UINT32_MAX);
841 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
842
843#else
844# error "port me"
845#endif
846 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
847 return off;
848}
849
850
851#ifdef RT_ARCH_AMD64
852/**
853 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
854 */
855DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
856{
857 if (offVCpu < 128)
858 {
859 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
860 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
861 }
862 else
863 {
864 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
865 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
866 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
867 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
868 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
869 }
870 return off;
871}
872#elif RT_ARCH_ARM64
873/**
874 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
875 */
876DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
877 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
878{
879 /*
880 * There are a couple of ldr variants that takes an immediate offset, so
881 * try use those if we can, otherwise we have to use the temporary register
882 * help with the addressing.
883 */
884 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
885 {
886 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
887 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
888 AssertReturn(pu32CodeBuf, UINT32_MAX);
889 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
890 }
891 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
892 {
893 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
894 AssertReturn(pu32CodeBuf, UINT32_MAX);
895 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
896 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
897 }
898 else
899 {
900 /* The offset is too large, so we must load it into a register and use
901 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
902 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
903 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
904
905 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
906 AssertReturn(pu32CodeBuf, UINT32_MAX);
907 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
908 }
909 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
910 return off;
911}
912#endif
913
914
915/**
916 * Emits a 64-bit GPR load of a VCpu value.
917 */
918DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
919{
920#ifdef RT_ARCH_AMD64
921 /* mov reg64, mem64 */
922 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
923 AssertReturn(pbCodeBuf, UINT32_MAX);
924 if (iGpr < 8)
925 pbCodeBuf[off++] = X86_OP_REX_W;
926 else
927 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
928 pbCodeBuf[off++] = 0x8b;
929 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
930 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
931
932#elif RT_ARCH_ARM64
933 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
934
935#else
936# error "port me"
937#endif
938 return off;
939}
940
941
942/**
943 * Emits a 32-bit GPR load of a VCpu value.
944 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
945 */
946DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
947{
948#ifdef RT_ARCH_AMD64
949 /* mov reg32, mem32 */
950 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
951 AssertReturn(pbCodeBuf, UINT32_MAX);
952 if (iGpr >= 8)
953 pbCodeBuf[off++] = X86_OP_REX_R;
954 pbCodeBuf[off++] = 0x8b;
955 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
956 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
957
958#elif RT_ARCH_ARM64
959 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
960
961#else
962# error "port me"
963#endif
964 return off;
965}
966
967
968/**
969 * Emits a 16-bit GPR load of a VCpu value.
970 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
971 */
972DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
973{
974#ifdef RT_ARCH_AMD64
975 /* movzx reg32, mem16 */
976 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
977 AssertReturn(pbCodeBuf, UINT32_MAX);
978 if (iGpr >= 8)
979 pbCodeBuf[off++] = X86_OP_REX_R;
980 pbCodeBuf[off++] = 0x0f;
981 pbCodeBuf[off++] = 0xb7;
982 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
983 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
984
985#elif RT_ARCH_ARM64
986 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
987
988#else
989# error "port me"
990#endif
991 return off;
992}
993
994
995/**
996 * Emits a 8-bit GPR load of a VCpu value.
997 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
998 */
999DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1000{
1001#ifdef RT_ARCH_AMD64
1002 /* movzx reg32, mem8 */
1003 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1004 AssertReturn(pbCodeBuf, UINT32_MAX);
1005 if (iGpr >= 8)
1006 pbCodeBuf[off++] = X86_OP_REX_R;
1007 pbCodeBuf[off++] = 0x0f;
1008 pbCodeBuf[off++] = 0xb6;
1009 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1010 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1011
1012#elif RT_ARCH_ARM64
1013 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
1014
1015#else
1016# error "port me"
1017#endif
1018 return off;
1019}
1020
1021
1022/**
1023 * Emits a store of a GPR value to a 64-bit VCpu field.
1024 */
1025DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1026{
1027#ifdef RT_ARCH_AMD64
1028 /* mov mem64, reg64 */
1029 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1030 AssertReturn(pbCodeBuf, UINT32_MAX);
1031 if (iGpr < 8)
1032 pbCodeBuf[off++] = X86_OP_REX_W;
1033 else
1034 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1035 pbCodeBuf[off++] = 0x89;
1036 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
1037 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1038
1039#elif RT_ARCH_ARM64
1040 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
1041
1042#else
1043# error "port me"
1044#endif
1045 return off;
1046}
1047
1048
1049/**
1050 * Emits a store of a GPR value to a 32-bit VCpu field.
1051 */
1052DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1053{
1054#ifdef RT_ARCH_AMD64
1055 /* mov mem32, reg32 */
1056 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1057 AssertReturn(pbCodeBuf, UINT32_MAX);
1058 if (iGpr >= 8)
1059 pbCodeBuf[off++] = X86_OP_REX_R;
1060 pbCodeBuf[off++] = 0x89;
1061 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1062 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1063
1064#elif RT_ARCH_ARM64
1065 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1066
1067#else
1068# error "port me"
1069#endif
1070 return off;
1071}
1072
1073
1074/**
1075 * Emits a store of a GPR value to a 16-bit VCpu field.
1076 */
1077DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1078{
1079#ifdef RT_ARCH_AMD64
1080 /* mov mem16, reg16 */
1081 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1082 AssertReturn(pbCodeBuf, UINT32_MAX);
1083 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1084 if (iGpr >= 8)
1085 pbCodeBuf[off++] = X86_OP_REX_R;
1086 pbCodeBuf[off++] = 0x89;
1087 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1088 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1089
1090#elif RT_ARCH_ARM64
1091 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1092
1093#else
1094# error "port me"
1095#endif
1096 return off;
1097}
1098
1099
1100/**
1101 * Emits a store of a GPR value to a 8-bit VCpu field.
1102 */
1103DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1104{
1105#ifdef RT_ARCH_AMD64
1106 /* mov mem8, reg8 */
1107 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1108 AssertReturn(pbCodeBuf, UINT32_MAX);
1109 if (iGpr >= 8)
1110 pbCodeBuf[off++] = X86_OP_REX_R;
1111 pbCodeBuf[off++] = 0x88;
1112 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1113 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1114
1115#elif RT_ARCH_ARM64
1116 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1117
1118#else
1119# error "port me"
1120#endif
1121 return off;
1122}
1123
1124
1125/**
1126 * Emits a gprdst = gprsrc load.
1127 */
1128DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1129{
1130#ifdef RT_ARCH_AMD64
1131 /* mov gprdst, gprsrc */
1132 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1133 AssertReturn(pbCodeBuf, UINT32_MAX);
1134 if ((iGprDst | iGprSrc) >= 8)
1135 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1136 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1137 : X86_OP_REX_W | X86_OP_REX_R;
1138 else
1139 pbCodeBuf[off++] = X86_OP_REX_W;
1140 pbCodeBuf[off++] = 0x8b;
1141 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1142
1143#elif RT_ARCH_ARM64
1144 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1145 AssertReturn(pu32CodeBuf, UINT32_MAX);
1146 /* mov dst, src; alias for: orr dst, xzr, src */
1147 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1148
1149#else
1150# error "port me"
1151#endif
1152 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1153 return off;
1154}
1155
1156#ifdef RT_ARCH_AMD64
1157/**
1158 * Common bit of iemNativeEmitLoadGprByBp and friends.
1159 */
1160DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,
1161 PIEMRECOMPILERSTATE pReNativeAssert)
1162{
1163 if (offDisp < 128 && offDisp >= -128)
1164 {
1165 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1166 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1167 }
1168 else
1169 {
1170 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1171 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1172 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1173 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1174 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1175 }
1176 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);
1177 return off;
1178}
1179#endif
1180
1181
1182#ifdef RT_ARCH_AMD64
1183/**
1184 * Emits a 64-bit GRP load instruction with an BP relative source address.
1185 */
1186DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1187{
1188 /* mov gprdst, qword [rbp + offDisp] */
1189 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1190 AssertReturn(pbCodeBuf, UINT32_MAX);
1191 if (iGprDst < 8)
1192 pbCodeBuf[off++] = X86_OP_REX_W;
1193 else
1194 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1195 pbCodeBuf[off++] = 0x8b;
1196 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1197}
1198#endif
1199
1200
1201#ifdef RT_ARCH_AMD64
1202/**
1203 * Emits a 32-bit GRP load instruction with an BP relative source address.
1204 */
1205DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1206{
1207 /* mov gprdst, dword [rbp + offDisp] */
1208 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1209 AssertReturn(pbCodeBuf, UINT32_MAX);
1210 if (iGprDst >= 8)
1211 pbCodeBuf[off++] = X86_OP_REX_R;
1212 pbCodeBuf[off++] = 0x8b;
1213 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1214}
1215#endif
1216
1217
1218#ifdef RT_ARCH_AMD64
1219/**
1220 * Emits a load effective address to a GRP with an BP relative source address.
1221 */
1222DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1223{
1224 /* lea gprdst, [rbp + offDisp] */
1225 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1226 AssertReturn(pbCodeBuf, UINT32_MAX);
1227 if (iGprDst < 8)
1228 pbCodeBuf[off++] = X86_OP_REX_W;
1229 else
1230 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1231 pbCodeBuf[off++] = 0x8d;
1232 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
1233}
1234#endif
1235
1236
1237/**
1238 * Emits a 64-bit GPR store with an BP relative destination address.
1239 *
1240 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1241 */
1242DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1243{
1244#ifdef RT_ARCH_AMD64
1245 /* mov qword [rbp + offDisp], gprdst */
1246 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1247 AssertReturn(pbCodeBuf, UINT32_MAX);
1248 if (iGprSrc < 8)
1249 pbCodeBuf[off++] = X86_OP_REX_W;
1250 else
1251 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1252 pbCodeBuf[off++] = 0x89;
1253 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);
1254
1255#elif defined(RT_ARCH_ARM64)
1256 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1257 {
1258 /* str w/ unsigned imm12 (scaled) */
1259 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1260 AssertReturn(pu32CodeBuf, UINT32_MAX);
1261 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1262 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1263 }
1264 else if (offDisp >= -256 && offDisp <= 256)
1265 {
1266 /* stur w/ signed imm9 (unscaled) */
1267 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1268 AssertReturn(pu32CodeBuf, UINT32_MAX);
1269 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1270 }
1271 else
1272 {
1273 /* Use temporary indexing register. */
1274 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1275 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1276 AssertReturn(pu32CodeBuf, UINT32_MAX);
1277 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1278 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1279 }
1280 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1281 return off;
1282
1283#else
1284# error "Port me!"
1285#endif
1286}
1287
1288
1289/**
1290 * Emits a 64-bit immediate store with an BP relative destination address.
1291 *
1292 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1293 */
1294DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1295{
1296#ifdef RT_ARCH_AMD64
1297 if ((int64_t)uImm64 == (int32_t)uImm64)
1298 {
1299 /* mov qword [rbp + offDisp], imm32 - sign extended */
1300 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1301 AssertReturn(pbCodeBuf, UINT32_MAX);
1302
1303 pbCodeBuf[off++] = X86_OP_REX_W;
1304 pbCodeBuf[off++] = 0xc7;
1305 if (offDisp < 128 && offDisp >= -128)
1306 {
1307 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1308 pbCodeBuf[off++] = (uint8_t)offDisp;
1309 }
1310 else
1311 {
1312 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1313 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1314 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1315 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1316 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1317 }
1318 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1319 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1320 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1321 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1322 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1323 return off;
1324 }
1325#endif
1326
1327 /* Load tmp0, imm64; Store tmp to bp+disp. */
1328 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1329 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1330}
1331
1332
1333/*********************************************************************************************************************************
1334* Subtraction and Additions *
1335*********************************************************************************************************************************/
1336
1337
1338#ifdef RT_ARCH_AMD64
1339/**
1340 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1341 */
1342DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1343{
1344 /* sub gprdst, imm8/imm32 */
1345 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1346 AssertReturn(pbCodeBuf, UINT32_MAX);
1347 if (iGprDst < 8)
1348 pbCodeBuf[off++] = X86_OP_REX_W;
1349 else
1350 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1351 if (iSubtrahend < 128 && iSubtrahend >= -128)
1352 {
1353 pbCodeBuf[off++] = 0x83;
1354 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1355 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1356 }
1357 else
1358 {
1359 pbCodeBuf[off++] = 0x81;
1360 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1361 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1362 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1363 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1364 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1365 }
1366 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1367 return off;
1368}
1369#endif
1370
1371
1372/**
1373 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1374 * @note The AMD64 version sets flags.
1375 */
1376DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1377{
1378#if defined(RT_ARCH_AMD64)
1379 /* add Gv,Ev */
1380 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1381 AssertReturn(pbCodeBuf, UINT32_MAX);
1382 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1383 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1384 pbCodeBuf[off++] = 0x04;
1385 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1386
1387#elif defined(RT_ARCH_ARM64)
1388 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1389 AssertReturn(pu32CodeBuf, UINT32_MAX);
1390 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1391
1392#else
1393# error "Port me"
1394#endif
1395 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1396 return off;
1397}
1398
1399
1400/**
1401 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1402 */
1403DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1404{
1405#if defined(RT_ARCH_AMD64)
1406 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1407 AssertReturn(pbCodeBuf, UINT32_MAX);
1408 /* add or inc */
1409 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1410 if (iImm8 != 1)
1411 {
1412 pbCodeBuf[off++] = 0x83;
1413 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1414 pbCodeBuf[off++] = (uint8_t)iImm8;
1415 }
1416 else
1417 {
1418 pbCodeBuf[off++] = 0xff;
1419 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1420 }
1421
1422#elif defined(RT_ARCH_ARM64)
1423 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1424 AssertReturn(pu32CodeBuf, UINT32_MAX);
1425 if (iImm8 >= 0)
1426 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1427 else
1428 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1429
1430#else
1431# error "Port me"
1432#endif
1433 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1434 return off;
1435}
1436
1437
1438/**
1439 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1440 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1441 */
1442DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1443{
1444#if defined(RT_ARCH_AMD64)
1445 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1446 AssertReturn(pbCodeBuf, UINT32_MAX);
1447 /* add or inc */
1448 if (iGprDst >= 8)
1449 pbCodeBuf[off++] = X86_OP_REX_B;
1450 if (iImm8 != 1)
1451 {
1452 pbCodeBuf[off++] = 0x83;
1453 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1454 pbCodeBuf[off++] = (uint8_t)iImm8;
1455 }
1456 else
1457 {
1458 pbCodeBuf[off++] = 0xff;
1459 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1460 }
1461
1462#elif defined(RT_ARCH_ARM64)
1463 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1464 AssertReturn(pu32CodeBuf, UINT32_MAX);
1465 if (iImm8 >= 0)
1466 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1467 else
1468 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1469
1470#else
1471# error "Port me"
1472#endif
1473 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1474 return off;
1475}
1476
1477
1478/**
1479 * Emits a 64-bit GPR additions with a 64-bit signed addend.
1480 */
1481DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1482{
1483#if defined(RT_ARCH_AMD64)
1484 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1485 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1486
1487 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1488 {
1489 /* add grp, imm32 */
1490 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1491 AssertReturn(pbCodeBuf, UINT32_MAX);
1492 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1493 pbCodeBuf[off++] = 0x81;
1494 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1495 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1496 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1497 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1498 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1499 }
1500 else
1501 {
1502 /* Best to use a temporary register to deal with this in the simplest way: */
1503 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1504 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1505
1506 /* add dst, tmpreg */
1507 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1508 AssertReturn(pbCodeBuf, UINT32_MAX);
1509 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1510 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1511 pbCodeBuf[off++] = 0x03;
1512 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1513
1514 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1515 }
1516
1517#elif defined(RT_ARCH_ARM64)
1518 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1519 {
1520 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1521 AssertReturn(pu32CodeBuf, UINT32_MAX);
1522 if (iAddend >= 0)
1523 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
1524 else
1525 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
1526 }
1527 else
1528 {
1529 /* Use temporary register for the immediate. */
1530 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1531 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1532
1533 /* add gprdst, gprdst, tmpreg */
1534 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1535 AssertReturn(pu32CodeBuf, UINT32_MAX);
1536 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
1537
1538 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1539 }
1540
1541#else
1542# error "Port me"
1543#endif
1544 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1545 return off;
1546}
1547
1548
1549/**
1550 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
1551 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1552 */
1553DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1554{
1555#if defined(RT_ARCH_AMD64)
1556 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1557 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1558
1559 /* add grp, imm32 */
1560 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1561 AssertReturn(pbCodeBuf, UINT32_MAX);
1562 if (iGprDst >= 8)
1563 pbCodeBuf[off++] = X86_OP_REX_B;
1564 pbCodeBuf[off++] = 0x81;
1565 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1566 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1567 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1568 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1569 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1570
1571#elif defined(RT_ARCH_ARM64)
1572 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1573 {
1574 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1575 AssertReturn(pu32CodeBuf, UINT32_MAX);
1576 if (iAddend >= 0)
1577 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
1578 else
1579 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
1580 }
1581 else
1582 {
1583 /* Use temporary register for the immediate. */
1584 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1585 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1586
1587 /* add gprdst, gprdst, tmpreg */
1588 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1589 AssertReturn(pu32CodeBuf, UINT32_MAX);
1590 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
1591
1592 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1593 }
1594
1595#else
1596# error "Port me"
1597#endif
1598 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1599 return off;
1600}
1601
1602
1603
1604/*********************************************************************************************************************************
1605* Bit Operations *
1606*********************************************************************************************************************************/
1607
1608/**
1609 * Emits code for clearing bits 16 thru 63 in the GPR.
1610 */
1611DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1612{
1613#if defined(RT_ARCH_AMD64)
1614 /* movzx reg32, reg16 */
1615 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1616 AssertReturn(pbCodeBuf, UINT32_MAX);
1617 if (iGprDst >= 8)
1618 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
1619 pbCodeBuf[off++] = 0x0f;
1620 pbCodeBuf[off++] = 0xb7;
1621 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1622
1623#elif defined(RT_ARCH_ARM64)
1624 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1625 AssertReturn(pu32CodeBuf, UINT32_MAX);
1626# if 1
1627 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1628# else
1629 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1630 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1631# endif
1632#else
1633# error "Port me"
1634#endif
1635 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1636 return off;
1637}
1638
1639
1640/**
1641 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1642 */
1643DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1644{
1645#if defined(RT_ARCH_AMD64)
1646 /* shr dst, cShift */
1647 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1648 AssertReturn(pbCodeBuf, UINT32_MAX);
1649 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1650 pbCodeBuf[off++] = 0xc0;
1651 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1652 pbCodeBuf[off++] = cShift;
1653 Assert(cShift > 0 && cShift < 64);
1654
1655#elif defined(RT_ARCH_ARM64)
1656 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1657 AssertReturn(pu32CodeBuf, UINT32_MAX);
1658 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1659#else
1660# error "Port me"
1661#endif
1662 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1663 return off;
1664}
1665
1666
1667/*********************************************************************************************************************************
1668* Compare and Testing *
1669*********************************************************************************************************************************/
1670
1671
1672#ifdef RT_ARCH_ARM64
1673/**
1674 * Emits an ARM64 compare instruction.
1675 */
1676DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1677 bool f64Bit = true, uint32_t cShift = 0,
1678 ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1679{
1680 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1681 AssertReturn(pu32CodeBuf, UINT32_MAX);
1682 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
1683 f64Bit, true /*fSetFlags*/, cShift, enmShift);
1684 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1685 return off;
1686}
1687#endif
1688
1689
1690/**
1691 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1692 * with conditional instruction.
1693 */
1694DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1695{
1696#ifdef RT_ARCH_AMD64
1697 /* cmp Gv, Ev */
1698 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1699 AssertReturn(pbCodeBuf, UINT32_MAX);
1700 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1701 pbCodeBuf[off++] = 0x3b;
1702 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1703 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1704
1705#elif defined(RT_ARCH_ARM64)
1706 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1707
1708#else
1709# error "Port me!"
1710#endif
1711 return off;
1712}
1713
1714
1715/**
1716 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1717 * with conditional instruction.
1718 */
1719DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1720 uint8_t iGprLeft, uint8_t iGprRight)
1721{
1722#ifdef RT_ARCH_AMD64
1723 /* cmp Gv, Ev */
1724 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1725 AssertReturn(pbCodeBuf, UINT32_MAX);
1726 if (iGprLeft >= 8 || iGprRight >= 8)
1727 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1728 pbCodeBuf[off++] = 0x3b;
1729 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1730 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1731
1732#elif defined(RT_ARCH_ARM64)
1733 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1734
1735#else
1736# error "Port me!"
1737#endif
1738 return off;
1739}
1740
1741
1742
1743/*********************************************************************************************************************************
1744* Branching *
1745*********************************************************************************************************************************/
1746
1747/**
1748 * Emits a JMP rel32 / B imm19 to the given label.
1749 */
1750DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1751{
1752 Assert(idxLabel < pReNative->cLabels);
1753
1754#ifdef RT_ARCH_AMD64
1755 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1756 AssertReturn(pbCodeBuf, UINT32_MAX);
1757 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1758 {
1759 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
1760 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
1761 {
1762 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
1763 pbCodeBuf[off++] = (uint8_t)offRel;
1764 off++;
1765 }
1766 else
1767 {
1768 offRel -= 3;
1769 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1770 pbCodeBuf[off++] = RT_BYTE1(offRel);
1771 pbCodeBuf[off++] = RT_BYTE2(offRel);
1772 pbCodeBuf[off++] = RT_BYTE3(offRel);
1773 pbCodeBuf[off++] = RT_BYTE4(offRel);
1774 }
1775 }
1776 else
1777 {
1778 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1779 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1780 pbCodeBuf[off++] = 0xfe;
1781 pbCodeBuf[off++] = 0xff;
1782 pbCodeBuf[off++] = 0xff;
1783 pbCodeBuf[off++] = 0xff;
1784 }
1785 pbCodeBuf[off++] = 0xcc; /* int3 poison */
1786
1787#elif defined(RT_ARCH_ARM64)
1788 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1789 AssertReturn(pu32CodeBuf, UINT32_MAX);
1790 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1791 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
1792 else
1793 {
1794 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1795 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
1796 }
1797
1798#else
1799# error "Port me!"
1800#endif
1801 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1802 return off;
1803}
1804
1805
1806/**
1807 * Emits a JMP rel32 / B imm19 to a new undefined label.
1808 */
1809DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1810 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1811{
1812 uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1813 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1814 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
1815}
1816
1817/** Condition type. */
1818#ifdef RT_ARCH_AMD64
1819typedef enum IEMNATIVEINSTRCOND : uint8_t
1820{
1821 kIemNativeInstrCond_o = 0,
1822 kIemNativeInstrCond_no,
1823 kIemNativeInstrCond_c,
1824 kIemNativeInstrCond_nc,
1825 kIemNativeInstrCond_e,
1826 kIemNativeInstrCond_ne,
1827 kIemNativeInstrCond_be,
1828 kIemNativeInstrCond_nbe,
1829 kIemNativeInstrCond_s,
1830 kIemNativeInstrCond_ns,
1831 kIemNativeInstrCond_p,
1832 kIemNativeInstrCond_np,
1833 kIemNativeInstrCond_l,
1834 kIemNativeInstrCond_nl,
1835 kIemNativeInstrCond_le,
1836 kIemNativeInstrCond_nle
1837} IEMNATIVEINSTRCOND;
1838#elif defined(RT_ARCH_ARM64)
1839typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
1840#else
1841# error "Port me!"
1842#endif
1843
1844
1845/**
1846 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
1847 */
1848DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1849 uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
1850{
1851 Assert(idxLabel < pReNative->cLabels);
1852
1853#ifdef RT_ARCH_AMD64
1854 /* jcc rel32 */
1855 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1856 AssertReturn(pbCodeBuf, UINT32_MAX);
1857 pbCodeBuf[off++] = 0x0f;
1858 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
1859 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1860 pbCodeBuf[off++] = 0x00;
1861 pbCodeBuf[off++] = 0x00;
1862 pbCodeBuf[off++] = 0x00;
1863 pbCodeBuf[off++] = 0x00;
1864
1865#elif defined(RT_ARCH_ARM64)
1866 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1867 AssertReturn(pu32CodeBuf, UINT32_MAX);
1868 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1869 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
1870
1871#else
1872# error "Port me!"
1873#endif
1874 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1875 return off;
1876}
1877
1878
1879/**
1880 * Emits a Jcc rel32 / B.cc imm19 to a new label.
1881 */
1882DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1883 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
1884{
1885 uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1886 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1887 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
1888}
1889
1890
1891/**
1892 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.
1893 */
1894DECLINLINE(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1895{
1896#ifdef RT_ARCH_AMD64
1897 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);
1898#elif defined(RT_ARCH_ARM64)
1899 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);
1900#else
1901# error "Port me!"
1902#endif
1903}
1904
1905/**
1906 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
1907 */
1908DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1909 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1910{
1911#ifdef RT_ARCH_AMD64
1912 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);
1913#elif defined(RT_ARCH_ARM64)
1914 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
1915#else
1916# error "Port me!"
1917#endif
1918}
1919
1920
1921/**
1922 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.
1923 */
1924DECLINLINE(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1925{
1926#ifdef RT_ARCH_AMD64
1927 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);
1928#elif defined(RT_ARCH_ARM64)
1929 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);
1930#else
1931# error "Port me!"
1932#endif
1933}
1934
1935/**
1936 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
1937 */
1938DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1939 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1940{
1941#ifdef RT_ARCH_AMD64
1942 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);
1943#elif defined(RT_ARCH_ARM64)
1944 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
1945#else
1946# error "Port me!"
1947#endif
1948}
1949
1950
1951/**
1952 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.
1953 */
1954DECLINLINE(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1955{
1956#ifdef RT_ARCH_AMD64
1957 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);
1958#elif defined(RT_ARCH_ARM64)
1959 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);
1960#else
1961# error "Port me!"
1962#endif
1963}
1964
1965/**
1966 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
1967 */
1968DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1969 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1970{
1971#ifdef RT_ARCH_AMD64
1972 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);
1973#elif defined(RT_ARCH_ARM64)
1974 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
1975#else
1976# error "Port me!"
1977#endif
1978}
1979
1980
1981/**
1982 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.
1983 */
1984DECLINLINE(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1985{
1986#ifdef RT_ARCH_AMD64
1987 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);
1988#elif defined(RT_ARCH_ARM64)
1989 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);
1990#else
1991# error "Port me!"
1992#endif
1993}
1994
1995/**
1996 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
1997 */
1998DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1999 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
2000{
2001#ifdef RT_ARCH_AMD64
2002 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);
2003#elif defined(RT_ARCH_ARM64)
2004 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
2005#else
2006# error "Port me!"
2007#endif
2008}
2009
2010
2011/**
2012 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
2013 * How @a offJmp is applied is are target specific.
2014 */
2015DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2016 int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
2017{
2018#ifdef RT_ARCH_AMD64
2019 /* jcc rel32 */
2020 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
2021 AssertReturn(pbCodeBuf, UINT32_MAX);
2022 if (offTarget < 128 && offTarget >= -128)
2023 {
2024 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
2025 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2026 }
2027 else
2028 {
2029 pbCodeBuf[off++] = 0x0f;
2030 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
2031 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
2032 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
2033 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
2034 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
2035 }
2036
2037#elif defined(RT_ARCH_ARM64)
2038 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2039 AssertReturn(pu32CodeBuf, UINT32_MAX);
2040 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
2041
2042#else
2043# error "Port me!"
2044#endif
2045 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2046 return off;
2047}
2048
2049
2050/**
2051 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
2052 * How @a offJmp is applied is are target specific.
2053 */
2054DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2055{
2056#ifdef RT_ARCH_AMD64
2057 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);
2058#elif defined(RT_ARCH_ARM64)
2059 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
2060#else
2061# error "Port me!"
2062#endif
2063}
2064
2065
2066/**
2067 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
2068 * How @a offJmp is applied is are target specific.
2069 */
2070DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2071{
2072#ifdef RT_ARCH_AMD64
2073 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);
2074#elif defined(RT_ARCH_ARM64)
2075 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
2076#else
2077# error "Port me!"
2078#endif
2079}
2080
2081
2082/**
2083 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
2084 * How @a offJmp is applied is are target specific.
2085 */
2086DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2087{
2088#ifdef RT_ARCH_AMD64
2089 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);
2090#elif defined(RT_ARCH_ARM64)
2091 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
2092#else
2093# error "Port me!"
2094#endif
2095}
2096
2097
2098/**
2099 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
2100 * How @a offJmp is applied is are target specific.
2101 */
2102DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
2103{
2104#ifdef RT_ARCH_AMD64
2105 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);
2106#elif defined(RT_ARCH_ARM64)
2107 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
2108#else
2109# error "Port me!"
2110#endif
2111}
2112
2113
2114/**
2115 * Fixes up a conditional jump to a fixed label.
2116 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
2117 */
2118DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
2119{
2120# if defined(RT_ARCH_AMD64)
2121 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
2122 if (pbCodeBuf[offFixup] != 0x0f)
2123 {
2124 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
2125 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
2126 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
2127 }
2128 else
2129 {
2130 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
2131 uint32_t const offRel32 = offTarget - (offFixup + 6);
2132 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
2133 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
2134 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
2135 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
2136 }
2137
2138# elif defined(RT_ARCH_ARM64)
2139 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
2140 Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
2141 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
2142 | (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
2143
2144# endif
2145}
2146
2147
2148/**
2149 * Internal helper, don't call directly.
2150 */
2151DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2152 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel,
2153 bool fJmpIfSet)
2154{
2155 Assert(iBitNo < 64);
2156#ifdef RT_ARCH_AMD64
2157 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
2158 AssertReturn(pbCodeBuf, UINT32_MAX);
2159 if (iBitNo < 8)
2160 {
2161 /* test Eb, imm8 */
2162 if (iGprSrc >= 8)
2163 pbCodeBuf[off++] = X86_OP_REX_B;
2164 pbCodeBuf[off++] = 0xf6;
2165 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
2166 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
2167 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
2168 }
2169 else
2170 {
2171 /* bt Ev, imm8 */
2172 if (iBitNo >= 32)
2173 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
2174 else if (iGprSrc >= 8)
2175 pbCodeBuf[off++] = X86_OP_REX_B;
2176 pbCodeBuf[off++] = 0x0f;
2177 pbCodeBuf[off++] = 0xba;
2178 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
2179 pbCodeBuf[off++] = iBitNo;
2180 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
2181 }
2182
2183#elif defined(RT_ARCH_ARM64)
2184 /* Use the TBNZ instruction here. */
2185 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2186 AssertReturn(pu32CodeBuf, UINT32_MAX);
2187 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5), UINT32_MAX);
2188 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
2189
2190#else
2191# error "Port me!"
2192#endif
2193 return off;
2194}
2195
2196
2197/**
2198 * Emits a jump to @a idxLabel con the condition that bit@a iBitNo _is_ _set_ in
2199 * @a iGprSrc.
2200 *
2201 * @note On ARM64 the range is only +/-8191 instructions.
2202 */
2203DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2204 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2205{
2206 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
2207}
2208
2209
2210/**
2211 * Emits a jump to @a idxLabel con the condition that bit@a iBitNo _is_ _not_
2212 * _set_ in @a iGprSrc.
2213 *
2214 * @note On ARM64 the range is only +/-8191 instructions.
2215 */
2216DECLINLINE(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2217 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
2218{
2219 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
2220}
2221
2222
2223/**
2224 * Emits a call to a 64-bit address.
2225 */
2226DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
2227{
2228#ifdef RT_ARCH_AMD64
2229 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
2230
2231 /* call rax */
2232 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
2233 AssertReturn(pbCodeBuf, UINT32_MAX);
2234 pbCodeBuf[off++] = 0xff;
2235 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
2236
2237#elif defined(RT_ARCH_ARM64)
2238 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
2239
2240 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
2241 AssertReturn(pu32CodeBuf, UINT32_MAX);
2242 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
2243#else
2244# error "port me"
2245#endif
2246 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
2247 return off;
2248}
2249
2250
2251
2252/** @} */
2253
2254#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2255
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette