VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101523

Last change on this file since 101523 was 101523, checked in by vboxsync, 18 months ago

VMM/IEM: Cleanups. bugref:10371
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 72.4 KB
Line 
1/* $Id: IEMN8veRecompiler.h 101523 2023-10-20 14:46:13Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @name Stack Frame Layout
41 *
42 * @{ */
43/** The size of the area for stack variables and spills and stuff.
44 * @note This limit is duplicated in the python script(s). */
45#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
46#ifdef RT_ARCH_AMD64
47/** Number of stack arguments slots for calls made from the frame. */
48# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
49/** An stack alignment adjustment (between non-volatile register pushes and
50 * the stack variable area, so the latter better aligned). */
51# define IEMNATIVE_FRAME_ALIGN_SIZE 8
52/** Number of any shadow arguments (spill area) for calls we make. */
53# ifdef RT_OS_WINDOWS
54# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
55# else
56# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
57# endif
58
59/** Frame pointer (RBP) relative offset of the last push. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
62# else
63# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
64# endif
65/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
66 * address for it). */
67# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
68/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
69# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
70/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
71# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
72/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
73# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
74/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
75# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
76
77# ifdef RT_OS_WINDOWS
78/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
79# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
80/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
81# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
82/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
83# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
84/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
85# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
86# endif
87
88#elif RT_ARCH_ARM64
89/** No stack argument slots, enough got 8 registers for arguments. */
90# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
91/** There are no argument spill area. */
92# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
93
94/** Number of saved registers at the top of our stack frame.
95 * This includes the return address and old frame pointer, so x19 thru x30. */
96# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
97/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
98# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
99
100/** Frame pointer (BP) relative offset of the last push. */
101# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
102
103/** Frame pointer (BP) relative offset of the stack variable area (the lowest
104 * address for it). */
105# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
106
107#else
108# error "port me"
109#endif
110/** @} */
111
112
113/** @name Fixed Register Allocation(s)
114 * @{ */
115/** @def IEMNATIVE_REG_FIXED_PVMCPU
116 * The number of the register holding the pVCpu pointer. */
117/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
118 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
119 * @note This not available on AMD64, only ARM64. */
120/** @def IEMNATIVE_REG_FIXED_TMP0
121 * Dedicated temporary register.
122 * @todo replace this by a register allocator and content tracker. */
123/** @def IEMNATIVE_REG_FIXED_MASK
124 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
125 * architecture. */
126#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
127# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
128# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
129# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
130 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
131 | RT_BIT_32(X86_GREG_xSP) \
132 | RT_BIT_32(X86_GREG_xBP) )
133
134#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
135# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
136# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
137# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
138# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
139 | RT_BIT_32(ARMV8_A64_REG_LR) \
140 | RT_BIT_32(ARMV8_A64_REG_BP) \
141 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
142 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
143 | RT_BIT_32(ARMV8_A64_REG_X18) \
144 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
145
146#else
147# error "port me"
148#endif
149/** @} */
150
151/** @name Call related registers.
152 * @{ */
153/** @def IEMNATIVE_CALL_RET_GREG
154 * The return value register. */
155/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
156 * Number of arguments in registers. */
157/** @def IEMNATIVE_CALL_ARG0_GREG
158 * The general purpose register carrying argument \#0. */
159/** @def IEMNATIVE_CALL_ARG1_GREG
160 * The general purpose register carrying argument \#1. */
161/** @def IEMNATIVE_CALL_ARG2_GREG
162 * The general purpose register carrying argument \#2. */
163/** @def IEMNATIVE_CALL_ARG3_GREG
164 * The general purpose register carrying argument \#3. */
165/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
166 * Mask of registers the callee will not save and may trash. */
167#ifdef RT_ARCH_AMD64
168# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
169
170# ifdef RT_OS_WINDOWS
171# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
172# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
173# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
174# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
175# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
176# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
177 | RT_BIT_32(X86_GREG_xCX) \
178 | RT_BIT_32(X86_GREG_xDX) \
179 | RT_BIT_32(X86_GREG_x8) \
180 | RT_BIT_32(X86_GREG_x9) \
181 | RT_BIT_32(X86_GREG_x10) \
182 | RT_BIT_32(X86_GREG_x11) )
183# else
184# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
185# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
186# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
187# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
188# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
189# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
190# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
191# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
192 | RT_BIT_32(X86_GREG_xCX) \
193 | RT_BIT_32(X86_GREG_xDX) \
194 | RT_BIT_32(X86_GREG_xDI) \
195 | RT_BIT_32(X86_GREG_xSI) \
196 | RT_BIT_32(X86_GREG_x8) \
197 | RT_BIT_32(X86_GREG_x9) \
198 | RT_BIT_32(X86_GREG_x10) \
199 | RT_BIT_32(X86_GREG_x11) )
200# endif
201
202#elif defined(RT_ARCH_ARM64)
203# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
204# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
205# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
206# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
207# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
208# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
209# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
210# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
211# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
212# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
213# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
214 | RT_BIT_32(ARMV8_A64_REG_X1) \
215 | RT_BIT_32(ARMV8_A64_REG_X2) \
216 | RT_BIT_32(ARMV8_A64_REG_X3) \
217 | RT_BIT_32(ARMV8_A64_REG_X4) \
218 | RT_BIT_32(ARMV8_A64_REG_X5) \
219 | RT_BIT_32(ARMV8_A64_REG_X6) \
220 | RT_BIT_32(ARMV8_A64_REG_X7) \
221 | RT_BIT_32(ARMV8_A64_REG_X8) \
222 | RT_BIT_32(ARMV8_A64_REG_X9) \
223 | RT_BIT_32(ARMV8_A64_REG_X10) \
224 | RT_BIT_32(ARMV8_A64_REG_X11) \
225 | RT_BIT_32(ARMV8_A64_REG_X12) \
226 | RT_BIT_32(ARMV8_A64_REG_X13) \
227 | RT_BIT_32(ARMV8_A64_REG_X14) \
228 | RT_BIT_32(ARMV8_A64_REG_X15) \
229 | RT_BIT_32(ARMV8_A64_REG_X16) \
230 | RT_BIT_32(ARMV8_A64_REG_X17) )
231
232#endif
233
234/** @} */
235
236
237/** @def IEMNATIVE_HST_GREG_COUNT
238 * Number of host general purpose registers we tracker. */
239/** @def IEMNATIVE_HST_GREG_MASK
240 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
241 * inverted register masks and such to get down to a correct set of regs. */
242#ifdef RT_ARCH_AMD64
243# define IEMNATIVE_HST_GREG_COUNT 16
244# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
245
246#elif defined(RT_ARCH_ARM64)
247# define IEMNATIVE_HST_GREG_COUNT 32
248# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
249#else
250# error "Port me!"
251#endif
252
253
254/** Native code generator label types. */
255typedef enum
256{
257 kIemNativeLabelType_Invalid = 0,
258 kIemNativeLabelType_Return,
259 kIemNativeLabelType_NonZeroRetOrPassUp,
260 kIemNativeLabelType_RaiseGp0,
261 kIemNativeLabelType_End
262} IEMNATIVELABELTYPE;
263
264/** Native code generator label definition. */
265typedef struct IEMNATIVELABEL
266{
267 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
268 * the epilog. */
269 uint32_t off;
270 /** The type of label (IEMNATIVELABELTYPE). */
271 uint16_t enmType;
272 /** Additional label data, type specific. */
273 uint16_t uData;
274} IEMNATIVELABEL;
275/** Pointer to a label. */
276typedef IEMNATIVELABEL *PIEMNATIVELABEL;
277
278
279/** Native code generator fixup types. */
280typedef enum
281{
282 kIemNativeFixupType_Invalid = 0,
283#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
284 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
285 kIemNativeFixupType_Rel32,
286#elif defined(RT_ARCH_ARM64)
287 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ). */
288 kIemNativeFixupType_RelImm19At5,
289#endif
290 kIemNativeFixupType_End
291} IEMNATIVEFIXUPTYPE;
292
293/** Native code generator fixup. */
294typedef struct IEMNATIVEFIXUP
295{
296 /** Code offset of the fixup location. */
297 uint32_t off;
298 /** The IEMNATIVELABEL this is a fixup for. */
299 uint16_t idxLabel;
300 /** The fixup type (IEMNATIVEFIXUPTYPE). */
301 uint8_t enmType;
302 /** Addend or other data. */
303 int8_t offAddend;
304} IEMNATIVEFIXUP;
305/** Pointer to a native code generator fixup. */
306typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
307
308
309/**
310 * Guest registers that can be shadowed in GPRs.
311 */
312typedef enum IEMNATIVEGSTREG : uint8_t
313{
314 kIemNativeGstReg_GprFirst = 0,
315 kIemNativeGstReg_GprLast = 15,
316 kIemNativeGstReg_Pc,
317 kIemNativeGstReg_Rflags,
318 /* gap: 18..23 */
319 kIemNativeGstReg_SegSelFirst = 24,
320 kIemNativeGstReg_SegSelLast = 29,
321 kIemNativeGstReg_SegBaseFirst = 30,
322 kIemNativeGstReg_SegBaseLast = 35,
323 kIemNativeGstReg_SegLimitFirst = 36,
324 kIemNativeGstReg_SegLimitLast = 41,
325 kIemNativeGstReg_End
326} IEMNATIVEGSTREG;
327
328/**
329 * Guest registers (classes) that can be referenced.
330 */
331typedef enum IEMNATIVEGSTREGREF : uint8_t
332{
333 kIemNativeGstRegRef_Invalid = 0,
334 kIemNativeGstRegRef_Gpr,
335 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
336 kIemNativeGstRegRef_EFlags,
337 kIemNativeGstRegRef_MxCsr,
338 kIemNativeGstRegRef_FpuReg,
339 kIemNativeGstRegRef_MReg,
340 kIemNativeGstRegRef_XReg,
341 kIemNativeGstRegRef_YReg,
342 kIemNativeGstRegRef_End
343} IEMNATIVEGSTREGREF;
344
345
346/** Variable kinds. */
347typedef enum IEMNATIVEVARKIND : uint8_t
348{
349 /** Customary invalid zero value. */
350 kIemNativeVarKind_Invalid = 0,
351 /** This is either in a register or on the stack. */
352 kIemNativeVarKind_Stack,
353 /** Immediate value - loaded into register when needed, or can live on the
354 * stack if referenced (in theory). */
355 kIemNativeVarKind_Immediate,
356 /** Variable reference - loaded into register when needed, never stack. */
357 kIemNativeVarKind_VarRef,
358 /** Guest register reference - loaded into register when needed, never stack. */
359 kIemNativeVarKind_GstRegRef,
360 /** End of valid values. */
361 kIemNativeVarKind_End
362} IEMNATIVEVARKIND;
363
364
365/** Variable or argument. */
366typedef struct IEMNATIVEVAR
367{
368 /** The kind of variable. */
369 IEMNATIVEVARKIND enmKind;
370 /** The variable size in bytes. */
371 uint8_t cbVar;
372 /** The first stack slot (uint64_t), except for immediate and references
373 * where it usually is UINT8_MAX. */
374 uint8_t idxStackSlot;
375 /** The host register allocated for the variable, UINT8_MAX if not. */
376 uint8_t idxReg;
377 /** The argument number if argument, UINT8_MAX if regular variable. */
378 uint8_t uArgNo;
379 /** If referenced, the index of the variable referencing this one, otherwise
380 * UINT8_MAX. A referenced variable must only be placed on the stack and
381 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
382 uint8_t idxReferrerVar;
383 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
384 IEMNATIVEGSTREG enmGstReg;
385 uint8_t bAlign;
386
387 union
388 {
389 /** kIemNativeVarKind_Immediate: The immediate value. */
390 uint64_t uValue;
391 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
392 uint8_t idxRefVar;
393 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
394 struct
395 {
396 /** The class of register. */
397 IEMNATIVEGSTREGREF enmClass;
398 /** Index within the class. */
399 uint8_t idx;
400 } GstRegRef;
401 } u;
402} IEMNATIVEVAR;
403
404/** What is being kept in a host register. */
405typedef enum IEMNATIVEWHAT : uint8_t
406{
407 /** The traditional invalid zero value. */
408 kIemNativeWhat_Invalid = 0,
409 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
410 kIemNativeWhat_Var,
411 /** Temporary register, this is typically freed when a MC completes. */
412 kIemNativeWhat_Tmp,
413 /** Call argument w/o a variable mapping. This is free (via
414 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
415 kIemNativeWhat_Arg,
416 /** Return status code.
417 * @todo not sure if we need this... */
418 kIemNativeWhat_rc,
419 /** The fixed pVCpu (PVMCPUCC) register.
420 * @todo consider offsetting this on amd64 to use negative offsets to access
421 * more members using 8-byte disp. */
422 kIemNativeWhat_pVCpuFixed,
423 /** The fixed pCtx (PCPUMCTX) register.
424 * @todo consider offsetting this on amd64 to use negative offsets to access
425 * more members using 8-byte disp. */
426 kIemNativeWhat_pCtxFixed,
427 /** Fixed temporary register. */
428 kIemNativeWhat_FixedTmp,
429 /** Register reserved by the CPU or OS architecture. */
430 kIemNativeWhat_FixedReserved,
431 /** End of valid values. */
432 kIemNativeWhat_End
433} IEMNATIVEWHAT;
434
435/**
436 * Host general register entry.
437 *
438 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
439 *
440 * @todo Track immediate values in host registers similarlly to how we track the
441 * guest register shadow copies. For it to be real helpful, though,
442 * we probably need to know which will be reused and put them into
443 * non-volatile registers, otherwise it's going to be more or less
444 * restricted to an instruction or two.
445 */
446typedef struct IEMNATIVEHSTREG
447{
448 /** Set of guest registers this one shadows.
449 *
450 * Using a bitmap here so we can designate the same host register as a copy
451 * for more than one guest register. This is expected to be useful in
452 * situations where one value is copied to several registers in a sequence.
453 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
454 * sequence we'd want to let this register follow to be a copy of and there
455 * will always be places where we'd be picking the wrong one.
456 */
457 uint64_t fGstRegShadows;
458 /** What is being kept in this register. */
459 IEMNATIVEWHAT enmWhat;
460 /** Variable index if holding a variable, otherwise UINT8_MAX. */
461 uint8_t idxVar;
462 /** Alignment padding. */
463 uint8_t abAlign[6];
464} IEMNATIVEHSTREG;
465
466
467/**
468 * Native recompiler state.
469 */
470typedef struct IEMRECOMPILERSTATE
471{
472 /** Size of the buffer that pbNativeRecompileBufR3 points to in
473 * IEMNATIVEINSTR units. */
474 uint32_t cInstrBufAlloc;
475 uint32_t uPadding; /* We don't keep track of this here... */
476 /** Fixed temporary code buffer for native recompilation. */
477 PIEMNATIVEINSTR pInstrBuf;
478
479 /** Bitmaps with the label types used. */
480 uint64_t bmLabelTypes;
481 /** Actual number of labels in paLabels. */
482 uint32_t cLabels;
483 /** Max number of entries allowed in paLabels before reallocating it. */
484 uint32_t cLabelsAlloc;
485 /** Labels defined while recompiling (referenced by fixups). */
486 PIEMNATIVELABEL paLabels;
487
488 /** Actual number of fixups paFixups. */
489 uint32_t cFixups;
490 /** Max number of entries allowed in paFixups before reallocating it. */
491 uint32_t cFixupsAlloc;
492 /** Buffer used by the recompiler for recording fixups when generating code. */
493 PIEMNATIVEFIXUP paFixups;
494
495 /** The translation block being recompiled. */
496 PCIEMTB pTbOrg;
497
498 /** Allocation bitmap fro aHstRegs. */
499 uint32_t bmHstRegs;
500
501 /** Bitmap marking which host register contains guest register shadow copies.
502 * This is used during register allocation to try preserve copies. */
503 uint32_t bmHstRegsWithGstShadow;
504 /** Bitmap marking valid entries in aidxGstRegShadows. */
505 uint64_t bmGstRegShadows;
506
507 /** Allocation bitmap for aVars. */
508 uint32_t bmVars;
509 uint32_t u32Align;
510 union
511 {
512 /** Index of variable arguments, UINT8_MAX if not valid. */
513 uint8_t aidxArgVars[8];
514 /** For more efficient resetting. */
515 uint64_t u64ArgVars;
516 };
517
518 /** Host register allocation tracking. */
519 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
520 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
521 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
522 * (A shadow copy of a guest register can only be held in a one host register,
523 * there are no duplicate copies or ambiguities like that). */
524 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
525 /** Variables and arguments. */
526 IEMNATIVEVAR aVars[16];
527} IEMRECOMPILERSTATE;
528/** Pointer to a native recompiler state. */
529typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
530
531
532/**
533 * Native recompiler worker for a threaded function.
534 *
535 * @returns New code buffer offset, UINT32_MAX in case of failure.
536 * @param pReNative The native recompiler state.
537 * @param off The current code buffer offset.
538 * @param pCallEntry The threaded call entry.
539 *
540 * @note This is not allowed to throw anything atm.
541 */
542typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
543 PCIEMTHRDEDCALLENTRY pCallEntry));
544/** Pointer to a native recompiler worker for a threaded function. */
545typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
546
547/** Defines a native recompiler worker for a threaded function. */
548#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
549 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
550/** Prototypes a native recompiler function for a threaded function. */
551#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
552
553
554DECLHIDDEN(uint32_t) iemNativeMakeLabel(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
555 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
556DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
557 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
558DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
559 uint32_t cInstrReq) RT_NOEXCEPT;
560
561DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
562 bool fPreferVolatile = true) RT_NOEXCEPT;
563DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
564 bool fPreferVolatile = true) RT_NOEXCEPT;
565DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
566 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
567DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
568DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
569DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
570DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
571DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
572DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
573DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
574DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
575
576DECLHIDDEN(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
577 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
578DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
579 uint8_t idxInstr) RT_NOEXCEPT;
580
581
582/**
583 * Ensures that there is sufficient space in the instruction output buffer.
584 *
585 * This will reallocate the buffer if needed and allowed.
586 *
587 * @returns Pointer to the instruction output buffer on success, NULL on
588 * failure.
589 * @param pReNative The native recompile state.
590 * @param off Current instruction offset. Works safely for UINT32_MAX
591 * as well.
592 * @param cInstrReq Number of instruction about to be added. It's okay to
593 * overestimate this a bit.
594 */
595DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
596{
597 if (RT_LIKELY(off + (uint64_t)cInstrReq <= pReNative->cInstrBufAlloc))
598 return pReNative->pInstrBuf;
599 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
600}
601
602
603/**
604 * Emit a simple marker instruction to more easily tell where something starts
605 * in the disassembly.
606 */
607DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
608{
609#ifdef RT_ARCH_AMD64
610 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
611 AssertReturn(pbCodeBuf, UINT32_MAX);
612 if (uInfo == 0)
613 {
614 /* nop */
615 pbCodeBuf[off++] = 0x90;
616 }
617 else
618 {
619 /* nop [disp32] */
620 pbCodeBuf[off++] = 0x0f;
621 pbCodeBuf[off++] = 0x1f;
622 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
623 pbCodeBuf[off++] = RT_BYTE1(uInfo);
624 pbCodeBuf[off++] = RT_BYTE2(uInfo);
625 pbCodeBuf[off++] = RT_BYTE3(uInfo);
626 pbCodeBuf[off++] = RT_BYTE4(uInfo);
627 }
628#elif RT_ARCH_ARM64
629 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
630 AssertReturn(pu32CodeBuf, UINT32_MAX);
631 /* nop */
632 pu32CodeBuf[off++] = 0xd503201f;
633
634 RT_NOREF(uInfo);
635#else
636# error "port me"
637#endif
638 return off;
639}
640
641
642/**
643 * Emits setting a GPR to zero.
644 */
645DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
646{
647#ifdef RT_ARCH_AMD64
648 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
649 AssertReturn(pbCodeBuf, UINT32_MAX);
650 /* xor gpr32, gpr32 */
651 if (iGpr >= 8)
652 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
653 pbCodeBuf[off++] = 0x33;
654 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
655
656#elif RT_ARCH_ARM64
657 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
658 AssertReturn(pu32CodeBuf, UINT32_MAX);
659 /* mov gpr, #0x0 */
660 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
661
662#else
663# error "port me"
664#endif
665 RT_NOREF(pReNative);
666 return off;
667}
668
669
670/**
671 * Emits loading a constant into a 64-bit GPR
672 */
673DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
674{
675 if (!uImm64)
676 return iemNativeEmitGprZero(pReNative, off, iGpr);
677
678#ifdef RT_ARCH_AMD64
679 if (uImm64 <= UINT32_MAX)
680 {
681 /* mov gpr, imm32 */
682 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
683 AssertReturn(pbCodeBuf, UINT32_MAX);
684 if (iGpr >= 8)
685 pbCodeBuf[off++] = X86_OP_REX_B;
686 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
687 pbCodeBuf[off++] = RT_BYTE1(uImm64);
688 pbCodeBuf[off++] = RT_BYTE2(uImm64);
689 pbCodeBuf[off++] = RT_BYTE3(uImm64);
690 pbCodeBuf[off++] = RT_BYTE4(uImm64);
691 }
692 else
693 {
694 /* mov gpr, imm64 */
695 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
696 AssertReturn(pbCodeBuf, UINT32_MAX);
697 if (iGpr < 8)
698 pbCodeBuf[off++] = X86_OP_REX_W;
699 else
700 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
701 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
702 pbCodeBuf[off++] = RT_BYTE1(uImm64);
703 pbCodeBuf[off++] = RT_BYTE2(uImm64);
704 pbCodeBuf[off++] = RT_BYTE3(uImm64);
705 pbCodeBuf[off++] = RT_BYTE4(uImm64);
706 pbCodeBuf[off++] = RT_BYTE5(uImm64);
707 pbCodeBuf[off++] = RT_BYTE6(uImm64);
708 pbCodeBuf[off++] = RT_BYTE7(uImm64);
709 pbCodeBuf[off++] = RT_BYTE8(uImm64);
710 }
711
712#elif RT_ARCH_ARM64
713 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
714 AssertReturn(pu32CodeBuf, UINT32_MAX);
715
716 /*
717 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
718 * supply remaining bits using 'movk grp, imm16, lsl #x'.
719 *
720 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
721 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
722 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
723 * after the first non-zero immediate component so we switch to movk for
724 * the remainder.
725 */
726 uint32_t fMovK = 0;
727 /* mov gpr, imm16 */
728 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
729 if (uImmPart)
730 {
731 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
732 fMovK |= RT_BIT_32(29);
733 }
734 /* mov[k] gpr, imm16, lsl #16 */
735 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
736 if (uImmPart)
737 {
738 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
739 fMovK |= RT_BIT_32(29);
740 }
741 /* mov[k] gpr, imm16, lsl #32 */
742 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
743 if (uImmPart)
744 {
745 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
746 fMovK |= RT_BIT_32(29);
747 }
748 /* mov[k] gpr, imm16, lsl #48 */
749 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
750 if (uImmPart)
751 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
752
753 /** @todo there is an inverted mask variant we might want to explore if it
754 * reduces the number of instructions... */
755 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
756 * clang 12.x does that, only to use the 'x' version for the
757 * addressing in the following ldr). */
758
759#else
760# error "port me"
761#endif
762 return off;
763}
764
765
766/**
767 * Emits loading a constant into a 8-bit GPR
768 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,
769 * only the ARM64 version does that.
770 */
771DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
772{
773#ifdef RT_ARCH_AMD64
774 /* mov gpr, imm8 */
775 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
776 AssertReturn(pbCodeBuf, UINT32_MAX);
777 if (iGpr >= 8)
778 pbCodeBuf[off++] = X86_OP_REX_B;
779 else if (iGpr >= 4)
780 pbCodeBuf[off++] = X86_OP_REX;
781 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
782 pbCodeBuf[off++] = RT_BYTE1(uImm8);
783
784#elif RT_ARCH_ARM64
785 /* movz gpr, imm16, lsl #0 */
786 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
787 AssertReturn(pu32CodeBuf, UINT32_MAX);
788 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;
789
790#else
791# error "port me"
792#endif
793 return off;
794}
795
796
797#ifdef RT_ARCH_AMD64
798/**
799 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
800 */
801DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
802{
803 if (offVCpu < 128)
804 {
805 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
806 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
807 }
808 else
809 {
810 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
811 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
812 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
813 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
814 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
815 }
816 return off;
817}
818#elif RT_ARCH_ARM64
819/**
820 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
821 */
822DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
823 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
824{
825 /*
826 * There are a couple of ldr variants that takes an immediate offset, so
827 * try use those if we can, otherwise we have to use the temporary register
828 * help with the addressing.
829 */
830 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
831 {
832 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
833 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
834 AssertReturn(pu32CodeBuf, UINT32_MAX);
835 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
836 }
837 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
838 {
839 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
840 AssertReturn(pu32CodeBuf, UINT32_MAX);
841 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
842 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
843 }
844 else
845 {
846 /* The offset is too large, so we must load it into a register and use
847 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
848 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
849 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
850
851 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
852 AssertReturn(pu32CodeBuf, UINT32_MAX);
853 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
854 }
855 return off;
856}
857#endif
858
859
860/**
861 * Emits a 64-bit GPR load of a VCpu value.
862 */
863DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
864{
865#ifdef RT_ARCH_AMD64
866 /* mov reg64, mem64 */
867 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
868 AssertReturn(pbCodeBuf, UINT32_MAX);
869 if (iGpr < 8)
870 pbCodeBuf[off++] = X86_OP_REX_W;
871 else
872 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
873 pbCodeBuf[off++] = 0x8b;
874 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
875
876#elif RT_ARCH_ARM64
877 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
878
879#else
880# error "port me"
881#endif
882 return off;
883}
884
885
886/**
887 * Emits a 32-bit GPR load of a VCpu value.
888 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
889 */
890DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
891{
892#ifdef RT_ARCH_AMD64
893 /* mov reg32, mem32 */
894 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
895 AssertReturn(pbCodeBuf, UINT32_MAX);
896 if (iGpr >= 8)
897 pbCodeBuf[off++] = X86_OP_REX_R;
898 pbCodeBuf[off++] = 0x8b;
899 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
900
901#elif RT_ARCH_ARM64
902 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
903
904#else
905# error "port me"
906#endif
907 return off;
908}
909
910
911/**
912 * Emits a 16-bit GPR load of a VCpu value.
913 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
914 */
915DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
916{
917#ifdef RT_ARCH_AMD64
918 /* movzx reg32, mem16 */
919 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
920 AssertReturn(pbCodeBuf, UINT32_MAX);
921 if (iGpr >= 8)
922 pbCodeBuf[off++] = X86_OP_REX_R;
923 pbCodeBuf[off++] = 0x0f;
924 pbCodeBuf[off++] = 0xb7;
925 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
926
927#elif RT_ARCH_ARM64
928 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
929
930#else
931# error "port me"
932#endif
933 return off;
934}
935
936
937/**
938 * Emits a 8-bit GPR load of a VCpu value.
939 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
940 */
941DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
942{
943#ifdef RT_ARCH_AMD64
944 /* movzx reg32, mem8 */
945 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
946 AssertReturn(pbCodeBuf, UINT32_MAX);
947 if (iGpr >= 8)
948 pbCodeBuf[off++] = X86_OP_REX_R;
949 pbCodeBuf[off++] = 0x0f;
950 pbCodeBuf[off++] = 0xb6;
951 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
952
953#elif RT_ARCH_ARM64
954 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
955
956#else
957# error "port me"
958#endif
959 return off;
960}
961
962
963/**
964 * Emits a store of a GPR value to a 64-bit VCpu field.
965 */
966DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
967{
968#ifdef RT_ARCH_AMD64
969 /* mov mem64, reg64 */
970 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
971 AssertReturn(pbCodeBuf, UINT32_MAX);
972 if (iGpr < 8)
973 pbCodeBuf[off++] = X86_OP_REX_W;
974 else
975 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
976 pbCodeBuf[off++] = 0x89;
977 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
978
979#elif RT_ARCH_ARM64
980 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
981
982#else
983# error "port me"
984#endif
985 return off;
986}
987
988
989/**
990 * Emits a store of a GPR value to a 32-bit VCpu field.
991 */
992DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
993{
994#ifdef RT_ARCH_AMD64
995 /* mov mem32, reg32 */
996 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
997 AssertReturn(pbCodeBuf, UINT32_MAX);
998 if (iGpr >= 8)
999 pbCodeBuf[off++] = X86_OP_REX_R;
1000 pbCodeBuf[off++] = 0x89;
1001 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1002
1003#elif RT_ARCH_ARM64
1004 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1005
1006#else
1007# error "port me"
1008#endif
1009 return off;
1010}
1011
1012
1013/**
1014 * Emits a store of a GPR value to a 16-bit VCpu field.
1015 */
1016DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1017{
1018#ifdef RT_ARCH_AMD64
1019 /* mov mem16, reg16 */
1020 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1021 AssertReturn(pbCodeBuf, UINT32_MAX);
1022 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1023 if (iGpr >= 8)
1024 pbCodeBuf[off++] = X86_OP_REX_R;
1025 pbCodeBuf[off++] = 0x89;
1026 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1027
1028#elif RT_ARCH_ARM64
1029 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1030
1031#else
1032# error "port me"
1033#endif
1034 return off;
1035}
1036
1037
1038/**
1039 * Emits a store of a GPR value to a 8-bit VCpu field.
1040 */
1041DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1042{
1043#ifdef RT_ARCH_AMD64
1044 /* mov mem8, reg8 */
1045 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1046 AssertReturn(pbCodeBuf, UINT32_MAX);
1047 if (iGpr >= 8)
1048 pbCodeBuf[off++] = X86_OP_REX_R;
1049 pbCodeBuf[off++] = 0x88;
1050 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1051
1052#elif RT_ARCH_ARM64
1053 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1054
1055#else
1056# error "port me"
1057#endif
1058 return off;
1059}
1060
1061
1062/**
1063 * Emits a gprdst = gprsrc load.
1064 */
1065DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1066{
1067#ifdef RT_ARCH_AMD64
1068 /* mov gprdst, gprsrc */
1069 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1070 AssertReturn(pbCodeBuf, UINT32_MAX);
1071 if ((iGprDst | iGprSrc) >= 8)
1072 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1073 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1074 : X86_OP_REX_W | X86_OP_REX_R;
1075 else
1076 pbCodeBuf[off++] = X86_OP_REX_W;
1077 pbCodeBuf[off++] = 0x8b;
1078 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1079
1080#elif RT_ARCH_ARM64
1081 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1082 AssertReturn(pu32CodeBuf, UINT32_MAX);
1083 /* mov dst, src; alias for: orr dst, xzr, src */
1084 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1085
1086#else
1087# error "port me"
1088#endif
1089 return off;
1090}
1091
1092#ifdef RT_ARCH_AMD64
1093/**
1094 * Common bit of iemNativeEmitLoadGprByBp and friends.
1095 */
1096DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp)
1097{
1098 if (offDisp < 128 && offDisp >= -128)
1099 {
1100 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1101 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1102 }
1103 else
1104 {
1105 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1106 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1107 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1108 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1109 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1110 }
1111 return off;
1112}
1113#endif
1114
1115
1116#ifdef RT_ARCH_AMD64
1117/**
1118 * Emits a 64-bit GRP load instruction with an BP relative source address.
1119 */
1120DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1121{
1122 /* mov gprdst, qword [rbp + offDisp] */
1123 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1124 AssertReturn(pbCodeBuf, UINT32_MAX);
1125 if (iGprDst < 8)
1126 pbCodeBuf[off++] = X86_OP_REX_W;
1127 else
1128 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1129 pbCodeBuf[off++] = 0x8b;
1130 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1131}
1132#endif
1133
1134
1135#ifdef RT_ARCH_AMD64
1136/**
1137 * Emits a 32-bit GRP load instruction with an BP relative source address.
1138 */
1139DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1140{
1141 /* mov gprdst, dword [rbp + offDisp] */
1142 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1143 AssertReturn(pbCodeBuf, UINT32_MAX);
1144 if (iGprDst >= 8)
1145 pbCodeBuf[off++] = X86_OP_REX_R;
1146 pbCodeBuf[off++] = 0x8b;
1147 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1148}
1149#endif
1150
1151
1152#ifdef RT_ARCH_AMD64
1153/**
1154 * Emits a load effective address to a GRP with an BP relative source address.
1155 */
1156DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1157{
1158 /* lea gprdst, [rbp + offDisp] */
1159 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1160 AssertReturn(pbCodeBuf, UINT32_MAX);
1161 if (iGprDst < 8)
1162 pbCodeBuf[off++] = X86_OP_REX_W;
1163 else
1164 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1165 pbCodeBuf[off++] = 0x8d;
1166 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1167}
1168#endif
1169
1170
1171/**
1172 * Emits a 64-bit GPR store with an BP relative destination address.
1173 *
1174 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1175 */
1176DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1177{
1178#ifdef RT_ARCH_AMD64
1179 /* mov qword [rbp + offDisp], gprdst */
1180 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1181 AssertReturn(pbCodeBuf, UINT32_MAX);
1182 if (iGprSrc < 8)
1183 pbCodeBuf[off++] = X86_OP_REX_W;
1184 else
1185 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1186 pbCodeBuf[off++] = 0x89;
1187 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp);
1188
1189#elif defined(RT_ARCH_ARM64)
1190 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1191 {
1192 /* str w/ unsigned imm12 (scaled) */
1193 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1194 AssertReturn(pu32CodeBuf, UINT32_MAX);
1195 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1196 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1197 }
1198 else if (offDisp >= -256 && offDisp <= 256)
1199 {
1200 /* stur w/ signed imm9 (unscaled) */
1201 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1202 AssertReturn(pu32CodeBuf, UINT32_MAX);
1203 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1204 }
1205 else
1206 {
1207 /* Use temporary indexing register. */
1208 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1209 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1210 AssertReturn(pu32CodeBuf, UINT32_MAX);
1211 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1212 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1213 }
1214 return off;
1215
1216#else
1217# error "Port me!"
1218#endif
1219}
1220
1221
1222/**
1223 * Emits a 64-bit immediate store with an BP relative destination address.
1224 *
1225 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1226 */
1227DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1228{
1229#ifdef RT_ARCH_AMD64
1230 if ((int64_t)uImm64 == (int32_t)uImm64)
1231 {
1232 /* mov qword [rbp + offDisp], imm32 - sign extended */
1233 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1234 AssertReturn(pbCodeBuf, UINT32_MAX);
1235
1236 pbCodeBuf[off++] = X86_OP_REX_W;
1237 pbCodeBuf[off++] = 0xc7;
1238 if (offDisp < 128 && offDisp >= -128)
1239 {
1240 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1241 pbCodeBuf[off++] = (uint8_t)offDisp;
1242 }
1243 else
1244 {
1245 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1246 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1247 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1248 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1249 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1250 }
1251 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1252 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1253 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1254 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1255 return off;
1256 }
1257#endif
1258
1259 /* Load tmp0, imm64; Store tmp to bp+disp. */
1260 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1261 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1262}
1263
1264
1265#ifdef RT_ARCH_AMD64
1266/**
1267 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1268 */
1269DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1270{
1271 /* sub gprdst, imm8/imm32 */
1272 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1273 AssertReturn(pbCodeBuf, UINT32_MAX);
1274 if (iGprDst < 8)
1275 pbCodeBuf[off++] = X86_OP_REX_W;
1276 else
1277 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1278 if (iSubtrahend < 128 && iSubtrahend >= -128)
1279 {
1280 pbCodeBuf[off++] = 0x83;
1281 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1282 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1283 }
1284 else
1285 {
1286 pbCodeBuf[off++] = 0x81;
1287 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1288 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1289 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1290 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1291 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1292 }
1293 return off;
1294}
1295#endif
1296
1297
1298/**
1299 * Emits adding a 64-bit GPR to another, storing the result in the frist.
1300 * @note The AMD64 version sets flags.
1301 */
1302DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1303{
1304#if defined(RT_ARCH_AMD64)
1305 /* add Gv,Ev */
1306 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1307 AssertReturn(pbCodeBuf, UINT32_MAX);
1308 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1309 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1310 pbCodeBuf[off++] = 0x04;
1311 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1312
1313#elif defined(RT_ARCH_ARM64)
1314 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1315 AssertReturn(pu32CodeBuf, UINT32_MAX);
1316 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
1317
1318#else
1319# error "Port me"
1320#endif
1321 return off;
1322}
1323
1324
1325/**
1326 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
1327 */
1328DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1329{
1330#if defined(RT_ARCH_AMD64)
1331 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1332 AssertReturn(pbCodeBuf, UINT32_MAX);
1333 /* add or inc */
1334 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1335 if (iImm8 != 1)
1336 {
1337 pbCodeBuf[off++] = 0x83;
1338 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1339 pbCodeBuf[off++] = (uint8_t)iImm8;
1340 }
1341 else
1342 {
1343 pbCodeBuf[off++] = 0xff;
1344 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1345 }
1346
1347#elif defined(RT_ARCH_ARM64)
1348 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1349 AssertReturn(pu32CodeBuf, UINT32_MAX);
1350 if (iImm8 >= 0)
1351 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
1352 else
1353 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
1354
1355#else
1356# error "Port me"
1357#endif
1358 return off;
1359}
1360
1361
1362/**
1363 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1364 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1365 */
1366DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1367{
1368#if defined(RT_ARCH_AMD64)
1369 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1370 AssertReturn(pbCodeBuf, UINT32_MAX);
1371 /* add or inc */
1372 if (iGprDst >= 8)
1373 pbCodeBuf[off++] = X86_OP_REX_B;
1374 if (iImm8 != 1)
1375 {
1376 pbCodeBuf[off++] = 0x83;
1377 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1378 pbCodeBuf[off++] = (uint8_t)iImm8;
1379 }
1380 else
1381 {
1382 pbCodeBuf[off++] = 0xff;
1383 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1384 }
1385
1386#elif defined(RT_ARCH_ARM64)
1387 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1388 AssertReturn(pu32CodeBuf, UINT32_MAX);
1389 if (iImm8 >= 0)
1390 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1391 else
1392 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1393
1394#else
1395# error "Port me"
1396#endif
1397 return off;
1398}
1399
1400
1401/**
1402 * Emits a 64-bit GPR additions with a 64-bit signed addend.
1403 */
1404DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1405{
1406#if defined(RT_ARCH_AMD64)
1407 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1408 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1409
1410 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1411 {
1412 /* add grp, imm32 */
1413 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1414 AssertReturn(pbCodeBuf, UINT32_MAX);
1415 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1416 pbCodeBuf[off++] = 0x81;
1417 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1418 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1419 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1420 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1421 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1422 }
1423 else
1424 {
1425 /* Best to use a temporary register to deal with this in the simplest way: */
1426 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1427 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1428
1429 /* add dst, tmpreg */
1430 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1431 AssertReturn(pbCodeBuf, UINT32_MAX);
1432 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
1433 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1434 pbCodeBuf[off++] = 0x03;
1435 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1436
1437 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1438 }
1439
1440#elif defined(RT_ARCH_ARM64)
1441 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1442 {
1443 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1444 AssertReturn(pu32CodeBuf, UINT32_MAX);
1445 if (iAddend >= 0)
1446 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);
1447 else
1448 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);
1449 }
1450 else
1451 {
1452 /* Use temporary register for the immediate. */
1453 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1454 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1455
1456 /* add gprdst, gprdst, tmpreg */
1457 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1458 AssertReturn(pu32CodeBuf, UINT32_MAX);
1459 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);
1460
1461 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1462 }
1463
1464#else
1465# error "Port me"
1466#endif
1467 return off;
1468}
1469
1470
1471/**
1472 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
1473 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1474 */
1475DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1476{
1477#if defined(RT_ARCH_AMD64)
1478 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1479 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1480
1481 /* add grp, imm32 */
1482 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1483 AssertReturn(pbCodeBuf, UINT32_MAX);
1484 if (iGprDst >= 8)
1485 pbCodeBuf[off++] = X86_OP_REX_B;
1486 pbCodeBuf[off++] = 0x81;
1487 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1488 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1489 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1490 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1491 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1492
1493#elif defined(RT_ARCH_ARM64)
1494 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1495 {
1496 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1497 AssertReturn(pu32CodeBuf, UINT32_MAX);
1498 if (iAddend >= 0)
1499 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);
1500 else
1501 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);
1502 }
1503 else
1504 {
1505 /* Use temporary register for the immediate. */
1506 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1507 AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1508
1509 /* add gprdst, gprdst, tmpreg */
1510 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1511 AssertReturn(pu32CodeBuf, UINT32_MAX);
1512 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
1513
1514 iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1515 }
1516
1517#else
1518# error "Port me"
1519#endif
1520 return off;
1521}
1522
1523
1524/**
1525 * Emits code for clearing bits 16 thru 63 in the GPR.
1526 */
1527DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1528{
1529#if defined(RT_ARCH_AMD64)
1530 /* movzx reg32, reg16 */
1531 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1532 AssertReturn(pbCodeBuf, UINT32_MAX);
1533 if (iGprDst >= 8)
1534 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;
1535 pbCodeBuf[off++] = 0x0f;
1536 pbCodeBuf[off++] = 0xb7;
1537 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1538
1539#elif defined(RT_ARCH_ARM64)
1540 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1541 AssertReturn(pu32CodeBuf, UINT32_MAX);
1542# if 1
1543 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1544# else
1545 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1546 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1547# endif
1548#else
1549# error "Port me"
1550#endif
1551 return off;
1552}
1553
1554
1555/**
1556 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1557 */
1558DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1559{
1560#if defined(RT_ARCH_AMD64)
1561 /* shr dst, cShift */
1562 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1563 AssertReturn(pbCodeBuf, UINT32_MAX);
1564 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
1565 pbCodeBuf[off++] = 0xc0;
1566 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1567 pbCodeBuf[off++] = cShift;
1568 Assert(cShift > 0 && cShift < 64);
1569
1570#elif defined(RT_ARCH_ARM64)
1571 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1572 AssertReturn(pu32CodeBuf, UINT32_MAX);
1573 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1574#else
1575# error "Port me"
1576#endif
1577 return off;
1578}
1579
1580
1581#ifdef RT_ARCH_ARM64
1582/**
1583 * Emits an ARM64 compare instruction.
1584 */
1585DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1586 bool f64Bit = true, uint32_t cShift = 0,
1587 ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1588{
1589 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1590 AssertReturn(pu32CodeBuf, UINT32_MAX);
1591 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,
1592 f64Bit, true /*fSetFlags*/, cShift, enmShift);
1593 return off;
1594}
1595#endif
1596
1597
1598/**
1599 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1600 * with conditional instruction.
1601 */
1602DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1603{
1604#ifdef RT_ARCH_AMD64
1605 /* cmp Gv, Ev */
1606 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1607 AssertReturn(pbCodeBuf, UINT32_MAX);
1608 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1609 pbCodeBuf[off++] = 0x3b;
1610 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1611
1612#elif defined(RT_ARCH_ARM64)
1613 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1614
1615#else
1616# error "Port me!"
1617#endif
1618 return off;
1619}
1620
1621
1622/**
1623 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1624 * with conditional instruction.
1625 */
1626DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1627 uint8_t iGprLeft, uint8_t iGprRight)
1628{
1629#ifdef RT_ARCH_AMD64
1630 /* cmp Gv, Ev */
1631 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1632 AssertReturn(pbCodeBuf, UINT32_MAX);
1633 if (iGprLeft >= 8 || iGprRight >= 8)
1634 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
1635 pbCodeBuf[off++] = 0x3b;
1636 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1637
1638#elif defined(RT_ARCH_ARM64)
1639 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
1640
1641#else
1642# error "Port me!"
1643#endif
1644 return off;
1645}
1646
1647
1648/**
1649 * Emits a JMP rel32 / B imm19 to the given label.
1650 */
1651DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1652{
1653 Assert(idxLabel < pReNative->cLabels);
1654
1655#ifdef RT_ARCH_AMD64
1656 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1657 AssertReturn(pbCodeBuf, UINT32_MAX);
1658 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1659 {
1660 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);
1661 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
1662 {
1663 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */
1664 pbCodeBuf[off++] = (uint8_t)offRel;
1665 off++;
1666 }
1667 else
1668 {
1669 offRel -= 3;
1670 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1671 pbCodeBuf[off++] = RT_BYTE1(offRel);
1672 pbCodeBuf[off++] = RT_BYTE2(offRel);
1673 pbCodeBuf[off++] = RT_BYTE3(offRel);
1674 pbCodeBuf[off++] = RT_BYTE4(offRel);
1675 }
1676 }
1677 else
1678 {
1679 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */
1680 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1681 pbCodeBuf[off++] = 0xfe;
1682 pbCodeBuf[off++] = 0xff;
1683 pbCodeBuf[off++] = 0xff;
1684 pbCodeBuf[off++] = 0xff;
1685 }
1686 pbCodeBuf[off++] = 0xcc; /* int3 poison */
1687
1688#elif defined(RT_ARCH_ARM64)
1689 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1690 AssertReturn(pu32CodeBuf, UINT32_MAX);
1691 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
1692 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxReturnLabel].off - off);
1693 else
1694 {
1695 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1696 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
1697 }
1698
1699#else
1700# error "Port me!"
1701#endif
1702 return off;
1703}
1704
1705
1706/**
1707 * Emits a JMP rel32 / B imm19 to a new undefined label.
1708 */
1709DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1710 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1711{
1712 uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1713 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1714 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
1715}
1716
1717/** Condition type. */
1718#ifdef RT_ARCH_AMD64
1719typedef uint8_t IEMNATIVEINSTRCOND;
1720#elif defined(RT_ARCH_ARM64)
1721typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
1722#else
1723# error "Port me!"
1724#endif
1725
1726
1727/**
1728 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
1729 */
1730DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1731 uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
1732{
1733 Assert(idxLabel < pReNative->cLabels);
1734
1735#ifdef RT_ARCH_AMD64
1736 /* jcc rel32 */
1737 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1738 AssertReturn(pbCodeBuf, UINT32_MAX);
1739 pbCodeBuf[off++] = 0x0f;
1740 pbCodeBuf[off++] = enmCond | 0x80;
1741 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1742 pbCodeBuf[off++] = 0x00;
1743 pbCodeBuf[off++] = 0x00;
1744 pbCodeBuf[off++] = 0x00;
1745 pbCodeBuf[off++] = 0x00;
1746
1747#elif defined(RT_ARCH_ARM64)
1748 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1749 AssertReturn(pu32CodeBuf, UINT32_MAX);
1750 AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1751 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
1752
1753#else
1754# error "Port me!"
1755#endif
1756 return off;
1757}
1758
1759
1760/**
1761 * Emits a Jcc rel32 / B.cc imm19 to a new label.
1762 */
1763DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1764 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
1765{
1766 uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
1767 AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1768 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
1769}
1770
1771
1772/**
1773 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
1774 */
1775DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1776 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1777{
1778#ifdef RT_ARCH_AMD64
1779 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x4);
1780#elif defined(RT_ARCH_ARM64)
1781 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
1782#else
1783# error "Port me!"
1784#endif
1785}
1786
1787
1788/**
1789 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
1790 */
1791DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1792 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1793{
1794#ifdef RT_ARCH_AMD64
1795 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x5);
1796#elif defined(RT_ARCH_ARM64)
1797 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
1798#else
1799# error "Port me!"
1800#endif
1801}
1802
1803
1804/**
1805 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
1806 */
1807DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1808 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1809{
1810#ifdef RT_ARCH_AMD64
1811 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x6);
1812#elif defined(RT_ARCH_ARM64)
1813 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
1814#else
1815# error "Port me!"
1816#endif
1817}
1818
1819
1820/**
1821 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
1822 */
1823DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1824 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1825{
1826#ifdef RT_ARCH_AMD64
1827 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x7);
1828#elif defined(RT_ARCH_ARM64)
1829 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
1830#else
1831# error "Port me!"
1832#endif
1833}
1834
1835
1836/**
1837 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
1838 * How @a offJmp is applied is are target specific.
1839 */
1840DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1841 int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
1842{
1843#ifdef RT_ARCH_AMD64
1844 /* jcc rel32 */
1845 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1846 AssertReturn(pbCodeBuf, UINT32_MAX);
1847 if (offTarget < 128 && offTarget >= -128)
1848 {
1849 pbCodeBuf[off++] = enmCond | 0x70;
1850 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
1851 }
1852 else
1853 {
1854 pbCodeBuf[off++] = 0x0f;
1855 pbCodeBuf[off++] = enmCond | 0x80;
1856 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
1857 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
1858 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
1859 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
1860 }
1861
1862#elif defined(RT_ARCH_ARM64)
1863 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1864 AssertReturn(pu32CodeBuf, UINT32_MAX);
1865 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
1866
1867#else
1868# error "Port me!"
1869#endif
1870 return off;
1871}
1872
1873
1874/**
1875 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
1876 * How @a offJmp is applied is are target specific.
1877 */
1878DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1879{
1880#ifdef RT_ARCH_AMD64
1881 return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x4);
1882#elif defined(RT_ARCH_ARM64)
1883 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
1884#else
1885# error "Port me!"
1886#endif
1887}
1888
1889
1890/**
1891 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
1892 * How @a offJmp is applied is are target specific.
1893 */
1894DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1895{
1896#ifdef RT_ARCH_AMD64
1897 return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x5);
1898#elif defined(RT_ARCH_ARM64)
1899 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
1900#else
1901# error "Port me!"
1902#endif
1903}
1904
1905
1906/**
1907 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
1908 * How @a offJmp is applied is are target specific.
1909 */
1910DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1911{
1912#ifdef RT_ARCH_AMD64
1913 return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x6);
1914#elif defined(RT_ARCH_ARM64)
1915 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
1916#else
1917# error "Port me!"
1918#endif
1919}
1920
1921
1922/**
1923 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
1924 * How @a offJmp is applied is are target specific.
1925 */
1926DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1927{
1928#ifdef RT_ARCH_AMD64
1929 return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x7);
1930#elif defined(RT_ARCH_ARM64)
1931 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
1932#else
1933# error "Port me!"
1934#endif
1935}
1936
1937
1938/**
1939 * Fixes up a conditional jump to a fixed label.
1940 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
1941 */
1942DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
1943{
1944# if defined(RT_ARCH_AMD64)
1945 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
1946 if (pbCodeBuf[offFixup] != 0x0f)
1947 {
1948 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
1949 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
1950 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
1951 }
1952 else
1953 {
1954 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
1955 uint32_t const offRel32 = offTarget - (offFixup + 6);
1956 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
1957 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
1958 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
1959 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
1960 }
1961
1962# elif defined(RT_ARCH_ARM64)
1963 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
1964 Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
1965 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
1966 | (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
1967
1968# endif
1969}
1970
1971
1972/**
1973 * Emits a call to a 64-bit address.
1974 */
1975DECLINLINE(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
1976{
1977#ifdef RT_ARCH_AMD64
1978 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
1979
1980 /* call rax */
1981 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1982 AssertReturn(pbCodeBuf, UINT32_MAX);
1983 pbCodeBuf[off++] = 0xff;
1984 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
1985
1986#elif defined(RT_ARCH_ARM64)
1987 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);
1988
1989 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1990 AssertReturn(pu32CodeBuf, UINT32_MAX);
1991 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);
1992#else
1993# error "port me"
1994#endif
1995 return off;
1996}
1997
1998
1999
2000/** @} */
2001
2002#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2003
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette