VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103750

Last change on this file since 103750 was 103750, checked in by vboxsync, 13 months ago

VMM/IEM: Implement some ofthe amd64 emitters for SSE/AVX, bugref:10614
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 70.4 KB
Line 
1/* $Id: IEMN8veRecompiler.h 103750 2024-03-10 20:12:55Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47 * Enables liveness analysis. */
48#if 1 || defined(DOXYGEN_RUNNING)
49# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
51#endif
52
53#ifdef VBOX_WITH_STATISTICS
54/** Always count instructions for now. */
55# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
56#endif
57
58
59/** @name Stack Frame Layout
60 *
61 * @{ */
62/** The size of the area for stack variables and spills and stuff.
63 * @note This limit is duplicated in the python script(s). We add 0x40 for
64 * alignment padding. */
65#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
66/** Number of 64-bit variable slots (0x100 / 8 = 32. */
67#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
68AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
69
70#ifdef RT_ARCH_AMD64
71/** An stack alignment adjustment (between non-volatile register pushes and
72 * the stack variable area, so the latter better aligned). */
73# define IEMNATIVE_FRAME_ALIGN_SIZE 8
74
75/** Number of stack arguments slots for calls made from the frame. */
76# ifdef RT_OS_WINDOWS
77# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
78# else
79# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
80# endif
81/** Number of any shadow arguments (spill area) for calls we make. */
82# ifdef RT_OS_WINDOWS
83# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
84# else
85# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
86# endif
87
88/** Frame pointer (RBP) relative offset of the last push. */
89# ifdef RT_OS_WINDOWS
90# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
91# else
92# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
93# endif
94/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
95 * address for it). */
96# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
97/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
98# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
99/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
100# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
101# ifdef RT_OS_WINDOWS
102/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
103# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
104/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
105# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
106# endif
107
108# ifdef RT_OS_WINDOWS
109/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
110# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
111/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
112# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
113/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
114# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
115/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
116# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
117# endif
118
119#elif RT_ARCH_ARM64
120/** No alignment padding needed for arm64. */
121# define IEMNATIVE_FRAME_ALIGN_SIZE 0
122/** No stack argument slots, got 8 registers for arguments will suffice. */
123# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
124/** There are no argument spill area. */
125# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
126
127/** Number of saved registers at the top of our stack frame.
128 * This includes the return address and old frame pointer, so x19 thru x30. */
129# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
130/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
131# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
132
133/** Frame pointer (BP) relative offset of the last push. */
134# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
135
136/** Frame pointer (BP) relative offset of the stack variable area (the lowest
137 * address for it). */
138# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
139
140#else
141# error "port me"
142#endif
143/** @} */
144
145
146/** @name Fixed Register Allocation(s)
147 * @{ */
148/** @def IEMNATIVE_REG_FIXED_PVMCPU
149 * The number of the register holding the pVCpu pointer. */
150/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
151 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
152 * @note This not available on AMD64, only ARM64. */
153/** @def IEMNATIVE_REG_FIXED_TMP0
154 * Dedicated temporary register.
155 * @todo replace this by a register allocator and content tracker. */
156/** @def IEMNATIVE_REG_FIXED_MASK
157 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
158 * architecture. */
159#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
160/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
161 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
162 * architecture. */
163/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
164 * Dedicated temporary SIMD register. */
165#endif
166#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
167# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
168# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
169# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
170 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
171 | RT_BIT_32(X86_GREG_xSP) \
172 | RT_BIT_32(X86_GREG_xBP) )
173
174# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
175# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
176# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) || !defined(_MSC_VER)
177# define IEMNATIVE_SIMD_REG_FIXED_MASK 0
178# else
179/** On Windows xmm6 through xmm15 are marked as callee saved. */
180# define IEMNATIVE_SIMD_REG_FIXED_MASK (UINT32_C(0xffc0))
181# endif
182# endif
183
184#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
185# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
186# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
187# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
188# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
189# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
190# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
191# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
192 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
193# else
194# define IEMNATIVE_REG_FIXED_MASK_ADD 0
195# endif
196# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
197 | RT_BIT_32(ARMV8_A64_REG_LR) \
198 | RT_BIT_32(ARMV8_A64_REG_BP) \
199 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
200 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
201 | RT_BIT_32(ARMV8_A64_REG_X18) \
202 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
203 | IEMNATIVE_REG_FIXED_MASK_ADD)
204
205# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
206# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
207# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
208# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
209# else
210/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
211# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
212 | RT_BIT_32(ARMV8_A64_REG_Q30))
213# endif
214# endif
215
216#else
217# error "port me"
218#endif
219/** @} */
220
221/** @name Call related registers.
222 * @{ */
223/** @def IEMNATIVE_CALL_RET_GREG
224 * The return value register. */
225/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
226 * Number of arguments in registers. */
227/** @def IEMNATIVE_CALL_ARG0_GREG
228 * The general purpose register carrying argument \#0. */
229/** @def IEMNATIVE_CALL_ARG1_GREG
230 * The general purpose register carrying argument \#1. */
231/** @def IEMNATIVE_CALL_ARG2_GREG
232 * The general purpose register carrying argument \#2. */
233/** @def IEMNATIVE_CALL_ARG3_GREG
234 * The general purpose register carrying argument \#3. */
235/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
236 * Mask of registers the callee will not save and may trash. */
237#ifdef RT_ARCH_AMD64
238# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
239
240# ifdef RT_OS_WINDOWS
241# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
242# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
243# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
244# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
245# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
246# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
247 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
248 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
249 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
250# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
251 | RT_BIT_32(X86_GREG_xCX) \
252 | RT_BIT_32(X86_GREG_xDX) \
253 | RT_BIT_32(X86_GREG_x8) \
254 | RT_BIT_32(X86_GREG_x9) \
255 | RT_BIT_32(X86_GREG_x10) \
256 | RT_BIT_32(X86_GREG_x11) )
257# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
258/* xmm0 - xmm5 are marked as volatile. */
259# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
260# endif
261
262# else
263# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
264# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
265# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
266# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
267# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
268# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
269# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
270# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
271 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
272 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
273 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
274 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
275 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
276# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
277 | RT_BIT_32(X86_GREG_xCX) \
278 | RT_BIT_32(X86_GREG_xDX) \
279 | RT_BIT_32(X86_GREG_xDI) \
280 | RT_BIT_32(X86_GREG_xSI) \
281 | RT_BIT_32(X86_GREG_x8) \
282 | RT_BIT_32(X86_GREG_x9) \
283 | RT_BIT_32(X86_GREG_x10) \
284 | RT_BIT_32(X86_GREG_x11) )
285# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
286/* xmm0 - xmm15 are marked as volatile. */
287# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
288# endif
289# endif
290
291#elif defined(RT_ARCH_ARM64)
292# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
293# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
294# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
295# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
296# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
297# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
298# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
299# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
300# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
301# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
302# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
303 | RT_BIT_32(ARMV8_A64_REG_X1) \
304 | RT_BIT_32(ARMV8_A64_REG_X2) \
305 | RT_BIT_32(ARMV8_A64_REG_X3) \
306 | RT_BIT_32(ARMV8_A64_REG_X4) \
307 | RT_BIT_32(ARMV8_A64_REG_X5) \
308 | RT_BIT_32(ARMV8_A64_REG_X6) \
309 | RT_BIT_32(ARMV8_A64_REG_X7) )
310# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
311 | RT_BIT_32(ARMV8_A64_REG_X1) \
312 | RT_BIT_32(ARMV8_A64_REG_X2) \
313 | RT_BIT_32(ARMV8_A64_REG_X3) \
314 | RT_BIT_32(ARMV8_A64_REG_X4) \
315 | RT_BIT_32(ARMV8_A64_REG_X5) \
316 | RT_BIT_32(ARMV8_A64_REG_X6) \
317 | RT_BIT_32(ARMV8_A64_REG_X7) \
318 | RT_BIT_32(ARMV8_A64_REG_X8) \
319 | RT_BIT_32(ARMV8_A64_REG_X9) \
320 | RT_BIT_32(ARMV8_A64_REG_X10) \
321 | RT_BIT_32(ARMV8_A64_REG_X11) \
322 | RT_BIT_32(ARMV8_A64_REG_X12) \
323 | RT_BIT_32(ARMV8_A64_REG_X13) \
324 | RT_BIT_32(ARMV8_A64_REG_X14) \
325 | RT_BIT_32(ARMV8_A64_REG_X15) \
326 | RT_BIT_32(ARMV8_A64_REG_X16) \
327 | RT_BIT_32(ARMV8_A64_REG_X17) )
328# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
329/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
330 * so to simplify our life a bit we just mark everything as volatile. */
331# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
332# endif
333
334#endif
335
336/** This is the maximum argument count we'll ever be needing. */
337#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
338# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
339#else
340# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
341#endif
342/** @} */
343
344
345/** @def IEMNATIVE_HST_GREG_COUNT
346 * Number of host general purpose registers we tracker. */
347/** @def IEMNATIVE_HST_GREG_MASK
348 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
349 * inverted register masks and such to get down to a correct set of regs. */
350#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
351/** @def IEMNATIVE_HST_SIMD_REG_COUNT
352 * Number of host SIMD registers we track. */
353/** @def IEMNATIVE_HST_SIMD_REG_MASK
354 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
355 * inverted register masks and such to get down to a correct set of regs. */
356#endif
357#ifdef RT_ARCH_AMD64
358# define IEMNATIVE_HST_GREG_COUNT 16
359# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
360
361# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
362# define IEMNATIVE_HST_SIMD_REG_COUNT 16
363# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
364# endif
365
366#elif defined(RT_ARCH_ARM64)
367# define IEMNATIVE_HST_GREG_COUNT 32
368# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
369
370# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
371# define IEMNATIVE_HST_SIMD_REG_COUNT 32
372# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
373# endif
374
375#else
376# error "Port me!"
377#endif
378
379
380/** Native code generator label types. */
381typedef enum
382{
383 kIemNativeLabelType_Invalid = 0,
384 /* Labels w/o data, only once instance per TB: */
385 kIemNativeLabelType_Return,
386 kIemNativeLabelType_ReturnBreak,
387 kIemNativeLabelType_ReturnWithFlags,
388 kIemNativeLabelType_NonZeroRetOrPassUp,
389 kIemNativeLabelType_RaiseGp0,
390 kIemNativeLabelType_RaiseNm,
391 kIemNativeLabelType_RaiseUd,
392 kIemNativeLabelType_RaiseMf,
393 kIemNativeLabelType_RaiseXf,
394 kIemNativeLabelType_ObsoleteTb,
395 kIemNativeLabelType_NeedCsLimChecking,
396 kIemNativeLabelType_CheckBranchMiss,
397 /* Labels with data, potentially multiple instances per TB: */
398 kIemNativeLabelType_FirstWithMultipleInstances,
399 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
400 kIemNativeLabelType_Else,
401 kIemNativeLabelType_Endif,
402 kIemNativeLabelType_CheckIrq,
403 kIemNativeLabelType_TlbLookup,
404 kIemNativeLabelType_TlbMiss,
405 kIemNativeLabelType_TlbDone,
406 kIemNativeLabelType_End
407} IEMNATIVELABELTYPE;
408
409/** Native code generator label definition. */
410typedef struct IEMNATIVELABEL
411{
412 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
413 * the epilog. */
414 uint32_t off;
415 /** The type of label (IEMNATIVELABELTYPE). */
416 uint16_t enmType;
417 /** Additional label data, type specific. */
418 uint16_t uData;
419} IEMNATIVELABEL;
420/** Pointer to a label. */
421typedef IEMNATIVELABEL *PIEMNATIVELABEL;
422
423
424/** Native code generator fixup types. */
425typedef enum
426{
427 kIemNativeFixupType_Invalid = 0,
428#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
429 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
430 kIemNativeFixupType_Rel32,
431#elif defined(RT_ARCH_ARM64)
432 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
433 kIemNativeFixupType_RelImm26At0,
434 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
435 kIemNativeFixupType_RelImm19At5,
436 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
437 kIemNativeFixupType_RelImm14At5,
438#endif
439 kIemNativeFixupType_End
440} IEMNATIVEFIXUPTYPE;
441
442/** Native code generator fixup. */
443typedef struct IEMNATIVEFIXUP
444{
445 /** Code offset of the fixup location. */
446 uint32_t off;
447 /** The IEMNATIVELABEL this is a fixup for. */
448 uint16_t idxLabel;
449 /** The fixup type (IEMNATIVEFIXUPTYPE). */
450 uint8_t enmType;
451 /** Addend or other data. */
452 int8_t offAddend;
453} IEMNATIVEFIXUP;
454/** Pointer to a native code generator fixup. */
455typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
456
457
458/**
459 * One bit of the state.
460 *
461 * Each register state takes up two bits. We keep the two bits in two separate
462 * 64-bit words to simplify applying them to the guest shadow register mask in
463 * the register allocator.
464 */
465typedef union IEMLIVENESSBIT
466{
467 uint64_t bm64;
468 RT_GCC_EXTENSION struct
469 { /* bit no */
470 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
471 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
472 uint64_t fCr0 : 1; /**< 0x11 / 17: */
473 uint64_t fFcw : 1; /**< 0x12 / 18: */
474 uint64_t fFsw : 1; /**< 0x13 / 19: */
475 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
476 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
477 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
478 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
479 uint64_t fCr4 : 1; /**< 0x2c / 44: */
480 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
481 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
482 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
483 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
484 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
485 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
486 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
487 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
488 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
489 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
490 };
491} IEMLIVENESSBIT;
492AssertCompileSize(IEMLIVENESSBIT, 8);
493
494#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
495#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
496#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
497#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
498#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
499#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
500#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
501
502
503/**
504 * A liveness state entry.
505 *
506 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
507 * Once we add a SSE register shadowing, we'll add another 64-bit element for
508 * that.
509 */
510typedef union IEMLIVENESSENTRY
511{
512#ifndef IEMLIVENESS_EXTENDED_LAYOUT
513 uint64_t bm64[16 / 8];
514 uint16_t bm32[16 / 4];
515 uint16_t bm16[16 / 2];
516 uint8_t bm8[ 16 / 1];
517 IEMLIVENESSBIT aBits[2];
518#else
519 uint64_t bm64[32 / 8];
520 uint16_t bm32[32 / 4];
521 uint16_t bm16[32 / 2];
522 uint8_t bm8[ 32 / 1];
523 IEMLIVENESSBIT aBits[4];
524#endif
525 RT_GCC_EXTENSION struct
526 {
527 /** Bit \#0 of the register states. */
528 IEMLIVENESSBIT Bit0;
529 /** Bit \#1 of the register states. */
530 IEMLIVENESSBIT Bit1;
531#ifdef IEMLIVENESS_EXTENDED_LAYOUT
532 /** Bit \#2 of the register states. */
533 IEMLIVENESSBIT Bit2;
534 /** Bit \#3 of the register states. */
535 IEMLIVENESSBIT Bit3;
536#endif
537 };
538} IEMLIVENESSENTRY;
539#ifndef IEMLIVENESS_EXTENDED_LAYOUT
540AssertCompileSize(IEMLIVENESSENTRY, 16);
541#else
542AssertCompileSize(IEMLIVENESSENTRY, 32);
543#endif
544/** Pointer to a liveness state entry. */
545typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
546/** Pointer to a const liveness state entry. */
547typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
548
549/** @name 64-bit value masks for IEMLIVENESSENTRY.
550 * @{ */ /* 0xzzzzyyyyxxxxwwww */
551#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
552
553#ifndef IEMLIVENESS_EXTENDED_LAYOUT
554# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
555# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
556
557# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
558# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
559#endif
560
561#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
562
563#ifndef IEMLIVENESS_EXTENDED_LAYOUT
564# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
565# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
566#endif
567/** @} */
568
569
570/** @name The liveness state for a register.
571 *
572 * The state values have been picked to with state accumulation in mind (what
573 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
574 * performance critical work done with the values.
575 *
576 * This is a compressed state that only requires 2 bits per register.
577 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
578 * 1. the incoming state from the following call,
579 * 2. the outgoing state for this call,
580 * 3. mask of the entries set in the 2nd.
581 *
582 * The mask entry (3rd one above) will be used both when updating the outgoing
583 * state and when merging in incoming state for registers not touched by the
584 * current call.
585 *
586 * @{ */
587#ifndef IEMLIVENESS_EXTENDED_LAYOUT
588/** The register will be clobbered and the current value thrown away.
589 *
590 * When this is applied to the state (2) we'll simply be AND'ing it with the
591 * (old) mask (3) and adding the register to the mask. This way we'll
592 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
593 * IEMLIVENESS_STATE_INPUT states. */
594# define IEMLIVENESS_STATE_CLOBBERED 0
595/** The register is unused in the remainder of the TB.
596 *
597 * This is an initial state and can not be set by any of the
598 * iemNativeLivenessFunc_xxxx callbacks. */
599# define IEMLIVENESS_STATE_UNUSED 1
600/** The register value is required in a potential call or exception.
601 *
602 * This means that the register value must be calculated and is best written to
603 * the state, but that any shadowing registers can be flushed thereafter as it's
604 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
605 *
606 * It is typically applied across the board, but we preserve incoming
607 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
608 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
609 * 1. r0 = old & ~mask;
610 * 2. r0 = t1 & (t1 >> 1)'
611 * 3. state |= r0 | 0b10;
612 * 4. mask = ~0;
613 */
614# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
615/** The register value is used as input.
616 *
617 * This means that the register value must be calculated and it is best to keep
618 * it in a register. It does not need to be writtent out as such. This is the
619 * highest priority state.
620 *
621 * Whether the call modifies the register or not isn't relevant to earlier
622 * calls, so that's not recorded.
623 *
624 * When applying this state we just or in the value in the outgoing state and
625 * mask. */
626# define IEMLIVENESS_STATE_INPUT 3
627/** Mask of the state bits. */
628# define IEMLIVENESS_STATE_MASK 3
629/** The number of bits per state. */
630# define IEMLIVENESS_STATE_BIT_COUNT 2
631/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
632# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
633/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
634# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
635/** Check if a register clobbering is expected given the (previous) liveness state.
636 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
637 * include INPUT if the register is used in more than one place. */
638# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
639
640#else /* IEMLIVENESS_EXTENDED_LAYOUT */
641/** The register is not used any more. */
642# define IEMLIVENESS_STATE_UNUSED 0
643/** Flag: The register is required in a potential exception or call. */
644# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
645# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
646/** Flag: The register is read. */
647# define IEMLIVENESS_STATE_READ 2
648# define IEMLIVENESS_BIT_READ 1
649/** Flag: The register is written. */
650# define IEMLIVENESS_STATE_WRITE 4
651# define IEMLIVENESS_BIT_WRITE 2
652/** Flag: Unconditional call (not needed, can be redefined for research). */
653# define IEMLIVENESS_STATE_CALL 8
654# define IEMLIVENESS_BIT_CALL 3
655# define IEMLIVENESS_BIT_OTHER 3 /**< More convenient name for this one. */
656# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
657 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
658# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
659# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
660#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
661/** @} */
662
663/** @name Liveness helpers for builtin functions and similar.
664 *
665 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
666 * own set of manimulator macros for those.
667 *
668 * @{ */
669/** Initializing the state as all unused. */
670#ifndef IEMLIVENESS_EXTENDED_LAYOUT
671# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
672 do { \
673 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
674 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
675 } while (0)
676#else
677# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
678 do { \
679 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
680 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
681 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
682 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
683 } while (0)
684#endif
685
686/** Initializing the outgoing state with a potential xcpt or call state.
687 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
688#ifndef IEMLIVENESS_EXTENDED_LAYOUT
689# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
690 do { \
691 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
692 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
693 } while (0)
694#else
695# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
696 do { \
697 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
698 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
699 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
700 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
701 } while (0)
702#endif
703
704/** Adds a segment base register as input to the outgoing state. */
705#ifndef IEMLIVENESS_EXTENDED_LAYOUT
706# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
707 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
708 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
709 } while (0)
710#else
711# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
712 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
713 } while (0)
714#endif
715
716/** Adds a segment attribute register as input to the outgoing state. */
717#ifndef IEMLIVENESS_EXTENDED_LAYOUT
718# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
719 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
720 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
721 } while (0)
722#else
723# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
724 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
725 } while (0)
726#endif
727
728/** Adds a segment limit register as input to the outgoing state. */
729#ifndef IEMLIVENESS_EXTENDED_LAYOUT
730# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
731 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
732 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
733 } while (0)
734#else
735# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
736 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
737 } while (0)
738#endif
739
740/** Adds a segment limit register as input to the outgoing state. */
741#ifndef IEMLIVENESS_EXTENDED_LAYOUT
742# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
743 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
744 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
745 } while (0)
746#else
747# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
748 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
749 } while (0)
750#endif
751/** @} */
752
753/**
754 * Guest registers that can be shadowed in GPRs.
755 *
756 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
757 * must be placed last, as the liveness state tracks it as 7 subcomponents and
758 * we don't want to waste space here.
759 *
760 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
761 * friends as well as IEMAllN8veLiveness.cpp.
762 */
763typedef enum IEMNATIVEGSTREG : uint8_t
764{
765 kIemNativeGstReg_GprFirst = 0,
766 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
767 kIemNativeGstReg_Pc,
768 kIemNativeGstReg_Cr0,
769 kIemNativeGstReg_FpuFcw,
770 kIemNativeGstReg_FpuFsw,
771 kIemNativeGstReg_SegBaseFirst,
772 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
773 kIemNativeGstReg_SegAttribFirst,
774 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
775 kIemNativeGstReg_SegLimitFirst,
776 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
777 kIemNativeGstReg_SegSelFirst,
778 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
779 kIemNativeGstReg_Cr4,
780 kIemNativeGstReg_Xcr0,
781 kIemNativeGstReg_MxCsr,
782 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
783 kIemNativeGstReg_End
784} IEMNATIVEGSTREG;
785AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
786AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
787
788/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
789 * @{ */
790#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
791#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
792#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
793#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
794#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
795/** @} */
796
797
798#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
799/**
800 * Guest registers that can be shadowed in host SIMD registers.
801 *
802 * @todo r=aeichner Liveness tracking
803 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
804 */
805typedef enum IEMNATIVEGSTSIMDREG : uint8_t
806{
807 kIemNativeGstSimdReg_SimdRegFirst = 0,
808 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
809 kIemNativeGstSimdReg_End
810} IEMNATIVEGSTSIMDREG;
811
812/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
813 * @{ */
814#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
815/** @} */
816
817/**
818 * The Load/store size for a SIMD guest register.
819 */
820typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
821{
822 /** Invalid size. */
823 kIemNativeGstSimdRegLdStSz_Invalid = 0,
824 /** Loads the low 128-bit of a guest SIMD register. */
825 kIemNativeGstSimdRegLdStSz_Low128,
826 /** Loads the high 128-bit of a guest SIMD register. */
827 kIemNativeGstSimdRegLdStSz_High128,
828 /** Loads the whole 256-bits of a guest SIMD register. */
829 kIemNativeGstSimdRegLdStSz_256,
830 /** End value. */
831 kIemNativeGstSimdRegLdStSz_End
832} IEMNATIVEGSTSIMDREGLDSTSZ;
833#endif
834
835
836/**
837 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
838 */
839typedef enum IEMNATIVEGSTREGUSE
840{
841 /** The usage is read-only, the register holding the guest register
842 * shadow copy will not be modified by the caller. */
843 kIemNativeGstRegUse_ReadOnly = 0,
844 /** The caller will update the guest register (think: PC += cbInstr).
845 * The guest shadow copy will follow the returned register. */
846 kIemNativeGstRegUse_ForUpdate,
847 /** The call will put an entirely new value in the guest register, so
848 * if new register is allocate it will be returned uninitialized. */
849 kIemNativeGstRegUse_ForFullWrite,
850 /** The caller will use the guest register value as input in a calculation
851 * and the host register will be modified.
852 * This means that the returned host register will not be marked as a shadow
853 * copy of the guest register. */
854 kIemNativeGstRegUse_Calculation
855} IEMNATIVEGSTREGUSE;
856
857/**
858 * Guest registers (classes) that can be referenced.
859 */
860typedef enum IEMNATIVEGSTREGREF : uint8_t
861{
862 kIemNativeGstRegRef_Invalid = 0,
863 kIemNativeGstRegRef_Gpr,
864 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
865 kIemNativeGstRegRef_EFlags,
866 kIemNativeGstRegRef_MxCsr,
867 kIemNativeGstRegRef_FpuReg,
868 kIemNativeGstRegRef_MReg,
869 kIemNativeGstRegRef_XReg,
870 //kIemNativeGstRegRef_YReg, - doesn't work.
871 kIemNativeGstRegRef_End
872} IEMNATIVEGSTREGREF;
873
874
875/** Variable kinds. */
876typedef enum IEMNATIVEVARKIND : uint8_t
877{
878 /** Customary invalid zero value. */
879 kIemNativeVarKind_Invalid = 0,
880 /** This is either in a register or on the stack. */
881 kIemNativeVarKind_Stack,
882 /** Immediate value - loaded into register when needed, or can live on the
883 * stack if referenced (in theory). */
884 kIemNativeVarKind_Immediate,
885 /** Variable reference - loaded into register when needed, never stack. */
886 kIemNativeVarKind_VarRef,
887 /** Guest register reference - loaded into register when needed, never stack. */
888 kIemNativeVarKind_GstRegRef,
889 /** End of valid values. */
890 kIemNativeVarKind_End
891} IEMNATIVEVARKIND;
892
893
894/** Variable or argument. */
895typedef struct IEMNATIVEVAR
896{
897 /** The kind of variable. */
898 IEMNATIVEVARKIND enmKind;
899 /** The variable size in bytes. */
900 uint8_t cbVar;
901 /** The first stack slot (uint64_t), except for immediate and references
902 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
903 * has a stack slot it has been initialized and has a value. Unused variables
904 * has neither a stack slot nor a host register assignment. */
905 uint8_t idxStackSlot;
906 /** The host register allocated for the variable, UINT8_MAX if not. */
907 uint8_t idxReg;
908 /** The argument number if argument, UINT8_MAX if regular variable. */
909 uint8_t uArgNo;
910 /** If referenced, the index (unpacked) of the variable referencing this one,
911 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
912 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
913 uint8_t idxReferrerVar;
914 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
915 * @todo not sure what this really is for... */
916 IEMNATIVEGSTREG enmGstReg;
917 /** Set if the registered is currently used exclusively, false if the
918 * variable is idle and the register can be grabbed. */
919 bool fRegAcquired;
920
921 union
922 {
923 /** kIemNativeVarKind_Immediate: The immediate value. */
924 uint64_t uValue;
925 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
926 uint8_t idxRefVar;
927 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
928 struct
929 {
930 /** The class of register. */
931 IEMNATIVEGSTREGREF enmClass;
932 /** Index within the class. */
933 uint8_t idx;
934 } GstRegRef;
935 } u;
936} IEMNATIVEVAR;
937/** Pointer to a variable or argument. */
938typedef IEMNATIVEVAR *PIEMNATIVEVAR;
939/** Pointer to a const variable or argument. */
940typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
941
942/** What is being kept in a host register. */
943typedef enum IEMNATIVEWHAT : uint8_t
944{
945 /** The traditional invalid zero value. */
946 kIemNativeWhat_Invalid = 0,
947 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
948 kIemNativeWhat_Var,
949 /** Temporary register, this is typically freed when a MC completes. */
950 kIemNativeWhat_Tmp,
951 /** Call argument w/o a variable mapping. This is free (via
952 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
953 kIemNativeWhat_Arg,
954 /** Return status code.
955 * @todo not sure if we need this... */
956 kIemNativeWhat_rc,
957 /** The fixed pVCpu (PVMCPUCC) register.
958 * @todo consider offsetting this on amd64 to use negative offsets to access
959 * more members using 8-byte disp. */
960 kIemNativeWhat_pVCpuFixed,
961 /** The fixed pCtx (PCPUMCTX) register.
962 * @todo consider offsetting this on amd64 to use negative offsets to access
963 * more members using 8-byte disp. */
964 kIemNativeWhat_pCtxFixed,
965 /** Fixed temporary register. */
966 kIemNativeWhat_FixedTmp,
967#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
968 /** Shadow RIP for the delayed RIP updating debugging. */
969 kIemNativeWhat_PcShadow,
970#endif
971 /** Register reserved by the CPU or OS architecture. */
972 kIemNativeWhat_FixedReserved,
973 /** End of valid values. */
974 kIemNativeWhat_End
975} IEMNATIVEWHAT;
976
977/**
978 * Host general register entry.
979 *
980 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
981 *
982 * @todo Track immediate values in host registers similarlly to how we track the
983 * guest register shadow copies. For it to be real helpful, though,
984 * we probably need to know which will be reused and put them into
985 * non-volatile registers, otherwise it's going to be more or less
986 * restricted to an instruction or two.
987 */
988typedef struct IEMNATIVEHSTREG
989{
990 /** Set of guest registers this one shadows.
991 *
992 * Using a bitmap here so we can designate the same host register as a copy
993 * for more than one guest register. This is expected to be useful in
994 * situations where one value is copied to several registers in a sequence.
995 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
996 * sequence we'd want to let this register follow to be a copy of and there
997 * will always be places where we'd be picking the wrong one.
998 */
999 uint64_t fGstRegShadows;
1000 /** What is being kept in this register. */
1001 IEMNATIVEWHAT enmWhat;
1002 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1003 uint8_t idxVar;
1004 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1005 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1006 * that scope. */
1007 uint8_t idxStackSlot;
1008 /** Alignment padding. */
1009 uint8_t abAlign[5];
1010} IEMNATIVEHSTREG;
1011
1012
1013#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1014/**
1015 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1016 * halves, on architectures where there is no 256-bit register available this entry will track
1017 * two adjacent 128-bit host registers.
1018 *
1019 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1020 */
1021typedef struct IEMNATIVEHSTSIMDREG
1022{
1023 /** Set of guest registers this one shadows.
1024 *
1025 * Using a bitmap here so we can designate the same host register as a copy
1026 * for more than one guest register. This is expected to be useful in
1027 * situations where one value is copied to several registers in a sequence.
1028 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1029 * sequence we'd want to let this register follow to be a copy of and there
1030 * will always be places where we'd be picking the wrong one.
1031 */
1032 uint64_t fGstRegShadows;
1033 /** What is being kept in this register. */
1034 IEMNATIVEWHAT enmWhat;
1035 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1036 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1037 /** Alignment padding. */
1038 uint8_t abAlign[6];
1039} IEMNATIVEHSTSIMDREG;
1040#endif
1041
1042
1043/**
1044 * Core state for the native recompiler, that is, things that needs careful
1045 * handling when dealing with branches.
1046 */
1047typedef struct IEMNATIVECORESTATE
1048{
1049#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1050 /** The current instruction offset in bytes from when the guest program counter
1051 * was updated last. Used for delaying the write to the guest context program counter
1052 * as long as possible. */
1053 uint32_t offPc;
1054 /** Number of instructions where we could skip the updating. */
1055 uint32_t cInstrPcUpdateSkipped;
1056#endif
1057 /** Allocation bitmap for aHstRegs. */
1058 uint32_t bmHstRegs;
1059
1060 /** Bitmap marking which host register contains guest register shadow copies.
1061 * This is used during register allocation to try preserve copies. */
1062 uint32_t bmHstRegsWithGstShadow;
1063 /** Bitmap marking valid entries in aidxGstRegShadows. */
1064 uint64_t bmGstRegShadows;
1065
1066#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1067 /** Allocation bitmap for aHstSimdRegs. */
1068 uint32_t bmHstSimdRegs;
1069
1070 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1071 * This is used during register allocation to try preserve copies. */
1072 uint32_t bmHstSimdRegsWithGstShadow;
1073 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1074 uint64_t bmGstSimdRegShadows;
1075 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1076 uint64_t bmGstSimdRegShadowDirtyLo128;
1077 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1078 uint64_t bmGstSimdRegShadowDirtyHi128;
1079#endif
1080
1081 union
1082 {
1083 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1084 uint8_t aidxArgVars[8];
1085 /** For more efficient resetting. */
1086 uint64_t u64ArgVars;
1087 };
1088
1089 /** Allocation bitmap for the stack. */
1090 uint32_t bmStack;
1091 /** Allocation bitmap for aVars. */
1092 uint32_t bmVars;
1093
1094 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1095 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1096 * (A shadow copy of a guest register can only be held in a one host register,
1097 * there are no duplicate copies or ambiguities like that). */
1098 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1099#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1100 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1101 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1102 * (A shadow copy of a guest register can only be held in a one host register,
1103 * there are no duplicate copies or ambiguities like that). */
1104 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1105#endif
1106
1107 /** Host register allocation tracking. */
1108 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1109#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1110 /** Host SIMD register allocation tracking. */
1111 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1112#endif
1113
1114 /** Variables and arguments. */
1115 IEMNATIVEVAR aVars[9];
1116} IEMNATIVECORESTATE;
1117/** Pointer to core state. */
1118typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1119/** Pointer to const core state. */
1120typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1121
1122/** @def IEMNATIVE_VAR_IDX_UNPACK
1123 * @returns Index into IEMNATIVECORESTATE::aVars.
1124 * @param a_idxVar Variable index w/ magic (in strict builds).
1125 */
1126/** @def IEMNATIVE_VAR_IDX_PACK
1127 * @returns Variable index w/ magic (in strict builds).
1128 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1129 */
1130#ifdef VBOX_STRICT
1131# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1132# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1133# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1134# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1135# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1136#else
1137# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1138# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1139#endif
1140
1141
1142#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1143/** Clear the dirty state of the given guest SIMD register. */
1144# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1145 do { \
1146 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1147 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1148 } while (0)
1149
1150/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1151# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1152 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1153/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1154# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1155 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1156/** Returns whether the given guest SIMD register is dirty. */
1157# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1158 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1159
1160/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1161# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1162 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1163/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1164# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1165 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1166#endif
1167
1168
1169/**
1170 * Conditional stack entry.
1171 */
1172typedef struct IEMNATIVECOND
1173{
1174 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1175 bool fInElse;
1176 /** The label for the IEM_MC_ELSE. */
1177 uint32_t idxLabelElse;
1178 /** The label for the IEM_MC_ENDIF. */
1179 uint32_t idxLabelEndIf;
1180 /** The initial state snapshot as the if-block starts executing. */
1181 IEMNATIVECORESTATE InitialState;
1182 /** The state snapshot at the end of the if-block. */
1183 IEMNATIVECORESTATE IfFinalState;
1184} IEMNATIVECOND;
1185/** Pointer to a condition stack entry. */
1186typedef IEMNATIVECOND *PIEMNATIVECOND;
1187
1188
1189/**
1190 * Native recompiler state.
1191 */
1192typedef struct IEMRECOMPILERSTATE
1193{
1194 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1195 * IEMNATIVEINSTR units. */
1196 uint32_t cInstrBufAlloc;
1197#ifdef VBOX_STRICT
1198 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1199 uint32_t offInstrBufChecked;
1200#else
1201 uint32_t uPadding1; /* We don't keep track of the size here... */
1202#endif
1203 /** Fixed temporary code buffer for native recompilation. */
1204 PIEMNATIVEINSTR pInstrBuf;
1205
1206 /** Bitmaps with the label types used. */
1207 uint64_t bmLabelTypes;
1208 /** Actual number of labels in paLabels. */
1209 uint32_t cLabels;
1210 /** Max number of entries allowed in paLabels before reallocating it. */
1211 uint32_t cLabelsAlloc;
1212 /** Labels defined while recompiling (referenced by fixups). */
1213 PIEMNATIVELABEL paLabels;
1214 /** Array with indexes of unique labels (uData always 0). */
1215 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1216
1217 /** Actual number of fixups paFixups. */
1218 uint32_t cFixups;
1219 /** Max number of entries allowed in paFixups before reallocating it. */
1220 uint32_t cFixupsAlloc;
1221 /** Buffer used by the recompiler for recording fixups when generating code. */
1222 PIEMNATIVEFIXUP paFixups;
1223
1224#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1225 /** Number of debug info entries allocated for pDbgInfo. */
1226 uint32_t cDbgInfoAlloc;
1227 uint32_t uPadding;
1228 /** Debug info. */
1229 PIEMTBDBG pDbgInfo;
1230#endif
1231
1232#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1233 /** The current call index (liveness array and threaded calls in TB). */
1234 uint32_t idxCurCall;
1235 /** Number of liveness entries allocated. */
1236 uint32_t cLivenessEntriesAlloc;
1237 /** Liveness entries for all the calls in the TB begin recompiled.
1238 * The entry for idxCurCall contains the info for what the next call will
1239 * require wrt registers. (Which means the last entry is the initial liveness
1240 * state.) */
1241 PIEMLIVENESSENTRY paLivenessEntries;
1242#endif
1243
1244 /** The translation block being recompiled. */
1245 PCIEMTB pTbOrg;
1246 /** The VMCPU structure of the EMT. */
1247 PVMCPUCC pVCpu;
1248
1249 /** Condition sequence number (for generating unique labels). */
1250 uint16_t uCondSeqNo;
1251 /** Check IRQ seqeunce number (for generating unique labels). */
1252 uint16_t uCheckIrqSeqNo;
1253 /** TLB load sequence number (for generating unique labels). */
1254 uint16_t uTlbSeqNo;
1255 /** The current condition stack depth (aCondStack). */
1256 uint8_t cCondDepth;
1257
1258 /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1259 uint8_t cArgs;
1260 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1261 uint32_t fCImpl;
1262 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1263 uint32_t fMc;
1264 /** The expected IEMCPU::fExec value for the current call/instruction. */
1265 uint32_t fExec;
1266
1267 /** Core state requiring care with branches. */
1268 IEMNATIVECORESTATE Core;
1269
1270 /** The condition nesting stack. */
1271 IEMNATIVECOND aCondStack[2];
1272
1273#ifndef IEM_WITH_THROW_CATCH
1274 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1275 * for recompilation error handling. */
1276 jmp_buf JmpBuf;
1277#endif
1278} IEMRECOMPILERSTATE;
1279/** Pointer to a native recompiler state. */
1280typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1281
1282
1283/** @def IEMNATIVE_TRY_SETJMP
1284 * Wrapper around setjmp / try, hiding all the ugly differences.
1285 *
1286 * @note Use with extreme care as this is a fragile macro.
1287 * @param a_pReNative The native recompile state.
1288 * @param a_rcTarget The variable that should receive the status code in case
1289 * of a longjmp/throw.
1290 */
1291/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1292 * Start wrapper for catch / setjmp-else.
1293 *
1294 * This will set up a scope.
1295 *
1296 * @note Use with extreme care as this is a fragile macro.
1297 * @param a_pReNative The native recompile state.
1298 * @param a_rcTarget The variable that should receive the status code in case
1299 * of a longjmp/throw.
1300 */
1301/** @def IEMNATIVE_CATCH_LONGJMP_END
1302 * End wrapper for catch / setjmp-else.
1303 *
1304 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1305 * up the state.
1306 *
1307 * @note Use with extreme care as this is a fragile macro.
1308 * @param a_pReNative The native recompile state.
1309 */
1310/** @def IEMNATIVE_DO_LONGJMP
1311 *
1312 * Wrapper around longjmp / throw.
1313 *
1314 * @param a_pReNative The native recompile state.
1315 * @param a_rc The status code jump back with / throw.
1316 */
1317#ifdef IEM_WITH_THROW_CATCH
1318# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1319 a_rcTarget = VINF_SUCCESS; \
1320 try
1321# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1322 catch (int rcThrown) \
1323 { \
1324 a_rcTarget = rcThrown
1325# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1326 } \
1327 ((void)0)
1328# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1329#else /* !IEM_WITH_THROW_CATCH */
1330# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1331 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1332# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1333 else \
1334 { \
1335 ((void)0)
1336# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1337 }
1338# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1339#endif /* !IEM_WITH_THROW_CATCH */
1340
1341
1342/**
1343 * Native recompiler worker for a threaded function.
1344 *
1345 * @returns New code buffer offset; throws VBox status code in case of a failure.
1346 * @param pReNative The native recompiler state.
1347 * @param off The current code buffer offset.
1348 * @param pCallEntry The threaded call entry.
1349 *
1350 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1351 */
1352typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1353/** Pointer to a native recompiler worker for a threaded function. */
1354typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1355
1356/** Defines a native recompiler worker for a threaded function.
1357 * @see FNIEMNATIVERECOMPFUNC */
1358#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1359 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1360
1361/** Prototypes a native recompiler function for a threaded function.
1362 * @see FNIEMNATIVERECOMPFUNC */
1363#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1364
1365
1366/**
1367 * Native recompiler liveness analysis worker for a threaded function.
1368 *
1369 * @param pCallEntry The threaded call entry.
1370 * @param pIncoming The incoming liveness state entry.
1371 * @param pOutgoing The outgoing liveness state entry.
1372 */
1373typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1374 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1375/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1376typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1377
1378/** Defines a native recompiler liveness analysis worker for a threaded function.
1379 * @see FNIEMNATIVELIVENESSFUNC */
1380#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1381 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1382
1383/** Prototypes a native recompiler liveness analysis function for a threaded function.
1384 * @see FNIEMNATIVELIVENESSFUNC */
1385#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1386
1387
1388/** Define a native recompiler helper function, safe to call from the TB code. */
1389#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1390 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1391/** Prototype a native recompiler helper function, safe to call from the TB code. */
1392#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1393 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1394
1395
1396DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1397 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1398DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1399DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1400 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1401DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1402
1403DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1404DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1405 bool fPreferVolatile = true);
1406DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1407 bool fPreferVolatile = true);
1408DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1409 IEMNATIVEGSTREG enmGstReg,
1410 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1411 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1412DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1413 IEMNATIVEGSTREG enmGstReg);
1414
1415DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1416DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1417DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1418DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1419DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1420DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1421DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1422DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExept = 0, bool fFlushShadows = true);
1423DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1424 uint32_t fKeepVars = 0);
1425DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1426DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1427DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1428 uint32_t fHstRegsActiveShadows);
1429
1430DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1431DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1432 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1433DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1434 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1435DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1436 uint32_t fHstRegsNotToSave);
1437DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1438 uint32_t fHstRegsNotToSave);
1439
1440DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1441 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1442DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1443DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1444 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1445 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1446DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1447 PCIEMTHRDEDCALLENTRY pCallEntry);
1448
1449#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1450DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1451DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1452 bool fPreferVolatile = true);
1453DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1454 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz, IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1455 bool fNoVolatileRegs = false);
1456DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1457 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1458 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1459#endif
1460
1461extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1462
1463
1464/**
1465 * Ensures that there is sufficient space in the instruction output buffer.
1466 *
1467 * This will reallocate the buffer if needed and allowed.
1468 *
1469 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1470 * allocation size.
1471 *
1472 * @returns Pointer to the instruction output buffer on success; throws VBox
1473 * status code on failure, so no need to check it.
1474 * @param pReNative The native recompile state.
1475 * @param off Current instruction offset. Works safely for UINT32_MAX
1476 * as well.
1477 * @param cInstrReq Number of instruction about to be added. It's okay to
1478 * overestimate this a bit.
1479 */
1480DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1481iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1482{
1483 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1484 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1485 {
1486#ifdef VBOX_STRICT
1487 pReNative->offInstrBufChecked = offChecked;
1488#endif
1489 return pReNative->pInstrBuf;
1490 }
1491 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1492}
1493
1494/**
1495 * Checks that we didn't exceed the space requested in the last
1496 * iemNativeInstrBufEnsure() call.
1497 */
1498#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1499 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1500 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1501
1502/**
1503 * Checks that a variable index is valid.
1504 */
1505#ifdef IEMNATIVE_VAR_IDX_MAGIC
1506# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1507 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1508 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1509 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1510 ("%s=%#x\n", #a_idxVar, a_idxVar))
1511#else
1512# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1513 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1514 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1515#endif
1516
1517/**
1518 * Checks that a variable index is valid and that the variable is assigned the
1519 * correct argument number.
1520 * This also adds a RT_NOREF of a_idxVar.
1521 */
1522#ifdef IEMNATIVE_VAR_IDX_MAGIC
1523# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1524 RT_NOREF_PV(a_idxVar); \
1525 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1526 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1527 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1528 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1529 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1530 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1531 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1532 a_uArgNo)); \
1533 } while (0)
1534#else
1535# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1536 RT_NOREF_PV(a_idxVar); \
1537 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1538 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1539 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1540 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1541 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1542 } while (0)
1543#endif
1544
1545
1546/**
1547 * Checks that a variable has the expected size.
1548 */
1549#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1550 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1551 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1552 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1553
1554
1555/**
1556 * Calculates the stack address of a variable as a [r]BP displacement value.
1557 */
1558DECL_FORCE_INLINE(int32_t)
1559iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1560{
1561 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1562 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1563}
1564
1565
1566/**
1567 * Releases the variable's register.
1568 *
1569 * The register must have been previously acquired calling
1570 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1571 * iemNativeVarRegisterSetAndAcquire().
1572 */
1573DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1574{
1575 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1576 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1577 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1578}
1579
1580/** @} */
1581
1582#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1583
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette