VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103875

Last change on this file since 103875 was 103874, checked in by vboxsync, 13 months ago

VMM/IEM: Comments on lables. bugref:10370
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 115.2 KB
Line 
1/* $Id: IEMN8veRecompiler.h 103874 2024-03-16 01:17:39Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47 * Enables liveness analysis. */
48#if 1 || defined(DOXYGEN_RUNNING)
49# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
51#endif
52
53/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54 * Enables skipping EFLAGS calculations/updating based on liveness info. */
55#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) || defined(DOXYGEN_RUNNING)
56# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57#endif
58
59
60/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61 * Enables strict consistency checks around EFLAGS skipping.
62 * @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) || defined(DOXYGEN_RUNNING)
64# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65#endif
66
67#ifdef VBOX_WITH_STATISTICS
68/** Always count instructions for now. */
69# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70#endif
71
72
73/** @name Stack Frame Layout
74 *
75 * @{ */
76/** The size of the area for stack variables and spills and stuff.
77 * @note This limit is duplicated in the python script(s). We add 0x40 for
78 * alignment padding. */
79#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84#ifdef RT_ARCH_AMD64
85/** An stack alignment adjustment (between non-volatile register pushes and
86 * the stack variable area, so the latter better aligned). */
87# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89/** Number of stack arguments slots for calls made from the frame. */
90# ifdef RT_OS_WINDOWS
91# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92# else
93# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94# endif
95/** Number of any shadow arguments (spill area) for calls we make. */
96# ifdef RT_OS_WINDOWS
97# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98# else
99# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100# endif
101
102/** Frame pointer (RBP) relative offset of the last push. */
103# ifdef RT_OS_WINDOWS
104# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105# else
106# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107# endif
108/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109 * address for it). */
110# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115# ifdef RT_OS_WINDOWS
116/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120# endif
121
122# ifdef RT_OS_WINDOWS
123/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131# endif
132
133#elif RT_ARCH_ARM64
134/** No alignment padding needed for arm64. */
135# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136/** No stack argument slots, got 8 registers for arguments will suffice. */
137# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138/** There are no argument spill area. */
139# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141/** Number of saved registers at the top of our stack frame.
142 * This includes the return address and old frame pointer, so x19 thru x30. */
143# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147/** Frame pointer (BP) relative offset of the last push. */
148# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151 * address for it). */
152# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154#else
155# error "port me"
156#endif
157/** @} */
158
159
160/** @name Fixed Register Allocation(s)
161 * @{ */
162/** @def IEMNATIVE_REG_FIXED_PVMCPU
163 * The number of the register holding the pVCpu pointer. */
164/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166 * @note This not available on AMD64, only ARM64. */
167/** @def IEMNATIVE_REG_FIXED_TMP0
168 * Dedicated temporary register.
169 * @todo replace this by a register allocator and content tracker. */
170/** @def IEMNATIVE_REG_FIXED_MASK
171 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172 * architecture. */
173#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176 * architecture. */
177/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178 * Dedicated temporary SIMD register. */
179#endif
180#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185 | RT_BIT_32(X86_GREG_xSP) \
186 | RT_BIT_32(X86_GREG_xBP) )
187
188# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) || !defined(_MSC_VER)
191# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0)
192# else
193/** On Windows xmm6 through xmm15 are marked as callee saved. */
194# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195 | RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196# endif
197# endif
198
199#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
200# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208# else
209# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210# endif
211# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212 | RT_BIT_32(ARMV8_A64_REG_LR) \
213 | RT_BIT_32(ARMV8_A64_REG_BP) \
214 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216 | RT_BIT_32(ARMV8_A64_REG_X18) \
217 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218 | IEMNATIVE_REG_FIXED_MASK_ADD)
219
220# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224# else
225/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
226# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
227 | RT_BIT_32(ARMV8_A64_REG_Q30))
228# endif
229# endif
230
231#else
232# error "port me"
233#endif
234/** @} */
235
236/** @name Call related registers.
237 * @{ */
238/** @def IEMNATIVE_CALL_RET_GREG
239 * The return value register. */
240/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
241 * Number of arguments in registers. */
242/** @def IEMNATIVE_CALL_ARG0_GREG
243 * The general purpose register carrying argument \#0. */
244/** @def IEMNATIVE_CALL_ARG1_GREG
245 * The general purpose register carrying argument \#1. */
246/** @def IEMNATIVE_CALL_ARG2_GREG
247 * The general purpose register carrying argument \#2. */
248/** @def IEMNATIVE_CALL_ARG3_GREG
249 * The general purpose register carrying argument \#3. */
250/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
251 * Mask of registers the callee will not save and may trash. */
252#ifdef RT_ARCH_AMD64
253# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
254
255# ifdef RT_OS_WINDOWS
256# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
257# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
258# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
259# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
260# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
261# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
262 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
263 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
264 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
265# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
266 | RT_BIT_32(X86_GREG_xCX) \
267 | RT_BIT_32(X86_GREG_xDX) \
268 | RT_BIT_32(X86_GREG_x8) \
269 | RT_BIT_32(X86_GREG_x9) \
270 | RT_BIT_32(X86_GREG_x10) \
271 | RT_BIT_32(X86_GREG_x11) )
272# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
273/* xmm0 - xmm5 are marked as volatile. */
274# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
275# endif
276
277# else
278# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
279# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
280# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
281# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
282# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
283# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
284# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
285# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
289 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
290 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
291# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
292 | RT_BIT_32(X86_GREG_xCX) \
293 | RT_BIT_32(X86_GREG_xDX) \
294 | RT_BIT_32(X86_GREG_xDI) \
295 | RT_BIT_32(X86_GREG_xSI) \
296 | RT_BIT_32(X86_GREG_x8) \
297 | RT_BIT_32(X86_GREG_x9) \
298 | RT_BIT_32(X86_GREG_x10) \
299 | RT_BIT_32(X86_GREG_x11) )
300# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
301/* xmm0 - xmm15 are marked as volatile. */
302# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
303# endif
304# endif
305
306#elif defined(RT_ARCH_ARM64)
307# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
308# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
309# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
310# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
311# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
312# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
313# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
314# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
315# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
316# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
317# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
318 | RT_BIT_32(ARMV8_A64_REG_X1) \
319 | RT_BIT_32(ARMV8_A64_REG_X2) \
320 | RT_BIT_32(ARMV8_A64_REG_X3) \
321 | RT_BIT_32(ARMV8_A64_REG_X4) \
322 | RT_BIT_32(ARMV8_A64_REG_X5) \
323 | RT_BIT_32(ARMV8_A64_REG_X6) \
324 | RT_BIT_32(ARMV8_A64_REG_X7) )
325# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
326 | RT_BIT_32(ARMV8_A64_REG_X1) \
327 | RT_BIT_32(ARMV8_A64_REG_X2) \
328 | RT_BIT_32(ARMV8_A64_REG_X3) \
329 | RT_BIT_32(ARMV8_A64_REG_X4) \
330 | RT_BIT_32(ARMV8_A64_REG_X5) \
331 | RT_BIT_32(ARMV8_A64_REG_X6) \
332 | RT_BIT_32(ARMV8_A64_REG_X7) \
333 | RT_BIT_32(ARMV8_A64_REG_X8) \
334 | RT_BIT_32(ARMV8_A64_REG_X9) \
335 | RT_BIT_32(ARMV8_A64_REG_X10) \
336 | RT_BIT_32(ARMV8_A64_REG_X11) \
337 | RT_BIT_32(ARMV8_A64_REG_X12) \
338 | RT_BIT_32(ARMV8_A64_REG_X13) \
339 | RT_BIT_32(ARMV8_A64_REG_X14) \
340 | RT_BIT_32(ARMV8_A64_REG_X15) \
341 | RT_BIT_32(ARMV8_A64_REG_X16) \
342 | RT_BIT_32(ARMV8_A64_REG_X17) )
343# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
344/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
345 * so to simplify our life a bit we just mark everything as volatile. */
346# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
347# endif
348
349#endif
350
351/** This is the maximum argument count we'll ever be needing. */
352#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
353# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
354#else
355# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
356#endif
357/** @} */
358
359
360/** @def IEMNATIVE_HST_GREG_COUNT
361 * Number of host general purpose registers we tracker. */
362/** @def IEMNATIVE_HST_GREG_MASK
363 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
364 * inverted register masks and such to get down to a correct set of regs. */
365#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
366/** @def IEMNATIVE_HST_SIMD_REG_COUNT
367 * Number of host SIMD registers we track. */
368/** @def IEMNATIVE_HST_SIMD_REG_MASK
369 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
370 * inverted register masks and such to get down to a correct set of regs. */
371#endif
372#ifdef RT_ARCH_AMD64
373# define IEMNATIVE_HST_GREG_COUNT 16
374# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
375
376# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
377# define IEMNATIVE_HST_SIMD_REG_COUNT 16
378# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
379# endif
380
381#elif defined(RT_ARCH_ARM64)
382# define IEMNATIVE_HST_GREG_COUNT 32
383# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
384
385# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
386# define IEMNATIVE_HST_SIMD_REG_COUNT 32
387# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
388# endif
389
390#else
391# error "Port me!"
392#endif
393
394
395/** Native code generator label types. */
396typedef enum
397{
398 kIemNativeLabelType_Invalid = 0,
399 /*
400 * Labels w/o data, only once instance per TB.
401 *
402 * Note! Jumps to these requires instructions that are capable of spanning
403 * the max TB length.
404 */
405 /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
406 kIemNativeLabelType_RaiseDe, /**< Raise (throw) X86_XCPT_DE (00h). */
407 kIemNativeLabelType_RaiseUd, /**< Raise (throw) X86_XCPT_UD (06h). */
408 kIemNativeLabelType_RaiseSseRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
409 kIemNativeLabelType_RaiseAvxRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
410 kIemNativeLabelType_RaiseNm, /**< Raise (throw) X86_XCPT_NM (07h). */
411 kIemNativeLabelType_RaiseGp0, /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
412 kIemNativeLabelType_RaiseMf, /**< Raise (throw) X86_XCPT_MF (10h). */
413 kIemNativeLabelType_RaiseXf, /**< Raise (throw) X86_XCPT_XF (13h). */
414 kIemNativeLabelType_ObsoleteTb,
415 kIemNativeLabelType_NeedCsLimChecking,
416 kIemNativeLabelType_CheckBranchMiss,
417 kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
418 /* Manually defined labels. */
419 kIemNativeLabelType_Return,
420 kIemNativeLabelType_ReturnBreak,
421 kIemNativeLabelType_ReturnWithFlags,
422 kIemNativeLabelType_NonZeroRetOrPassUp,
423
424 /*
425 * Labels with data, potentially multiple instances per TB:
426 *
427 * These are localized labels, so no fixed jump type restrictions here.
428 */
429 kIemNativeLabelType_FirstWithMultipleInstances,
430 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
431 kIemNativeLabelType_Else,
432 kIemNativeLabelType_Endif,
433 kIemNativeLabelType_CheckIrq,
434 kIemNativeLabelType_TlbLookup,
435 kIemNativeLabelType_TlbMiss,
436 kIemNativeLabelType_TlbDone,
437 kIemNativeLabelType_End
438} IEMNATIVELABELTYPE;
439
440/** Native code generator label definition. */
441typedef struct IEMNATIVELABEL
442{
443 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
444 * the epilog. */
445 uint32_t off;
446 /** The type of label (IEMNATIVELABELTYPE). */
447 uint16_t enmType;
448 /** Additional label data, type specific. */
449 uint16_t uData;
450} IEMNATIVELABEL;
451/** Pointer to a label. */
452typedef IEMNATIVELABEL *PIEMNATIVELABEL;
453
454
455/** Native code generator fixup types. */
456typedef enum
457{
458 kIemNativeFixupType_Invalid = 0,
459#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
460 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
461 kIemNativeFixupType_Rel32,
462#elif defined(RT_ARCH_ARM64)
463 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
464 kIemNativeFixupType_RelImm26At0,
465 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
466 kIemNativeFixupType_RelImm19At5,
467 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
468 kIemNativeFixupType_RelImm14At5,
469#endif
470 kIemNativeFixupType_End
471} IEMNATIVEFIXUPTYPE;
472
473/** Native code generator fixup. */
474typedef struct IEMNATIVEFIXUP
475{
476 /** Code offset of the fixup location. */
477 uint32_t off;
478 /** The IEMNATIVELABEL this is a fixup for. */
479 uint16_t idxLabel;
480 /** The fixup type (IEMNATIVEFIXUPTYPE). */
481 uint8_t enmType;
482 /** Addend or other data. */
483 int8_t offAddend;
484} IEMNATIVEFIXUP;
485/** Pointer to a native code generator fixup. */
486typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
487
488
489/**
490 * One bit of the state.
491 *
492 * Each register state takes up two bits. We keep the two bits in two separate
493 * 64-bit words to simplify applying them to the guest shadow register mask in
494 * the register allocator.
495 */
496typedef union IEMLIVENESSBIT
497{
498 uint64_t bm64;
499 RT_GCC_EXTENSION struct
500 { /* bit no */
501 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
502 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
503 uint64_t fCr0 : 1; /**< 0x11 / 17: */
504 uint64_t fFcw : 1; /**< 0x12 / 18: */
505 uint64_t fFsw : 1; /**< 0x13 / 19: */
506 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
507 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
508 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
509 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
510 uint64_t fCr4 : 1; /**< 0x2c / 44: */
511 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
512 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
513 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
514 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
515 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
516 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
517 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
518 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
519 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
520 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
521 };
522} IEMLIVENESSBIT;
523AssertCompileSize(IEMLIVENESSBIT, 8);
524
525#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
526#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
527#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
528#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
529#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
530#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
531#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
532
533
534/**
535 * A liveness state entry.
536 *
537 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
538 * Once we add a SSE register shadowing, we'll add another 64-bit element for
539 * that.
540 */
541typedef union IEMLIVENESSENTRY
542{
543#ifndef IEMLIVENESS_EXTENDED_LAYOUT
544 uint64_t bm64[16 / 8];
545 uint16_t bm32[16 / 4];
546 uint16_t bm16[16 / 2];
547 uint8_t bm8[ 16 / 1];
548 IEMLIVENESSBIT aBits[2];
549#else
550 uint64_t bm64[32 / 8];
551 uint16_t bm32[32 / 4];
552 uint16_t bm16[32 / 2];
553 uint8_t bm8[ 32 / 1];
554 IEMLIVENESSBIT aBits[4];
555#endif
556 RT_GCC_EXTENSION struct
557 {
558 /** Bit \#0 of the register states. */
559 IEMLIVENESSBIT Bit0;
560 /** Bit \#1 of the register states. */
561 IEMLIVENESSBIT Bit1;
562#ifdef IEMLIVENESS_EXTENDED_LAYOUT
563 /** Bit \#2 of the register states. */
564 IEMLIVENESSBIT Bit2;
565 /** Bit \#3 of the register states. */
566 IEMLIVENESSBIT Bit3;
567#endif
568 };
569} IEMLIVENESSENTRY;
570#ifndef IEMLIVENESS_EXTENDED_LAYOUT
571AssertCompileSize(IEMLIVENESSENTRY, 16);
572#else
573AssertCompileSize(IEMLIVENESSENTRY, 32);
574#endif
575/** Pointer to a liveness state entry. */
576typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
577/** Pointer to a const liveness state entry. */
578typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
579
580/** @name 64-bit value masks for IEMLIVENESSENTRY.
581 * @{ */ /* 0xzzzzyyyyxxxxwwww */
582#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
583
584#ifndef IEMLIVENESS_EXTENDED_LAYOUT
585# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
586# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
587
588# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
589# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
590#endif
591
592#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
593#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
594
595#ifndef IEMLIVENESS_EXTENDED_LAYOUT
596# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
597# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
598#endif
599/** @} */
600
601
602/** @name The liveness state for a register.
603 *
604 * The state values have been picked to with state accumulation in mind (what
605 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
606 * performance critical work done with the values.
607 *
608 * This is a compressed state that only requires 2 bits per register.
609 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
610 * 1. the incoming state from the following call,
611 * 2. the outgoing state for this call,
612 * 3. mask of the entries set in the 2nd.
613 *
614 * The mask entry (3rd one above) will be used both when updating the outgoing
615 * state and when merging in incoming state for registers not touched by the
616 * current call.
617 *
618 * @{ */
619#ifndef IEMLIVENESS_EXTENDED_LAYOUT
620/** The register will be clobbered and the current value thrown away.
621 *
622 * When this is applied to the state (2) we'll simply be AND'ing it with the
623 * (old) mask (3) and adding the register to the mask. This way we'll
624 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
625 * IEMLIVENESS_STATE_INPUT states. */
626# define IEMLIVENESS_STATE_CLOBBERED 0
627/** The register is unused in the remainder of the TB.
628 *
629 * This is an initial state and can not be set by any of the
630 * iemNativeLivenessFunc_xxxx callbacks. */
631# define IEMLIVENESS_STATE_UNUSED 1
632/** The register value is required in a potential call or exception.
633 *
634 * This means that the register value must be calculated and is best written to
635 * the state, but that any shadowing registers can be flushed thereafter as it's
636 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
637 *
638 * It is typically applied across the board, but we preserve incoming
639 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
640 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
641 * 1. r0 = old & ~mask;
642 * 2. r0 = t1 & (t1 >> 1)'
643 * 3. state |= r0 | 0b10;
644 * 4. mask = ~0;
645 */
646# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
647/** The register value is used as input.
648 *
649 * This means that the register value must be calculated and it is best to keep
650 * it in a register. It does not need to be writtent out as such. This is the
651 * highest priority state.
652 *
653 * Whether the call modifies the register or not isn't relevant to earlier
654 * calls, so that's not recorded.
655 *
656 * When applying this state we just or in the value in the outgoing state and
657 * mask. */
658# define IEMLIVENESS_STATE_INPUT 3
659/** Mask of the state bits. */
660# define IEMLIVENESS_STATE_MASK 3
661/** The number of bits per state. */
662# define IEMLIVENESS_STATE_BIT_COUNT 2
663/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
664# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
665/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
666# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
667/** Check if a register clobbering is expected given the (previous) liveness state.
668 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
669 * include INPUT if the register is used in more than one place. */
670# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
671
672/** Check if all status flags are going to be clobbered and doesn't need
673 * calculating in the current step.
674 * @param a_pCurEntry The current liveness entry. */
675# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
676 ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
677
678#else /* IEMLIVENESS_EXTENDED_LAYOUT */
679/** The register is not used any more. */
680# define IEMLIVENESS_STATE_UNUSED 0
681/** Flag: The register is required in a potential exception or call. */
682# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
683# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
684/** Flag: The register is read. */
685# define IEMLIVENESS_STATE_READ 2
686# define IEMLIVENESS_BIT_READ 1
687/** Flag: The register is written. */
688# define IEMLIVENESS_STATE_WRITE 4
689# define IEMLIVENESS_BIT_WRITE 2
690/** Flag: Unconditional call (not needed, can be redefined for research). */
691# define IEMLIVENESS_STATE_CALL 8
692# define IEMLIVENESS_BIT_CALL 3
693# define IEMLIVENESS_BIT_OTHER 3 /**< More convenient name for this one. */
694# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
695 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
696# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
697# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
698
699# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
700 ( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
701 && !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
702 & IEMLIVENESSBIT_STATUS_EFL_MASK) )
703
704#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
705/** @} */
706
707/** @name Liveness helpers for builtin functions and similar.
708 *
709 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
710 * own set of manimulator macros for those.
711 *
712 * @{ */
713/** Initializing the state as all unused. */
714#ifndef IEMLIVENESS_EXTENDED_LAYOUT
715# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
716 do { \
717 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
718 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
719 } while (0)
720#else
721# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
722 do { \
723 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
724 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
725 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
726 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
727 } while (0)
728#endif
729
730/** Initializing the outgoing state with a potential xcpt or call state.
731 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
732#ifndef IEMLIVENESS_EXTENDED_LAYOUT
733# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
734 do { \
735 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
736 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
737 } while (0)
738#else
739# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
740 do { \
741 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
742 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
743 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
744 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
745 } while (0)
746#endif
747
748/** Adds a segment base register as input to the outgoing state. */
749#ifndef IEMLIVENESS_EXTENDED_LAYOUT
750# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
751 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
752 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
753 } while (0)
754#else
755# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
756 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
757 } while (0)
758#endif
759
760/** Adds a segment attribute register as input to the outgoing state. */
761#ifndef IEMLIVENESS_EXTENDED_LAYOUT
762# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
763 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
764 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
765 } while (0)
766#else
767# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
768 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
769 } while (0)
770#endif
771
772/** Adds a segment limit register as input to the outgoing state. */
773#ifndef IEMLIVENESS_EXTENDED_LAYOUT
774# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
775 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
776 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
777 } while (0)
778#else
779# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
780 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
781 } while (0)
782#endif
783
784/** Adds a segment limit register as input to the outgoing state. */
785#ifndef IEMLIVENESS_EXTENDED_LAYOUT
786# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
787 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
788 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
789 } while (0)
790#else
791# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
792 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
793 } while (0)
794#endif
795/** @} */
796
797/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
798 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
799 * calculated and up to date. This is to double check that we haven't skipped
800 * EFLAGS calculations when we actually need them. NOP in non-strict builds.
801 * @note has to be placed in
802 */
803#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
804# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
805 do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
806#else
807# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
808#endif
809
810
811/**
812 * Guest registers that can be shadowed in GPRs.
813 *
814 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
815 * must be placed last, as the liveness state tracks it as 7 subcomponents and
816 * we don't want to waste space here.
817 *
818 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
819 * friends as well as IEMAllN8veLiveness.cpp.
820 */
821typedef enum IEMNATIVEGSTREG : uint8_t
822{
823 kIemNativeGstReg_GprFirst = 0,
824 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
825 kIemNativeGstReg_Pc,
826 kIemNativeGstReg_Cr0,
827 kIemNativeGstReg_FpuFcw,
828 kIemNativeGstReg_FpuFsw,
829 kIemNativeGstReg_SegBaseFirst,
830 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
831 kIemNativeGstReg_SegAttribFirst,
832 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
833 kIemNativeGstReg_SegLimitFirst,
834 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
835 kIemNativeGstReg_SegSelFirst,
836 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
837 kIemNativeGstReg_Cr4,
838 kIemNativeGstReg_Xcr0,
839 kIemNativeGstReg_MxCsr,
840 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
841 kIemNativeGstReg_End
842} IEMNATIVEGSTREG;
843AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
844AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
845
846/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
847 * @{ */
848#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
849#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
850#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
851#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
852#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
853/** @} */
854
855#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
856
857/**
858 * Guest registers that can be shadowed in host SIMD registers.
859 *
860 * @todo r=aeichner Liveness tracking
861 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
862 */
863typedef enum IEMNATIVEGSTSIMDREG : uint8_t
864{
865 kIemNativeGstSimdReg_SimdRegFirst = 0,
866 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
867 kIemNativeGstSimdReg_End
868} IEMNATIVEGSTSIMDREG;
869
870/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
871 * @{ */
872#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
873/** @} */
874
875/**
876 * The Load/store size for a SIMD guest register.
877 */
878typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
879{
880 /** Invalid size. */
881 kIemNativeGstSimdRegLdStSz_Invalid = 0,
882 /** Loads the low 128-bit of a guest SIMD register. */
883 kIemNativeGstSimdRegLdStSz_Low128,
884 /** Loads the high 128-bit of a guest SIMD register. */
885 kIemNativeGstSimdRegLdStSz_High128,
886 /** Loads the whole 256-bits of a guest SIMD register. */
887 kIemNativeGstSimdRegLdStSz_256,
888 /** End value. */
889 kIemNativeGstSimdRegLdStSz_End
890} IEMNATIVEGSTSIMDREGLDSTSZ;
891
892#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
893
894/**
895 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
896 */
897typedef enum IEMNATIVEGSTREGUSE
898{
899 /** The usage is read-only, the register holding the guest register
900 * shadow copy will not be modified by the caller. */
901 kIemNativeGstRegUse_ReadOnly = 0,
902 /** The caller will update the guest register (think: PC += cbInstr).
903 * The guest shadow copy will follow the returned register. */
904 kIemNativeGstRegUse_ForUpdate,
905 /** The call will put an entirely new value in the guest register, so
906 * if new register is allocate it will be returned uninitialized. */
907 kIemNativeGstRegUse_ForFullWrite,
908 /** The caller will use the guest register value as input in a calculation
909 * and the host register will be modified.
910 * This means that the returned host register will not be marked as a shadow
911 * copy of the guest register. */
912 kIemNativeGstRegUse_Calculation
913} IEMNATIVEGSTREGUSE;
914
915/**
916 * Guest registers (classes) that can be referenced.
917 */
918typedef enum IEMNATIVEGSTREGREF : uint8_t
919{
920 kIemNativeGstRegRef_Invalid = 0,
921 kIemNativeGstRegRef_Gpr,
922 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
923 kIemNativeGstRegRef_EFlags,
924 kIemNativeGstRegRef_MxCsr,
925 kIemNativeGstRegRef_FpuReg,
926 kIemNativeGstRegRef_MReg,
927 kIemNativeGstRegRef_XReg,
928 kIemNativeGstRegRef_X87,
929 kIemNativeGstRegRef_XState,
930 //kIemNativeGstRegRef_YReg, - doesn't work.
931 kIemNativeGstRegRef_End
932} IEMNATIVEGSTREGREF;
933
934
935/** Variable kinds. */
936typedef enum IEMNATIVEVARKIND : uint8_t
937{
938 /** Customary invalid zero value. */
939 kIemNativeVarKind_Invalid = 0,
940 /** This is either in a register or on the stack. */
941 kIemNativeVarKind_Stack,
942 /** Immediate value - loaded into register when needed, or can live on the
943 * stack if referenced (in theory). */
944 kIemNativeVarKind_Immediate,
945 /** Variable reference - loaded into register when needed, never stack. */
946 kIemNativeVarKind_VarRef,
947 /** Guest register reference - loaded into register when needed, never stack. */
948 kIemNativeVarKind_GstRegRef,
949 /** End of valid values. */
950 kIemNativeVarKind_End
951} IEMNATIVEVARKIND;
952
953
954/** Variable or argument. */
955typedef struct IEMNATIVEVAR
956{
957 /** The kind of variable. */
958 IEMNATIVEVARKIND enmKind;
959 /** The variable size in bytes. */
960 uint8_t cbVar;
961 /** The first stack slot (uint64_t), except for immediate and references
962 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
963 * has a stack slot it has been initialized and has a value. Unused variables
964 * has neither a stack slot nor a host register assignment. */
965 uint8_t idxStackSlot;
966 /** The host register allocated for the variable, UINT8_MAX if not. */
967 uint8_t idxReg;
968 /** The argument number if argument, UINT8_MAX if regular variable. */
969 uint8_t uArgNo;
970 /** If referenced, the index (unpacked) of the variable referencing this one,
971 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
972 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
973 uint8_t idxReferrerVar;
974 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
975 * @todo not sure what this really is for... */
976 IEMNATIVEGSTREG enmGstReg;
977 /** Set if the registered is currently used exclusively, false if the
978 * variable is idle and the register can be grabbed. */
979 bool fRegAcquired;
980
981 union
982 {
983 /** kIemNativeVarKind_Immediate: The immediate value. */
984 uint64_t uValue;
985 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
986 uint8_t idxRefVar;
987 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
988 struct
989 {
990 /** The class of register. */
991 IEMNATIVEGSTREGREF enmClass;
992 /** Index within the class. */
993 uint8_t idx;
994 } GstRegRef;
995 } u;
996} IEMNATIVEVAR;
997/** Pointer to a variable or argument. */
998typedef IEMNATIVEVAR *PIEMNATIVEVAR;
999/** Pointer to a const variable or argument. */
1000typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1001
1002/** What is being kept in a host register. */
1003typedef enum IEMNATIVEWHAT : uint8_t
1004{
1005 /** The traditional invalid zero value. */
1006 kIemNativeWhat_Invalid = 0,
1007 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1008 kIemNativeWhat_Var,
1009 /** Temporary register, this is typically freed when a MC completes. */
1010 kIemNativeWhat_Tmp,
1011 /** Call argument w/o a variable mapping. This is free (via
1012 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1013 kIemNativeWhat_Arg,
1014 /** Return status code.
1015 * @todo not sure if we need this... */
1016 kIemNativeWhat_rc,
1017 /** The fixed pVCpu (PVMCPUCC) register.
1018 * @todo consider offsetting this on amd64 to use negative offsets to access
1019 * more members using 8-byte disp. */
1020 kIemNativeWhat_pVCpuFixed,
1021 /** The fixed pCtx (PCPUMCTX) register.
1022 * @todo consider offsetting this on amd64 to use negative offsets to access
1023 * more members using 8-byte disp. */
1024 kIemNativeWhat_pCtxFixed,
1025 /** Fixed temporary register. */
1026 kIemNativeWhat_FixedTmp,
1027#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1028 /** Shadow RIP for the delayed RIP updating debugging. */
1029 kIemNativeWhat_PcShadow,
1030#endif
1031 /** Register reserved by the CPU or OS architecture. */
1032 kIemNativeWhat_FixedReserved,
1033 /** End of valid values. */
1034 kIemNativeWhat_End
1035} IEMNATIVEWHAT;
1036
1037/**
1038 * Host general register entry.
1039 *
1040 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1041 *
1042 * @todo Track immediate values in host registers similarlly to how we track the
1043 * guest register shadow copies. For it to be real helpful, though,
1044 * we probably need to know which will be reused and put them into
1045 * non-volatile registers, otherwise it's going to be more or less
1046 * restricted to an instruction or two.
1047 */
1048typedef struct IEMNATIVEHSTREG
1049{
1050 /** Set of guest registers this one shadows.
1051 *
1052 * Using a bitmap here so we can designate the same host register as a copy
1053 * for more than one guest register. This is expected to be useful in
1054 * situations where one value is copied to several registers in a sequence.
1055 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1056 * sequence we'd want to let this register follow to be a copy of and there
1057 * will always be places where we'd be picking the wrong one.
1058 */
1059 uint64_t fGstRegShadows;
1060 /** What is being kept in this register. */
1061 IEMNATIVEWHAT enmWhat;
1062 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1063 uint8_t idxVar;
1064 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1065 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1066 * that scope. */
1067 uint8_t idxStackSlot;
1068 /** Alignment padding. */
1069 uint8_t abAlign[5];
1070} IEMNATIVEHSTREG;
1071
1072
1073#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1074/**
1075 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1076 * halves, on architectures where there is no 256-bit register available this entry will track
1077 * two adjacent 128-bit host registers.
1078 *
1079 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1080 */
1081typedef struct IEMNATIVEHSTSIMDREG
1082{
1083 /** Set of guest registers this one shadows.
1084 *
1085 * Using a bitmap here so we can designate the same host register as a copy
1086 * for more than one guest register. This is expected to be useful in
1087 * situations where one value is copied to several registers in a sequence.
1088 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1089 * sequence we'd want to let this register follow to be a copy of and there
1090 * will always be places where we'd be picking the wrong one.
1091 */
1092 uint64_t fGstRegShadows;
1093 /** What is being kept in this register. */
1094 IEMNATIVEWHAT enmWhat;
1095 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1096 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1097 /** Alignment padding. */
1098 uint8_t abAlign[6];
1099} IEMNATIVEHSTSIMDREG;
1100#endif
1101
1102
1103/**
1104 * Core state for the native recompiler, that is, things that needs careful
1105 * handling when dealing with branches.
1106 */
1107typedef struct IEMNATIVECORESTATE
1108{
1109#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1110 /** The current instruction offset in bytes from when the guest program counter
1111 * was updated last. Used for delaying the write to the guest context program counter
1112 * as long as possible. */
1113 uint32_t offPc;
1114 /** Number of instructions where we could skip the updating. */
1115 uint32_t cInstrPcUpdateSkipped;
1116#endif
1117 /** Allocation bitmap for aHstRegs. */
1118 uint32_t bmHstRegs;
1119
1120 /** Bitmap marking which host register contains guest register shadow copies.
1121 * This is used during register allocation to try preserve copies. */
1122 uint32_t bmHstRegsWithGstShadow;
1123 /** Bitmap marking valid entries in aidxGstRegShadows. */
1124 uint64_t bmGstRegShadows;
1125
1126#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1127 /** Allocation bitmap for aHstSimdRegs. */
1128 uint32_t bmHstSimdRegs;
1129
1130 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1131 * This is used during register allocation to try preserve copies. */
1132 uint32_t bmHstSimdRegsWithGstShadow;
1133 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1134 uint64_t bmGstSimdRegShadows;
1135 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1136 uint64_t bmGstSimdRegShadowDirtyLo128;
1137 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1138 uint64_t bmGstSimdRegShadowDirtyHi128;
1139#endif
1140
1141 union
1142 {
1143 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1144 uint8_t aidxArgVars[8];
1145 /** For more efficient resetting. */
1146 uint64_t u64ArgVars;
1147 };
1148
1149 /** Allocation bitmap for the stack. */
1150 uint32_t bmStack;
1151 /** Allocation bitmap for aVars. */
1152 uint32_t bmVars;
1153
1154 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1155 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1156 * (A shadow copy of a guest register can only be held in a one host register,
1157 * there are no duplicate copies or ambiguities like that). */
1158 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1159#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1160 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1161 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1162 * (A shadow copy of a guest register can only be held in a one host register,
1163 * there are no duplicate copies or ambiguities like that). */
1164 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1165#endif
1166
1167 /** Host register allocation tracking. */
1168 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1169#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1170 /** Host SIMD register allocation tracking. */
1171 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1172#endif
1173
1174 /** Variables and arguments. */
1175 IEMNATIVEVAR aVars[9];
1176} IEMNATIVECORESTATE;
1177/** Pointer to core state. */
1178typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1179/** Pointer to const core state. */
1180typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1181
1182/** @def IEMNATIVE_VAR_IDX_UNPACK
1183 * @returns Index into IEMNATIVECORESTATE::aVars.
1184 * @param a_idxVar Variable index w/ magic (in strict builds).
1185 */
1186/** @def IEMNATIVE_VAR_IDX_PACK
1187 * @returns Variable index w/ magic (in strict builds).
1188 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1189 */
1190#ifdef VBOX_STRICT
1191# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1192# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1193# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1194# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1195# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1196#else
1197# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1198# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1199#endif
1200
1201
1202#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1203/** Clear the dirty state of the given guest SIMD register. */
1204# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1205 do { \
1206 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1207 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1208 } while (0)
1209
1210/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1211# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1212 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1213/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1214# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1215 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1216/** Returns whether the given guest SIMD register is dirty. */
1217# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1218 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1219
1220/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1221# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1222 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1223/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1224# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1225 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1226
1227/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1228# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1229/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1230# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1231/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1232# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1233#endif
1234
1235
1236/**
1237 * Conditional stack entry.
1238 */
1239typedef struct IEMNATIVECOND
1240{
1241 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1242 bool fInElse;
1243 /** The label for the IEM_MC_ELSE. */
1244 uint32_t idxLabelElse;
1245 /** The label for the IEM_MC_ENDIF. */
1246 uint32_t idxLabelEndIf;
1247 /** The initial state snapshot as the if-block starts executing. */
1248 IEMNATIVECORESTATE InitialState;
1249 /** The state snapshot at the end of the if-block. */
1250 IEMNATIVECORESTATE IfFinalState;
1251} IEMNATIVECOND;
1252/** Pointer to a condition stack entry. */
1253typedef IEMNATIVECOND *PIEMNATIVECOND;
1254
1255
1256/**
1257 * Native recompiler state.
1258 */
1259typedef struct IEMRECOMPILERSTATE
1260{
1261 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1262 * IEMNATIVEINSTR units. */
1263 uint32_t cInstrBufAlloc;
1264#ifdef VBOX_STRICT
1265 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1266 uint32_t offInstrBufChecked;
1267#else
1268 uint32_t uPadding1; /* We don't keep track of the size here... */
1269#endif
1270 /** Fixed temporary code buffer for native recompilation. */
1271 PIEMNATIVEINSTR pInstrBuf;
1272
1273 /** Bitmaps with the label types used. */
1274 uint64_t bmLabelTypes;
1275 /** Actual number of labels in paLabels. */
1276 uint32_t cLabels;
1277 /** Max number of entries allowed in paLabels before reallocating it. */
1278 uint32_t cLabelsAlloc;
1279 /** Labels defined while recompiling (referenced by fixups). */
1280 PIEMNATIVELABEL paLabels;
1281 /** Array with indexes of unique labels (uData always 0). */
1282 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1283
1284 /** Actual number of fixups paFixups. */
1285 uint32_t cFixups;
1286 /** Max number of entries allowed in paFixups before reallocating it. */
1287 uint32_t cFixupsAlloc;
1288 /** Buffer used by the recompiler for recording fixups when generating code. */
1289 PIEMNATIVEFIXUP paFixups;
1290
1291#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1292 /** Number of debug info entries allocated for pDbgInfo. */
1293 uint32_t cDbgInfoAlloc;
1294 uint32_t uPadding;
1295 /** Debug info. */
1296 PIEMTBDBG pDbgInfo;
1297#endif
1298
1299#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1300 /** The current call index (liveness array and threaded calls in TB). */
1301 uint32_t idxCurCall;
1302 /** Number of liveness entries allocated. */
1303 uint32_t cLivenessEntriesAlloc;
1304 /** Liveness entries for all the calls in the TB begin recompiled.
1305 * The entry for idxCurCall contains the info for what the next call will
1306 * require wrt registers. (Which means the last entry is the initial liveness
1307 * state.) */
1308 PIEMLIVENESSENTRY paLivenessEntries;
1309#endif
1310
1311 /** The translation block being recompiled. */
1312 PCIEMTB pTbOrg;
1313 /** The VMCPU structure of the EMT. */
1314 PVMCPUCC pVCpu;
1315
1316 /** Condition sequence number (for generating unique labels). */
1317 uint16_t uCondSeqNo;
1318 /** Check IRQ seqeunce number (for generating unique labels). */
1319 uint16_t uCheckIrqSeqNo;
1320 /** TLB load sequence number (for generating unique labels). */
1321 uint16_t uTlbSeqNo;
1322 /** The current condition stack depth (aCondStack). */
1323 uint8_t cCondDepth;
1324
1325 /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1326 uint8_t cArgs;
1327 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1328 uint32_t fCImpl;
1329 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1330 uint32_t fMc;
1331 /** The expected IEMCPU::fExec value for the current call/instruction. */
1332 uint32_t fExec;
1333#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1334 /** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1335 * we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1336 *
1337 * This is an optimization because these control registers can only be changed from
1338 * by calling a C helper we can catch. Should reduce the number of instructions in a TB
1339 * consisting of multiple SIMD instructions.
1340 */
1341 uint32_t fSimdRaiseXcptChecksEmitted;
1342#endif
1343
1344 /** Core state requiring care with branches. */
1345 IEMNATIVECORESTATE Core;
1346
1347 /** The condition nesting stack. */
1348 IEMNATIVECOND aCondStack[2];
1349
1350#ifndef IEM_WITH_THROW_CATCH
1351 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1352 * for recompilation error handling. */
1353 jmp_buf JmpBuf;
1354#endif
1355} IEMRECOMPILERSTATE;
1356/** Pointer to a native recompiler state. */
1357typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1358
1359
1360/** @def IEMNATIVE_TRY_SETJMP
1361 * Wrapper around setjmp / try, hiding all the ugly differences.
1362 *
1363 * @note Use with extreme care as this is a fragile macro.
1364 * @param a_pReNative The native recompile state.
1365 * @param a_rcTarget The variable that should receive the status code in case
1366 * of a longjmp/throw.
1367 */
1368/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1369 * Start wrapper for catch / setjmp-else.
1370 *
1371 * This will set up a scope.
1372 *
1373 * @note Use with extreme care as this is a fragile macro.
1374 * @param a_pReNative The native recompile state.
1375 * @param a_rcTarget The variable that should receive the status code in case
1376 * of a longjmp/throw.
1377 */
1378/** @def IEMNATIVE_CATCH_LONGJMP_END
1379 * End wrapper for catch / setjmp-else.
1380 *
1381 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1382 * up the state.
1383 *
1384 * @note Use with extreme care as this is a fragile macro.
1385 * @param a_pReNative The native recompile state.
1386 */
1387/** @def IEMNATIVE_DO_LONGJMP
1388 *
1389 * Wrapper around longjmp / throw.
1390 *
1391 * @param a_pReNative The native recompile state.
1392 * @param a_rc The status code jump back with / throw.
1393 */
1394#ifdef IEM_WITH_THROW_CATCH
1395# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1396 a_rcTarget = VINF_SUCCESS; \
1397 try
1398# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1399 catch (int rcThrown) \
1400 { \
1401 a_rcTarget = rcThrown
1402# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1403 } \
1404 ((void)0)
1405# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1406#else /* !IEM_WITH_THROW_CATCH */
1407# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1408 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1409# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1410 else \
1411 { \
1412 ((void)0)
1413# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1414 }
1415# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1416#endif /* !IEM_WITH_THROW_CATCH */
1417
1418
1419/**
1420 * Native recompiler worker for a threaded function.
1421 *
1422 * @returns New code buffer offset; throws VBox status code in case of a failure.
1423 * @param pReNative The native recompiler state.
1424 * @param off The current code buffer offset.
1425 * @param pCallEntry The threaded call entry.
1426 *
1427 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1428 */
1429typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1430/** Pointer to a native recompiler worker for a threaded function. */
1431typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1432
1433/** Defines a native recompiler worker for a threaded function.
1434 * @see FNIEMNATIVERECOMPFUNC */
1435#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1436 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1437
1438/** Prototypes a native recompiler function for a threaded function.
1439 * @see FNIEMNATIVERECOMPFUNC */
1440#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1441
1442
1443/**
1444 * Native recompiler liveness analysis worker for a threaded function.
1445 *
1446 * @param pCallEntry The threaded call entry.
1447 * @param pIncoming The incoming liveness state entry.
1448 * @param pOutgoing The outgoing liveness state entry.
1449 */
1450typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1451 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1452/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1453typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1454
1455/** Defines a native recompiler liveness analysis worker for a threaded function.
1456 * @see FNIEMNATIVELIVENESSFUNC */
1457#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1458 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1459
1460/** Prototypes a native recompiler liveness analysis function for a threaded function.
1461 * @see FNIEMNATIVELIVENESSFUNC */
1462#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1463
1464
1465/** Define a native recompiler helper function, safe to call from the TB code. */
1466#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1467 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1468/** Prototype a native recompiler helper function, safe to call from the TB code. */
1469#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1470 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1471/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1472#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1473 a_RetType (VBOXCALL *a_Name) a_ArgList
1474
1475
1476#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1477DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1478DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1479 uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1480# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1481DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1482 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1483 uint8_t idxHstSimdReg = UINT8_MAX,
1484 uint8_t idxHstSimdRegPrev = UINT8_MAX);
1485# endif
1486DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1487 uint32_t offPc, uint32_t cInstrSkipped);
1488#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1489
1490DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1491 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1492DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1493DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1494 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1495DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1496
1497DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1498DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1499 bool fPreferVolatile = true);
1500DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1501 bool fPreferVolatile = true);
1502DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1503 IEMNATIVEGSTREG enmGstReg,
1504 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1505 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1506DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1507 IEMNATIVEGSTREG enmGstReg);
1508
1509DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1510DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1511#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) || (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1512DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1513 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1514#endif
1515DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1516DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1517DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1518DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1519DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1520DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1521 uint32_t fKeepVars = 0);
1522DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1523DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1524DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1525 uint32_t fHstRegsActiveShadows);
1526#ifdef VBOX_STRICT
1527DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1528#endif
1529DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1530 uint64_t fGstShwExcept, bool fFlushShadows);
1531#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1532DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1533#endif
1534
1535
1536#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1537DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1538DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1539 bool fPreferVolatile = true);
1540DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1541 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1542 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1543 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1544 bool fNoVolatileRegs = false);
1545DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1546DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1547DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1548 IEMNATIVEGSTSIMDREG enmGstSimdReg);
1549DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1550 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1551 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1552#endif
1553
1554DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1555DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1556DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1557DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1558DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1559DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1560DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1561DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1562 IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1563DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1564DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1565 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1566DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1567 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1568DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1569 uint32_t fHstRegsNotToSave);
1570DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1571 uint32_t fHstRegsNotToSave);
1572DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1573DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1574
1575DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1576 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1577#ifdef VBOX_STRICT
1578DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1579DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1580 IEMNATIVEGSTREG enmGstReg);
1581# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1582DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1583 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1584 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1585# endif
1586DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1587#endif
1588#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1589DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1590#endif
1591DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1592DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1593DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1594 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1595 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1596DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1597 PCIEMTHRDEDCALLENTRY pCallEntry);
1598DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1599 uint8_t idxAddrReg, uint8_t idxInstr);
1600DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1601 uint8_t idxAddrReg, uint8_t idxInstr);
1602DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1603 IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1604
1605
1606IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1607IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1608IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1609IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1610IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1611IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1612IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1613IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1614IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1615IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1616
1617IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1618IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1619IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1620IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1621IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1622IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1623IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1624IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1625IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1626IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1627IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1628IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1629IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1630IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1631IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1632IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1633IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1634IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1635IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1636IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1637IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1638
1639IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1640IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1641IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1642IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1643IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1644IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1645IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1646IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1647IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1648IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1649IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1650IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1651IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1652IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1653IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1654IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1655IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1656IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1657IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1658IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1659IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1660
1661IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1662IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1663IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1664IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1665IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1666IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1667IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1668IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1669IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1670IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1671IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1672IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1673IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1674IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1675IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1676IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1677IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1678IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1679IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1680IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1681IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1682IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1683
1684IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1685IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1686IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1687IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1688IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1689IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1690IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1691IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1692IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1693IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1694IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1695IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1696IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1697IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1698IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1699IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1700IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1701IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1702IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1703IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1704IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1705IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1706
1707IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1708IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1709IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1710IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1711
1712
1713/**
1714 * Info about shadowed guest register values.
1715 * @see IEMNATIVEGSTREG
1716 */
1717typedef struct IEMANTIVEGSTREGINFO
1718{
1719 /** Offset in VMCPU. */
1720 uint32_t off;
1721 /** The field size. */
1722 uint8_t cb;
1723 /** Name (for logging). */
1724 const char *pszName;
1725} IEMANTIVEGSTREGINFO;
1726extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1727extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1728extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1729extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1730extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1731
1732
1733
1734/**
1735 * Ensures that there is sufficient space in the instruction output buffer.
1736 *
1737 * This will reallocate the buffer if needed and allowed.
1738 *
1739 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1740 * allocation size.
1741 *
1742 * @returns Pointer to the instruction output buffer on success; throws VBox
1743 * status code on failure, so no need to check it.
1744 * @param pReNative The native recompile state.
1745 * @param off Current instruction offset. Works safely for UINT32_MAX
1746 * as well.
1747 * @param cInstrReq Number of instruction about to be added. It's okay to
1748 * overestimate this a bit.
1749 */
1750DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1751iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1752{
1753 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1754 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1755 {
1756#ifdef VBOX_STRICT
1757 pReNative->offInstrBufChecked = offChecked;
1758#endif
1759 return pReNative->pInstrBuf;
1760 }
1761 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1762}
1763
1764/**
1765 * Checks that we didn't exceed the space requested in the last
1766 * iemNativeInstrBufEnsure() call.
1767 */
1768#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1769 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1770 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1771
1772/**
1773 * Checks that a variable index is valid.
1774 */
1775#ifdef IEMNATIVE_VAR_IDX_MAGIC
1776# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1777 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1778 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1779 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1780 ("%s=%#x\n", #a_idxVar, a_idxVar))
1781#else
1782# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1783 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1784 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1785#endif
1786
1787/**
1788 * Checks that a variable index is valid and that the variable is assigned the
1789 * correct argument number.
1790 * This also adds a RT_NOREF of a_idxVar.
1791 */
1792#ifdef IEMNATIVE_VAR_IDX_MAGIC
1793# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1794 RT_NOREF_PV(a_idxVar); \
1795 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1796 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1797 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1798 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1799 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1800 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1801 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1802 a_uArgNo)); \
1803 } while (0)
1804#else
1805# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1806 RT_NOREF_PV(a_idxVar); \
1807 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1808 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1809 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1810 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1811 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1812 } while (0)
1813#endif
1814
1815
1816/**
1817 * Checks that a variable has the expected size.
1818 */
1819#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1820 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1821 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1822 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1823
1824
1825/**
1826 * Calculates the stack address of a variable as a [r]BP displacement value.
1827 */
1828DECL_FORCE_INLINE(int32_t)
1829iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1830{
1831 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1832 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1833}
1834
1835
1836/**
1837 * Releases the variable's register.
1838 *
1839 * The register must have been previously acquired calling
1840 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1841 * iemNativeVarRegisterSetAndAcquire().
1842 */
1843DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1844{
1845 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1846 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1847 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1848}
1849
1850
1851/**
1852 * Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1853 *
1854 * @returns The flush mask.
1855 * @param fCImpl The IEM_CIMPL_F_XXX flags.
1856 * @param fGstShwFlush The starting flush mask.
1857 */
1858DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1859{
1860 if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1861 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1862 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1863 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1864 if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1865 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1866 | RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1867 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1868 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1869 else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1870 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1871 if (fCImpl & (IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_STATUS_FLAGS | IEM_CIMPL_F_INHIBIT_SHADOW))
1872 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
1873 return fGstShwFlush;
1874}
1875
1876
1877/** Number of hidden arguments for CIMPL calls.
1878 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1879#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1880# define IEM_CIMPL_HIDDEN_ARGS 3
1881#else
1882# define IEM_CIMPL_HIDDEN_ARGS 2
1883#endif
1884
1885
1886#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1887/** Number of hidden arguments for SSE_AIMPL calls. */
1888# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1889#endif
1890
1891
1892#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1893
1894# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1895/**
1896 * Helper for iemNativeLivenessGetStateByGstReg.
1897 *
1898 * @returns IEMLIVENESS_STATE_XXX
1899 * @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1900 * ORed together.
1901 */
1902DECL_FORCE_INLINE(uint32_t)
1903iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1904{
1905 /* INPUT trumps anything else. */
1906 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1907 return IEMLIVENESS_STATE_INPUT;
1908
1909 /* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1910 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1911 {
1912 /* If not all sub-fields are clobbered they must be considered INPUT. */
1913 if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) | RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1914 return IEMLIVENESS_STATE_INPUT;
1915 return IEMLIVENESS_STATE_CLOBBERED;
1916 }
1917
1918 /* XCPT_OR_CALL trumps UNUSED. */
1919 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1920 return IEMLIVENESS_STATE_XCPT_OR_CALL;
1921
1922 return IEMLIVENESS_STATE_UNUSED;
1923}
1924# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1925
1926
1927DECL_FORCE_INLINE(uint32_t)
1928iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1929{
1930# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1931 return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1932 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1933# else
1934 return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1935 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1936 | (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1937 | (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1938# endif
1939}
1940
1941
1942DECL_FORCE_INLINE(uint32_t)
1943iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
1944{
1945 uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
1946 if (enmGstReg == kIemNativeGstReg_EFlags)
1947 {
1948 /* Merge the eflags states to one. */
1949# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1950 uRet = RT_BIT_32(uRet);
1951 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflCf | (pLivenessEntry->Bit1.fEflCf << 1));
1952 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflPf | (pLivenessEntry->Bit1.fEflPf << 1));
1953 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflAf | (pLivenessEntry->Bit1.fEflAf << 1));
1954 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflZf | (pLivenessEntry->Bit1.fEflZf << 1));
1955 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflSf | (pLivenessEntry->Bit1.fEflSf << 1));
1956 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflOf | (pLivenessEntry->Bit1.fEflOf << 1));
1957 uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
1958# else
1959 AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
1960 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
1961 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
1962 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
1963 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
1964 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
1965 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
1966# endif
1967 }
1968 return uRet;
1969}
1970
1971
1972# ifdef VBOX_STRICT
1973/** For assertions only, user checks that idxCurCall isn't zerow. */
1974DECL_FORCE_INLINE(uint32_t)
1975iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
1976{
1977 return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
1978}
1979# endif /* VBOX_STRICT */
1980
1981#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
1982
1983
1984/**
1985 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
1986 */
1987DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
1988{
1989 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
1990 return IEM_CIMPL_HIDDEN_ARGS;
1991 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
1992 return 1;
1993 return 0;
1994}
1995
1996
1997DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
1998 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
1999{
2000 pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
2001
2002 pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2003 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2004 pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2005 return (uint8_t)idxReg;
2006}
2007
2008
2009
2010/*********************************************************************************************************************************
2011* Register Allocator (GPR) *
2012*********************************************************************************************************************************/
2013
2014/**
2015 * Marks host register @a idxHstReg as containing a shadow copy of guest
2016 * register @a enmGstReg.
2017 *
2018 * ASSUMES that caller has made sure @a enmGstReg is not associated with any
2019 * host register before calling.
2020 */
2021DECL_FORCE_INLINE(void)
2022iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2023{
2024 Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2025 Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2026 Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2027
2028 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2029 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2030 pReNative->Core.bmGstRegShadows |= RT_BIT_64(enmGstReg);
2031 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxHstReg);
2032#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2033 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2034 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2035#else
2036 RT_NOREF(off);
2037#endif
2038}
2039
2040
2041/**
2042 * Clear any guest register shadow claims from @a idxHstReg.
2043 *
2044 * The register does not need to be shadowing any guest registers.
2045 */
2046DECL_FORCE_INLINE(void)
2047iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2048{
2049 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2050 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2051 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2052 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2053 == RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2054
2055#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2056 uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2057 if (fGstRegs)
2058 {
2059 Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2060 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2061 while (fGstRegs)
2062 {
2063 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2064 fGstRegs &= ~RT_BIT_64(iGstReg);
2065 iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2066 }
2067 }
2068#else
2069 RT_NOREF(off);
2070#endif
2071
2072 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2073 pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2074 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2075}
2076
2077
2078/**
2079 * Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2080 * and global overview flags.
2081 */
2082DECL_FORCE_INLINE(void)
2083iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2084{
2085 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2086 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2087 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2088 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2089 Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2090 Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2091 Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2092
2093#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2094 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2095 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2096#else
2097 RT_NOREF(off);
2098#endif
2099
2100 uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2101 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2102 if (!fGstRegShadowsNew)
2103 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2104 pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2105}
2106
2107
2108#if 0 /* unused */
2109/**
2110 * Clear any guest register shadow claim for @a enmGstReg.
2111 */
2112DECL_FORCE_INLINE(void)
2113iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2114{
2115 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2116 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2117 {
2118 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2119 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2120 }
2121}
2122#endif
2123
2124
2125/**
2126 * Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2127 * as the new shadow of it.
2128 *
2129 * Unlike the other guest reg shadow helpers, this does the logging for you.
2130 * However, it is the liveness state is not asserted here, the caller must do
2131 * that.
2132 */
2133DECL_FORCE_INLINE(void)
2134iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2135 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2136{
2137 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2138 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2139 {
2140 uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2141 Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2142 if (idxHstRegOld == idxHstRegNew)
2143 return;
2144 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2145 g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2146 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2147 }
2148 else
2149 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2150 g_aGstShadowInfo[enmGstReg].pszName));
2151 iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2152}
2153
2154
2155/**
2156 * Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2157 * to @a idxRegTo.
2158 */
2159DECL_FORCE_INLINE(void)
2160iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2161 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2162{
2163 Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2164 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2165 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2166 == pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2167 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2168 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2169 == pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2170 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2171 == RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2172
2173 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2174 pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2175 if (!fGstRegShadowsFrom)
2176 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2177 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxRegTo);
2178 pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows |= RT_BIT_64(enmGstReg);
2179 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2180#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2181 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2182 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2183#else
2184 RT_NOREF(off);
2185#endif
2186}
2187
2188
2189/**
2190 * Flushes any delayed guest register writes.
2191 *
2192 * This must be called prior to calling CImpl functions and any helpers that use
2193 * the guest state (like raising exceptions) and such.
2194 *
2195 * This optimization has not yet been implemented. The first target would be
2196 * RIP updates, since these are the most common ones.
2197 */
2198DECL_INLINE_THROW(uint32_t)
2199iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2200{
2201#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2202 if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2203 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2204#else
2205 RT_NOREF(pReNative, fGstShwExcept);
2206#endif
2207
2208#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2209 /** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2210 /** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2211 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2212#else
2213 RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2214 return off;
2215#endif
2216}
2217
2218
2219
2220/*********************************************************************************************************************************
2221* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2222*********************************************************************************************************************************/
2223
2224#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2225
2226DECL_FORCE_INLINE(uint8_t)
2227iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2228 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2229{
2230 pReNative->Core.bmHstSimdRegs |= RT_BIT_32(idxSimdReg);
2231
2232 pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2233 pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2234 RT_NOREF(idxVar);
2235 return idxSimdReg;
2236}
2237
2238
2239/**
2240 * Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2241 * SIMD register @a enmGstSimdReg.
2242 *
2243 * ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2244 * host register before calling.
2245 */
2246DECL_FORCE_INLINE(void)
2247iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2248 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2249{
2250 Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2251 Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2252 Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2253
2254 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2255 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2256 pReNative->Core.bmGstSimdRegShadows |= RT_BIT_64(enmGstSimdReg);
2257 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxHstSimdReg);
2258#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2259 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2260 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2261#else
2262 RT_NOREF(off);
2263#endif
2264}
2265
2266
2267/**
2268 * Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2269 * to @a idxSimdRegTo.
2270 */
2271DECL_FORCE_INLINE(void)
2272iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2273 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2274{
2275 Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2276 Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2277 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2278 == pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2279 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2280 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2281 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2282 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2283 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2284 Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2285 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2286
2287 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2288 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2289 if (!fGstRegShadowsFrom)
2290 {
2291 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2292 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2293 }
2294 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxSimdRegTo);
2295 pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2296 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2297#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2298 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2299 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2300#else
2301 RT_NOREF(off);
2302#endif
2303}
2304
2305
2306/**
2307 * Clear any guest register shadow claims from @a idxHstSimdReg.
2308 *
2309 * The register does not need to be shadowing any guest registers.
2310 */
2311DECL_FORCE_INLINE(void)
2312iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2313{
2314 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2315 == pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2316 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2317 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2318 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2319 Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2320 && !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2321
2322#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2323 uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2324 if (fGstRegs)
2325 {
2326 Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2327 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2328 while (fGstRegs)
2329 {
2330 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2331 fGstRegs &= ~RT_BIT_64(iGstReg);
2332 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2333 }
2334 }
2335#else
2336 RT_NOREF(off);
2337#endif
2338
2339 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2340 pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2341 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2342 pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2343}
2344
2345#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2346
2347
2348#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2349/**
2350 * Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2351 */
2352DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2353{
2354 if (pReNative->Core.offPc)
2355 return iemNativeEmitPcWritebackSlow(pReNative, off);
2356 return off;
2357}
2358#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2359
2360
2361/** @} */
2362
2363#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2364
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette