VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106090

Last change on this file since 106090 was 106090, checked in by vboxsync, 7 months ago

VMM/IEM: More liveness work for delayed eflags updating. bugref:10720 bugref:10372
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 136.1 KB
Line 
1/* $Id: IEMN8veRecompiler.h 106090 2024-09-19 09:13:54Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43 * Enables generating internal debug info for better TB disassembly dumping. */
44#if defined(DEBUG) || defined(DOXYGEN_RUNNING) || 0
45# define IEMNATIVE_WITH_TB_DEBUG_INFO
46#endif
47
48/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49 * Enables liveness analysis. */
50#if 1 || defined(DOXYGEN_RUNNING)
51# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
53#endif
54
55/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56 * Enables skipping EFLAGS calculations/updating based on liveness info. */
57#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(DOXYGEN_RUNNING)
58# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59#endif
60
61/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62 * Enables strict consistency checks around EFLAGS skipping.
63 * @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65# ifdef VBOX_STRICT
66# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67# endif
68#elif defined(DOXYGEN_RUNNING)
69# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70#endif
71
72#ifdef VBOX_WITH_STATISTICS
73/** Always count instructions for now. */
74# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75#endif
76
77/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78 * Enables having only a single prologue for native TBs. */
79#if 1 || defined(DOXYGEN_RUNNING)
80# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81#endif
82
83/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84 * Enable this to use common epilogue and tail code for all TBs in a chunk. */
85#if 1 || defined(DOXYGEN_RUNNING)
86# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87#endif
88
89
90/** @name Stack Frame Layout
91 *
92 * @{ */
93/** The size of the area for stack variables and spills and stuff.
94 * @note This limit is duplicated in the python script(s). We add 0x40 for
95 * alignment padding. */
96#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101#ifdef RT_ARCH_AMD64
102/** An stack alignment adjustment (between non-volatile register pushes and
103 * the stack variable area, so the latter better aligned). */
104# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106/** Number of stack arguments slots for calls made from the frame. */
107# ifdef RT_OS_WINDOWS
108# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109# else
110# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111# endif
112/** Number of any shadow arguments (spill area) for calls we make. */
113# ifdef RT_OS_WINDOWS
114# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115# else
116# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117# endif
118
119/** Frame pointer (RBP) relative offset of the last push. */
120# ifdef RT_OS_WINDOWS
121# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122# else
123# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124# endif
125/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126 * address for it). */
127# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132# ifdef RT_OS_WINDOWS
133/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137# endif
138
139# ifdef RT_OS_WINDOWS
140/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148# endif
149
150#elif RT_ARCH_ARM64
151/** No alignment padding needed for arm64. */
152# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153/** No stack argument slots, got 8 registers for arguments will suffice. */
154# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155/** There are no argument spill area. */
156# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158/** Number of saved registers at the top of our stack frame.
159 * This includes the return address and old frame pointer, so x19 thru x30. */
160# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164/** Frame pointer (BP) relative offset of the last push. */
165# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168 * address for it). */
169# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171#else
172# error "port me"
173#endif
174/** @} */
175
176
177/** @name Fixed Register Allocation(s)
178 * @{ */
179/** @def IEMNATIVE_REG_FIXED_PVMCPU
180 * The number of the register holding the pVCpu pointer. */
181/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183 * @note This not available on AMD64, only ARM64. */
184/** @def IEMNATIVE_REG_FIXED_TMP0
185 * Dedicated temporary register.
186 * @note This has extremely short lifetime, must be used with great care to make
187 * sure any calling code or code being called is making use of it.
188 * It will definitely not survive a call or anything of that nature.
189 * @todo replace this by a register allocator and content tracker. */
190/** @def IEMNATIVE_REG_FIXED_MASK
191 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
192 * architecture. */
193#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
196 * architecture. */
197/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
198 * Dedicated temporary SIMD register. */
199#endif
200#if defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
201# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
202# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
203# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
204# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
205# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
206# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
207# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
208# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
209# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
210 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
211# else
212# define IEMNATIVE_REG_FIXED_MASK_ADD 0
213# endif
214# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
215 | RT_BIT_32(ARMV8_A64_REG_LR) \
216 | RT_BIT_32(ARMV8_A64_REG_BP) \
217 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
218 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
219 | RT_BIT_32(ARMV8_A64_REG_X18) \
220 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
221 | IEMNATIVE_REG_FIXED_MASK_ADD)
222
223# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
224# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
225# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
226# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
227# else
228/** @note
229 * ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
230 * support emulating 256-bit registers we pair two real registers statically to
231 * one virtual for now, leaving us with only 16 256-bit registers. We always
232 * pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
233 * the register allocator assumes that it will be always free when the lower is
234 * picked.
235 *
236 * Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
237 * touch them in order to avoid having to save and restore them in the
238 * prologue/epilogue.
239 */
240# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
241 | RT_BIT_32(ARMV8_A64_REG_Q31) \
242 | RT_BIT_32(ARMV8_A64_REG_Q30) \
243 | RT_BIT_32(ARMV8_A64_REG_Q29) \
244 | RT_BIT_32(ARMV8_A64_REG_Q27) \
245 | RT_BIT_32(ARMV8_A64_REG_Q25) \
246 | RT_BIT_32(ARMV8_A64_REG_Q23) \
247 | RT_BIT_32(ARMV8_A64_REG_Q21) \
248 | RT_BIT_32(ARMV8_A64_REG_Q19) \
249 | RT_BIT_32(ARMV8_A64_REG_Q17) \
250 | RT_BIT_32(ARMV8_A64_REG_Q15) \
251 | RT_BIT_32(ARMV8_A64_REG_Q13) \
252 | RT_BIT_32(ARMV8_A64_REG_Q11) \
253 | RT_BIT_32(ARMV8_A64_REG_Q9) \
254 | RT_BIT_32(ARMV8_A64_REG_Q7) \
255 | RT_BIT_32(ARMV8_A64_REG_Q5) \
256 | RT_BIT_32(ARMV8_A64_REG_Q3) \
257 | RT_BIT_32(ARMV8_A64_REG_Q1))
258# endif
259# endif
260
261#elif defined(RT_ARCH_AMD64)
262# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
263# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
264# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
265# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
266 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
267 | RT_BIT_32(X86_GREG_xSP) \
268 | RT_BIT_32(X86_GREG_xBP) )
269
270# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
271# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
272# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
273# ifndef _MSC_VER
274# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275# endif
276# endif
277# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
278# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
279# else
280/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
281# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
282 | RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
283# endif
284# endif
285
286#else
287# error "port me"
288#endif
289/** @} */
290
291/** @name Call related registers.
292 * @{ */
293/** @def IEMNATIVE_CALL_RET_GREG
294 * The return value register. */
295/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
296 * Number of arguments in registers. */
297/** @def IEMNATIVE_CALL_ARG0_GREG
298 * The general purpose register carrying argument \#0. */
299/** @def IEMNATIVE_CALL_ARG1_GREG
300 * The general purpose register carrying argument \#1. */
301/** @def IEMNATIVE_CALL_ARG2_GREG
302 * The general purpose register carrying argument \#2. */
303/** @def IEMNATIVE_CALL_ARG3_GREG
304 * The general purpose register carrying argument \#3. */
305/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
306 * Mask of registers the callee will not save and may trash. */
307#ifdef RT_ARCH_AMD64
308# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
309
310# ifdef RT_OS_WINDOWS
311# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
312# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
313# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
314# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
315# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
316# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
317 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
318 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
319 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
320# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
321 | RT_BIT_32(X86_GREG_xCX) \
322 | RT_BIT_32(X86_GREG_xDX) \
323 | RT_BIT_32(X86_GREG_x8) \
324 | RT_BIT_32(X86_GREG_x9) \
325 | RT_BIT_32(X86_GREG_x10) \
326 | RT_BIT_32(X86_GREG_x11) )
327# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
328/* xmm0 - xmm5 are marked as volatile. */
329# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
330# endif
331
332# else /* !RT_OS_WINDOWS */
333# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
334# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
335# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
336# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
337# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
338# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
339# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
340# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
341 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
342 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
343 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
344 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
345 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
346# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
347 | RT_BIT_32(X86_GREG_xCX) \
348 | RT_BIT_32(X86_GREG_xDX) \
349 | RT_BIT_32(X86_GREG_xDI) \
350 | RT_BIT_32(X86_GREG_xSI) \
351 | RT_BIT_32(X86_GREG_x8) \
352 | RT_BIT_32(X86_GREG_x9) \
353 | RT_BIT_32(X86_GREG_x10) \
354 | RT_BIT_32(X86_GREG_x11) )
355# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
356/* xmm0 - xmm15 are marked as volatile. */
357# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
358# endif
359# endif /* !RT_OS_WINDOWS */
360
361#elif defined(RT_ARCH_ARM64)
362# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
363# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
364# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
365# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
366# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
367# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
368# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
369# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
370# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
371# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
372# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373 | RT_BIT_32(ARMV8_A64_REG_X1) \
374 | RT_BIT_32(ARMV8_A64_REG_X2) \
375 | RT_BIT_32(ARMV8_A64_REG_X3) \
376 | RT_BIT_32(ARMV8_A64_REG_X4) \
377 | RT_BIT_32(ARMV8_A64_REG_X5) \
378 | RT_BIT_32(ARMV8_A64_REG_X6) \
379 | RT_BIT_32(ARMV8_A64_REG_X7) )
380# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
381 | RT_BIT_32(ARMV8_A64_REG_X1) \
382 | RT_BIT_32(ARMV8_A64_REG_X2) \
383 | RT_BIT_32(ARMV8_A64_REG_X3) \
384 | RT_BIT_32(ARMV8_A64_REG_X4) \
385 | RT_BIT_32(ARMV8_A64_REG_X5) \
386 | RT_BIT_32(ARMV8_A64_REG_X6) \
387 | RT_BIT_32(ARMV8_A64_REG_X7) \
388 | RT_BIT_32(ARMV8_A64_REG_X8) \
389 | RT_BIT_32(ARMV8_A64_REG_X9) \
390 | RT_BIT_32(ARMV8_A64_REG_X10) \
391 | RT_BIT_32(ARMV8_A64_REG_X11) \
392 | RT_BIT_32(ARMV8_A64_REG_X12) \
393 | RT_BIT_32(ARMV8_A64_REG_X13) \
394 | RT_BIT_32(ARMV8_A64_REG_X14) \
395 | RT_BIT_32(ARMV8_A64_REG_X15) \
396 | RT_BIT_32(ARMV8_A64_REG_X16) \
397 | RT_BIT_32(ARMV8_A64_REG_X17) )
398# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
399/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
400 * so to simplify our life a bit we just mark everything as volatile. */
401# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
402# endif
403
404#endif
405
406/** This is the maximum argument count we'll ever be needing. */
407#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
408#ifdef RT_OS_WINDOWS
409# ifdef VBOXSTRICTRC_STRICT_ENABLED
410# undef IEMNATIVE_CALL_MAX_ARG_COUNT
411# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
412# endif
413#endif
414
415/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
416 * Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
417 * IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
418#ifdef IEMNATIVE_REG_FIXED_TMP0
419# ifdef IEMNATIVE_REG_FIXED_TMP1
420# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
421 & ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
422 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
423# else
424# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
425# endif
426#else
427# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
428#endif
429/** @} */
430
431
432/** @def IEMNATIVE_HST_GREG_COUNT
433 * Number of host general purpose registers we tracker. */
434/** @def IEMNATIVE_HST_GREG_MASK
435 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
436 * inverted register masks and such to get down to a correct set of regs. */
437#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
438/** @def IEMNATIVE_HST_SIMD_REG_COUNT
439 * Number of host SIMD registers we track. */
440/** @def IEMNATIVE_HST_SIMD_REG_MASK
441 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
442 * inverted register masks and such to get down to a correct set of regs. */
443#endif
444#ifdef RT_ARCH_AMD64
445# define IEMNATIVE_HST_GREG_COUNT 16
446# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
447
448# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
449# define IEMNATIVE_HST_SIMD_REG_COUNT 16
450# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
451# endif
452
453#elif defined(RT_ARCH_ARM64)
454# define IEMNATIVE_HST_GREG_COUNT 32
455# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
456
457# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
458# define IEMNATIVE_HST_SIMD_REG_COUNT 32
459# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
460# endif
461
462#else
463# error "Port me!"
464#endif
465
466
467#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
468
469
470/** Native code generator label types. */
471typedef enum
472{
473 kIemNativeLabelType_Invalid = 0,
474 /*
475 * Labels w/o data, only once instance per TB - aka exit reasons.
476 *
477 * Note! Jumps to these requires instructions that are capable of spanning
478 * the max TB length.
479 */
480 /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
481 kIemNativeLabelType_RaiseDe, /**< Raise (throw) X86_XCPT_DE (00h). */
482 kIemNativeLabelType_RaiseUd, /**< Raise (throw) X86_XCPT_UD (06h). */
483 kIemNativeLabelType_RaiseSseRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
484 kIemNativeLabelType_RaiseAvxRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
485 kIemNativeLabelType_RaiseSseAvxFpRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
486 kIemNativeLabelType_RaiseNm, /**< Raise (throw) X86_XCPT_NM (07h). */
487 kIemNativeLabelType_RaiseGp0, /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
488 kIemNativeLabelType_RaiseMf, /**< Raise (throw) X86_XCPT_MF (10h). */
489 kIemNativeLabelType_RaiseXf, /**< Raise (throw) X86_XCPT_XF (13h). */
490 kIemNativeLabelType_ObsoleteTb,
491 kIemNativeLabelType_NeedCsLimChecking,
492 kIemNativeLabelType_CheckBranchMiss,
493 kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
494 /* Manually defined labels. */
495 kIemNativeLabelType_ReturnBreak,
496 kIemNativeLabelType_ReturnBreakFF,
497 kIemNativeLabelType_ReturnBreakViaLookup,
498 kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
499 kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
500 kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
501 kIemNativeLabelType_ReturnWithFlags,
502 kIemNativeLabelType_NonZeroRetOrPassUp,
503#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
504 kIemNativeLabelType_ReturnSuccess, /**< Sets eax/w0 to zero and returns. */
505#else
506 kIemNativeLabelType_Return,
507#endif
508 /** The last fixup for branches that can span almost the whole TB length.
509 * @note Whether kIemNativeLabelType_Return needs to be one of these is
510 * a bit questionable, since nobody jumps to it except other tail code. */
511#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
512 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
513#else
514 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
515#endif
516 /** The last fixup for branches that exits the TB. */
517#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
518 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_ReturnSuccess,
519#else
520 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
521#endif
522
523 /** Loop-jump target. */
524 kIemNativeLabelType_LoopJumpTarget,
525
526 /*
527 * Labels with data, potentially multiple instances per TB:
528 *
529 * These are localized labels, so no fixed jump type restrictions here.
530 */
531 kIemNativeLabelType_FirstWithMultipleInstances,
532 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
533 kIemNativeLabelType_Else,
534 kIemNativeLabelType_Endif,
535 kIemNativeLabelType_CheckIrq,
536 kIemNativeLabelType_TlbLookup,
537 kIemNativeLabelType_TlbMiss,
538 kIemNativeLabelType_TlbDone,
539 kIemNativeLabelType_End
540} IEMNATIVELABELTYPE;
541
542#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
543 ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
544
545
546/** Native code generator label definition. */
547typedef struct IEMNATIVELABEL
548{
549 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
550 * the epilog. */
551 uint32_t off;
552 /** The type of label (IEMNATIVELABELTYPE). */
553 uint16_t enmType;
554 /** Additional label data, type specific. */
555 uint16_t uData;
556} IEMNATIVELABEL;
557/** Pointer to a label. */
558typedef IEMNATIVELABEL *PIEMNATIVELABEL;
559
560
561/** Native code generator fixup types. */
562typedef enum
563{
564 kIemNativeFixupType_Invalid = 0,
565#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
566 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
567 kIemNativeFixupType_Rel32,
568#elif defined(RT_ARCH_ARM64)
569 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
570 kIemNativeFixupType_RelImm26At0,
571 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
572 kIemNativeFixupType_RelImm19At5,
573 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
574 kIemNativeFixupType_RelImm14At5,
575#endif
576 kIemNativeFixupType_End
577} IEMNATIVEFIXUPTYPE;
578
579/** Native code generator fixup. */
580typedef struct IEMNATIVEFIXUP
581{
582 /** Code offset of the fixup location. */
583 uint32_t off;
584 /** The IEMNATIVELABEL this is a fixup for. */
585 uint16_t idxLabel;
586 /** The fixup type (IEMNATIVEFIXUPTYPE). */
587 uint8_t enmType;
588 /** Addend or other data. */
589 int8_t offAddend;
590} IEMNATIVEFIXUP;
591/** Pointer to a native code generator fixup. */
592typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
593
594#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
595
596/** Native code generator fixup to per chunk TB tail code. */
597typedef struct IEMNATIVEEXITFIXUP
598{
599 /** Code offset of the fixup location. */
600 uint32_t off;
601 /** The exit reason. */
602 IEMNATIVELABELTYPE enmExitReason;
603} IEMNATIVEEXITFIXUP;
604/** Pointer to a native code generator TB exit fixup. */
605typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
606
607/**
608 * Per executable memory chunk context with addresses for common code.
609 */
610typedef struct IEMNATIVEPERCHUNKCTX
611{
612 /** Pointers to the exit labels */
613 PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
614} IEMNATIVEPERCHUNKCTX;
615/** Pointer to per-chunk recompiler context. */
616typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
617/** Pointer to const per-chunk recompiler context. */
618typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
619
620#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
621
622
623/**
624 * One bit of the state.
625 *
626 * Each register state takes up two bits. We keep the two bits in two separate
627 * 64-bit words to simplify applying them to the guest shadow register mask in
628 * the register allocator.
629 */
630typedef union IEMLIVENESSBIT
631{
632 uint64_t bm64;
633 RT_GCC_EXTENSION struct
634 { /* bit no */
635 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
636 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
637 uint64_t fCr0 : 1; /**< 0x11 / 17: */
638 uint64_t fFcw : 1; /**< 0x12 / 18: */
639 uint64_t fFsw : 1; /**< 0x13 / 19: */
640 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
641 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
642 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
643 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
644 uint64_t fCr4 : 1; /**< 0x2c / 44: */
645 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
646 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
647 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
648 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
649 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
650 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
651 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
652 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
653 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
654 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
655 };
656} IEMLIVENESSBIT;
657AssertCompileSize(IEMLIVENESSBIT, 8);
658
659#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
660#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
661#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
662#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
663#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
664#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
665#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
666
667
668/**
669 * A liveness state entry.
670 *
671 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
672 * Once we add a SSE register shadowing, we'll add another 64-bit element for
673 * that.
674 */
675typedef union IEMLIVENESSENTRY
676{
677#ifndef IEMLIVENESS_EXTENDED_LAYOUT
678 uint64_t bm64[16 / 8];
679 uint16_t bm32[16 / 4];
680 uint16_t bm16[16 / 2];
681 uint8_t bm8[ 16 / 1];
682 IEMLIVENESSBIT aBits[2];
683#else
684 uint64_t bm64[32 / 8];
685 uint16_t bm32[32 / 4];
686 uint16_t bm16[32 / 2];
687 uint8_t bm8[ 32 / 1];
688 IEMLIVENESSBIT aBits[4];
689#endif
690 RT_GCC_EXTENSION struct
691 {
692 /** Bit \#0 of the register states. */
693 IEMLIVENESSBIT Bit0;
694 /** Bit \#1 of the register states. */
695 IEMLIVENESSBIT Bit1;
696#ifdef IEMLIVENESS_EXTENDED_LAYOUT
697 /** Bit \#2 of the register states. */
698 IEMLIVENESSBIT Bit2;
699 /** Bit \#3 of the register states. */
700 IEMLIVENESSBIT Bit3;
701#endif
702 };
703} IEMLIVENESSENTRY;
704#ifndef IEMLIVENESS_EXTENDED_LAYOUT
705AssertCompileSize(IEMLIVENESSENTRY, 16);
706#else
707AssertCompileSize(IEMLIVENESSENTRY, 32);
708#endif
709/** Pointer to a liveness state entry. */
710typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
711/** Pointer to a const liveness state entry. */
712typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
713
714/** @name 64-bit value masks for IEMLIVENESSENTRY.
715 * @{ */ /* 0xzzzzyyyyxxxxwwww */
716/** @todo Changing this to 0x003ffffffffffffe would reduce the liveness code
717 * size by 3.2% on arm in extended layout. That means moving kIemNativeGstReg_Pc
718 * to zero, which may have other consequences so needs to be tested in full first. */
719#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
720
721#ifndef IEMLIVENESS_EXTENDED_LAYOUT
722# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
723# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
724
725# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
726# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
727#endif
728
729#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
730#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
731
732#ifndef IEMLIVENESS_EXTENDED_LAYOUT
733# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
734# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
735#endif
736/** @} */
737
738
739/** @name The liveness state for a register.
740 *
741 * The state values have been picked to with state accumulation in mind (what
742 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
743 * performance critical work done with the values.
744 *
745 * This is a compressed state that only requires 2 bits per register.
746 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
747 * 1. the incoming state from the following call,
748 * 2. the outgoing state for this call,
749 * 3. mask of the entries set in the 2nd.
750 *
751 * The mask entry (3rd one above) will be used both when updating the outgoing
752 * state and when merging in incoming state for registers not touched by the
753 * current call.
754 *
755 *
756 * Extended Layout:
757 *
758 * The extended layout variation differs from the above as it records the
759 * different register accesses as individual bits, and it is currently used for
760 * the delayed EFLAGS calculation experiments. The latter means that
761 * calls/tb-exits and potential calls/exceptions/tb-exits are recorded
762 * separately so the latter can be checked for in combination with clobbering.
763 *
764 * @{ */
765#ifndef IEMLIVENESS_EXTENDED_LAYOUT
766/** The register will be clobbered and the current value thrown away.
767 *
768 * When this is applied to the state (2) we'll simply be AND'ing it with the
769 * (old) mask (3) and adding the register to the mask. This way we'll
770 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
771 * IEMLIVENESS_STATE_INPUT states. */
772# define IEMLIVENESS_STATE_CLOBBERED 0
773/** The register is unused in the remainder of the TB.
774 *
775 * This is an initial state and can not be set by any of the
776 * iemNativeLivenessFunc_xxxx callbacks. */
777# define IEMLIVENESS_STATE_UNUSED 1
778/** The register value is required in a potential call or exception.
779 *
780 * This means that the register value must be calculated and is best written to
781 * the state, but that any shadowing registers can be flushed thereafter as it's
782 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
783 *
784 * It is typically applied across the board, but we preserve incoming
785 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
786 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
787 * 1. r0 = old & ~mask;
788 * 2. r0 = t1 & (t1 >> 1);
789 * 3. state |= r0 | 0b10;
790 * 4. mask = ~0;
791 */
792# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
793/** The register value is used as input.
794 *
795 * This means that the register value must be calculated and it is best to keep
796 * it in a register. It does not need to be writtent out as such. This is the
797 * highest priority state.
798 *
799 * Whether the call modifies the register or not isn't relevant to earlier
800 * calls, so that's not recorded.
801 *
802 * When applying this state we just or in the value in the outgoing state and
803 * mask. */
804# define IEMLIVENESS_STATE_INPUT 3
805/** Mask of the state bits. */
806# define IEMLIVENESS_STATE_MASK 3
807/** The number of bits per state. */
808# define IEMLIVENESS_STATE_BIT_COUNT 2
809
810/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state.
811 * @note only used in assertions. */
812# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
813/** Check if we're expecting read accesses to a register with the given (previous) liveness state.
814 * @note only used in assertions. */
815# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
816/** Check if a register clobbering is expected given the (previous) liveness state.
817 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
818 * include INPUT if the register is used in more than one place.
819 * @note only used in assertions. */
820# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
821
822/** Check if all status flags are going to be clobbered and doesn't need
823 * calculating in the current step.
824 * @param a_pCurEntry The current liveness entry.
825 * @note Used by actual code. */
826# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
827 ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
828
829/** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
830 * states, as these are no longer needed.
831 * @param a_pCurEntry The current liveness entry.
832 * @note Used by actual code. */
833AssertCompile(IEMLIVENESS_STATE_UNUSED == 1 && IEMLIVENESS_STATE_XCPT_OR_CALL == 2);
834# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
835 ( (a_pCurEntry)->Bit0.bm64 ^ (a_pCurEntry)->Bit1.bm64 )
836
837
838#else /* IEMLIVENESS_EXTENDED_LAYOUT */
839/** The register is not used any more. */
840# define IEMLIVENESS_STATE_UNUSED 0
841/** Flag: The register is required in a potential call or/and exception. */
842# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
843# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
844/** Flag: The register is read. */
845# define IEMLIVENESS_STATE_READ 2
846# define IEMLIVENESS_BIT_READ 1
847/** Flag: The register is written. */
848# define IEMLIVENESS_STATE_WRITE 4
849# define IEMLIVENESS_BIT_WRITE 2
850/** Flag: Unconditional call. */
851# define IEMLIVENESS_STATE_CALL 8
852# define IEMLIVENESS_BIT_CALL 3
853
854# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
855 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
856# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
857# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
858
859# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
860 ( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
861 && !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
862 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
863 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
864 & IEMLIVENESSBIT_STATUS_EFL_MASK) )
865
866/** Construct a mask of the registers not in the read or write state.
867 * @note We could skips writes, if they aren't from us, as this is just a hack
868 * to prevent trashing registers that have just been written or will be
869 * written when we retire the current instruction.
870 * @param a_pCurEntry The current liveness entry.
871 * @note Used by actual code. */
872# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
873 ( ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
874 & ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
875 & IEMLIVENESSBIT_MASK )
876
877
878#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
879/** @} */
880
881/** @name Liveness helpers for builtin functions and similar.
882 *
883 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
884 * own set of manipulator macros for those.
885 *
886 * @{ */
887/** Initializing the state as all unused. */
888#ifndef IEMLIVENESS_EXTENDED_LAYOUT
889# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
890 do { \
891 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
892 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
893 } while (0)
894#else
895# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
896 do { \
897 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
898 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
899 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
900 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
901 } while (0)
902#endif
903
904/** Initializing the outgoing state with a potential xcpt or call state.
905 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT.
906 *
907 * @note Must invoke IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL when done!
908 */
909#ifndef IEMLIVENESS_EXTENDED_LAYOUT
910# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
911 do { \
912 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
913 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
914 } while (0)
915#else
916# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
917 do { \
918 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
919 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
920 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
921 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
922 } while (0)
923#endif
924
925/** Completes IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL after applying any
926 * other state modifications.
927 */
928#ifndef IEMLIVENESS_EXTENDED_LAYOUT
929# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) ((void)0)
930#else
931# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
932 do { \
933 uint64_t const fInhMask = ~( (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL].bm64 \
934 | (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE].bm64); \
935 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 |= (a_pIncoming)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 & fInhMask; \
936 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 |= (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64 & fInhMask; \
937 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 |= (a_pIncoming)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & fInhMask; \
938 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 |= (a_pIncoming)->aBits[IEMLIVENESS_BIT_CALL].bm64 & fInhMask; \
939 } while (0)
940#endif
941
942/** Initializing the outgoing state with an unconditional call state.
943 * This should only really be used alone. */
944#ifndef IEMLIVENESS_EXTENDED_LAYOUT
945# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
946 do { \
947 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
948 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
949 } while (0)
950#else
951# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
952 do { \
953 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
954 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
955 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
956 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
957 RT_NOREF(a_pIncoming); \
958 } while (0)
959#endif
960
961#if 0 /* unused */
962/** Initializing the outgoing state with an unconditional call state as well as
963 * an potential call/exception preceeding it.
964 * This should only really be used alone. */
965#ifndef IEMLIVENESS_EXTENDED_LAYOUT
966# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
967 do { \
968 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
969 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
970 } while (0)
971#else
972# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
973 do { \
974 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
975 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
976 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
977 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
978 } while (0)
979#endif
980#endif
981
982/** Adds a segment base register as input to the outgoing state. */
983#ifndef IEMLIVENESS_EXTENDED_LAYOUT
984# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
985 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
986 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
987 } while (0)
988#else
989# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
990 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
991 } while (0)
992#endif
993
994/** Adds a segment attribute register as input to the outgoing state. */
995#ifndef IEMLIVENESS_EXTENDED_LAYOUT
996# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
997 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
998 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
999 } while (0)
1000#else
1001# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1002 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
1003 } while (0)
1004#endif
1005
1006/** Adds a segment limit register as input to the outgoing state. */
1007#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1008# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1009 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
1010 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
1011 } while (0)
1012#else
1013# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1014 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
1015 } while (0)
1016#endif
1017
1018/** Adds a segment limit register as input to the outgoing state. */
1019#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1020# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1021 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
1022 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
1023 } while (0)
1024#else
1025# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1026 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
1027 } while (0)
1028#endif
1029/** @} */
1030
1031/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
1032 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
1033 * calculated and up to date. This is to double check that we haven't skipped
1034 * EFLAGS calculations when we actually need them. NOP in non-strict builds.
1035 * @note has to be placed in
1036 */
1037#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1038# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
1039 do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
1040#else
1041# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
1042#endif
1043
1044
1045/**
1046 * Guest registers that can be shadowed in GPRs.
1047 *
1048 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
1049 * must be placed last, as the liveness state tracks it as 7 subcomponents and
1050 * we don't want to waste space here.
1051 *
1052 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
1053 * friends as well as IEMAllN8veLiveness.cpp.
1054 */
1055typedef enum IEMNATIVEGSTREG : uint8_t
1056{
1057 kIemNativeGstReg_GprFirst = 0,
1058 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
1059 kIemNativeGstReg_Pc,
1060 kIemNativeGstReg_Cr0,
1061 kIemNativeGstReg_FpuFcw,
1062 kIemNativeGstReg_FpuFsw,
1063 kIemNativeGstReg_SegBaseFirst,
1064 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1065 kIemNativeGstReg_SegAttribFirst,
1066 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1067 kIemNativeGstReg_SegLimitFirst,
1068 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1069 kIemNativeGstReg_SegSelFirst,
1070 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1071 kIemNativeGstReg_Cr4,
1072 kIemNativeGstReg_Xcr0,
1073 kIemNativeGstReg_MxCsr,
1074 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
1075 kIemNativeGstReg_End
1076} IEMNATIVEGSTREG;
1077AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1078AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1079
1080/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1081 * @{ */
1082#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1083#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1084#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1085#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1086#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1087/** @} */
1088
1089#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1090
1091/**
1092 * Guest registers that can be shadowed in host SIMD registers.
1093 *
1094 * @todo r=aeichner Liveness tracking
1095 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1096 */
1097typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1098{
1099 kIemNativeGstSimdReg_SimdRegFirst = 0,
1100 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1101 kIemNativeGstSimdReg_End
1102} IEMNATIVEGSTSIMDREG;
1103
1104/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1105 * @{ */
1106#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1107/** @} */
1108
1109/**
1110 * The Load/store size for a SIMD guest register.
1111 */
1112typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1113{
1114 /** Invalid size. */
1115 kIemNativeGstSimdRegLdStSz_Invalid = 0,
1116 /** Loads the low 128-bit of a guest SIMD register. */
1117 kIemNativeGstSimdRegLdStSz_Low128,
1118 /** Loads the high 128-bit of a guest SIMD register. */
1119 kIemNativeGstSimdRegLdStSz_High128,
1120 /** Loads the whole 256-bits of a guest SIMD register. */
1121 kIemNativeGstSimdRegLdStSz_256,
1122 /** End value. */
1123 kIemNativeGstSimdRegLdStSz_End
1124} IEMNATIVEGSTSIMDREGLDSTSZ;
1125
1126#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1127
1128/**
1129 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
1130 */
1131typedef enum IEMNATIVEGSTREGUSE
1132{
1133 /** The usage is read-only, the register holding the guest register
1134 * shadow copy will not be modified by the caller. */
1135 kIemNativeGstRegUse_ReadOnly = 0,
1136 /** The caller will update the guest register (think: PC += cbInstr).
1137 * The guest shadow copy will follow the returned register. */
1138 kIemNativeGstRegUse_ForUpdate,
1139 /** The call will put an entirely new value in the guest register, so
1140 * if new register is allocate it will be returned uninitialized. */
1141 kIemNativeGstRegUse_ForFullWrite,
1142 /** The caller will use the guest register value as input in a calculation
1143 * and the host register will be modified.
1144 * This means that the returned host register will not be marked as a shadow
1145 * copy of the guest register. */
1146 kIemNativeGstRegUse_Calculation
1147} IEMNATIVEGSTREGUSE;
1148
1149/**
1150 * Guest registers (classes) that can be referenced.
1151 */
1152typedef enum IEMNATIVEGSTREGREF : uint8_t
1153{
1154 kIemNativeGstRegRef_Invalid = 0,
1155 kIemNativeGstRegRef_Gpr,
1156 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
1157 kIemNativeGstRegRef_EFlags,
1158 kIemNativeGstRegRef_MxCsr,
1159 kIemNativeGstRegRef_FpuReg,
1160 kIemNativeGstRegRef_MReg,
1161 kIemNativeGstRegRef_XReg,
1162 kIemNativeGstRegRef_X87,
1163 kIemNativeGstRegRef_XState,
1164 //kIemNativeGstRegRef_YReg, - doesn't work.
1165 kIemNativeGstRegRef_End
1166} IEMNATIVEGSTREGREF;
1167
1168
1169/** Variable kinds. */
1170typedef enum IEMNATIVEVARKIND : uint8_t
1171{
1172 /** Customary invalid zero value. */
1173 kIemNativeVarKind_Invalid = 0,
1174 /** This is either in a register or on the stack. */
1175 kIemNativeVarKind_Stack,
1176 /** Immediate value - loaded into register when needed, or can live on the
1177 * stack if referenced (in theory). */
1178 kIemNativeVarKind_Immediate,
1179 /** Variable reference - loaded into register when needed, never stack. */
1180 kIemNativeVarKind_VarRef,
1181 /** Guest register reference - loaded into register when needed, never stack. */
1182 kIemNativeVarKind_GstRegRef,
1183 /** End of valid values. */
1184 kIemNativeVarKind_End
1185} IEMNATIVEVARKIND;
1186
1187
1188/** Variable or argument. */
1189typedef struct IEMNATIVEVAR
1190{
1191 /** The kind of variable. */
1192 IEMNATIVEVARKIND enmKind;
1193 /** The variable size in bytes. */
1194 uint8_t cbVar;
1195 /** The first stack slot (uint64_t), except for immediate and references
1196 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1197 * has a stack slot it has been initialized and has a value. Unused variables
1198 * has neither a stack slot nor a host register assignment. */
1199 uint8_t idxStackSlot;
1200 /** The host register allocated for the variable, UINT8_MAX if not. */
1201 uint8_t idxReg;
1202 /** The argument number if argument, UINT8_MAX if regular variable. */
1203 uint8_t uArgNo;
1204 /** If referenced, the index (unpacked) of the variable referencing this one,
1205 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1206 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1207 uint8_t idxReferrerVar;
1208 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1209 * @todo not sure what this really is for... */
1210 IEMNATIVEGSTREG enmGstReg;
1211#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1212 /** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1213 * only valid when idxReg is not UINT8_MAX. */
1214 bool fSimdReg : 1;
1215 /** Set if the registered is currently used exclusively, false if the
1216 * variable is idle and the register can be grabbed. */
1217 bool fRegAcquired : 1;
1218#else
1219 /** Set if the registered is currently used exclusively, false if the
1220 * variable is idle and the register can be grabbed. */
1221 bool fRegAcquired;
1222#endif
1223
1224 union
1225 {
1226 /** kIemNativeVarKind_Immediate: The immediate value. */
1227 uint64_t uValue;
1228 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1229 uint8_t idxRefVar;
1230 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1231 struct
1232 {
1233 /** The class of register. */
1234 IEMNATIVEGSTREGREF enmClass;
1235 /** Index within the class. */
1236 uint8_t idx;
1237 } GstRegRef;
1238 } u;
1239} IEMNATIVEVAR;
1240/** Pointer to a variable or argument. */
1241typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1242/** Pointer to a const variable or argument. */
1243typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1244
1245/** What is being kept in a host register. */
1246typedef enum IEMNATIVEWHAT : uint8_t
1247{
1248 /** The traditional invalid zero value. */
1249 kIemNativeWhat_Invalid = 0,
1250 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1251 kIemNativeWhat_Var,
1252 /** Temporary register, this is typically freed when a MC completes. */
1253 kIemNativeWhat_Tmp,
1254 /** Call argument w/o a variable mapping. This is free (via
1255 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1256 kIemNativeWhat_Arg,
1257 /** Return status code.
1258 * @todo not sure if we need this... */
1259 kIemNativeWhat_rc,
1260 /** The fixed pVCpu (PVMCPUCC) register.
1261 * @todo consider offsetting this on amd64 to use negative offsets to access
1262 * more members using 8-byte disp. */
1263 kIemNativeWhat_pVCpuFixed,
1264 /** The fixed pCtx (PCPUMCTX) register.
1265 * @todo consider offsetting this on amd64 to use negative offsets to access
1266 * more members using 8-byte disp. */
1267 kIemNativeWhat_pCtxFixed,
1268 /** Fixed temporary register. */
1269 kIemNativeWhat_FixedTmp,
1270#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1271 /** Shadow RIP for the delayed RIP updating debugging. */
1272 kIemNativeWhat_PcShadow,
1273#endif
1274 /** Register reserved by the CPU or OS architecture. */
1275 kIemNativeWhat_FixedReserved,
1276 /** End of valid values. */
1277 kIemNativeWhat_End
1278} IEMNATIVEWHAT;
1279
1280/**
1281 * Host general register entry.
1282 *
1283 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1284 *
1285 * @todo Track immediate values in host registers similarlly to how we track the
1286 * guest register shadow copies. For it to be real helpful, though,
1287 * we probably need to know which will be reused and put them into
1288 * non-volatile registers, otherwise it's going to be more or less
1289 * restricted to an instruction or two.
1290 */
1291typedef struct IEMNATIVEHSTREG
1292{
1293 /** Set of guest registers this one shadows.
1294 *
1295 * Using a bitmap here so we can designate the same host register as a copy
1296 * for more than one guest register. This is expected to be useful in
1297 * situations where one value is copied to several registers in a sequence.
1298 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1299 * sequence we'd want to let this register follow to be a copy of and there
1300 * will always be places where we'd be picking the wrong one.
1301 */
1302 uint64_t fGstRegShadows;
1303 /** What is being kept in this register. */
1304 IEMNATIVEWHAT enmWhat;
1305 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1306 uint8_t idxVar;
1307 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1308 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1309 * that scope. */
1310 uint8_t idxStackSlot;
1311 /** Alignment padding. */
1312 uint8_t abAlign[5];
1313} IEMNATIVEHSTREG;
1314
1315
1316#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1317/**
1318 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1319 * halves, on architectures where there is no 256-bit register available this entry will track
1320 * two adjacent 128-bit host registers.
1321 *
1322 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1323 */
1324typedef struct IEMNATIVEHSTSIMDREG
1325{
1326 /** Set of guest registers this one shadows.
1327 *
1328 * Using a bitmap here so we can designate the same host register as a copy
1329 * for more than one guest register. This is expected to be useful in
1330 * situations where one value is copied to several registers in a sequence.
1331 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1332 * sequence we'd want to let this register follow to be a copy of and there
1333 * will always be places where we'd be picking the wrong one.
1334 */
1335 uint64_t fGstRegShadows;
1336 /** What is being kept in this register. */
1337 IEMNATIVEWHAT enmWhat;
1338 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1339 uint8_t idxVar;
1340 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1341 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1342 /** Alignment padding. */
1343 uint8_t abAlign[5];
1344} IEMNATIVEHSTSIMDREG;
1345#endif
1346
1347
1348/**
1349 * Core state for the native recompiler, that is, things that needs careful
1350 * handling when dealing with branches.
1351 */
1352typedef struct IEMNATIVECORESTATE
1353{
1354 /** Allocation bitmap for aHstRegs. */
1355 uint32_t bmHstRegs;
1356
1357 /** Bitmap marking which host register contains guest register shadow copies.
1358 * This is used during register allocation to try preserve copies. */
1359 uint32_t bmHstRegsWithGstShadow;
1360 /** Bitmap marking valid entries in aidxGstRegShadows. */
1361 uint64_t bmGstRegShadows;
1362#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1363 /** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1364 uint64_t bmGstRegShadowDirty;
1365#endif
1366
1367#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1368 /** The current instruction offset in bytes from when the guest program counter
1369 * was updated last. Used for delaying the write to the guest context program counter
1370 * as long as possible. */
1371 int64_t offPc;
1372# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1373 /** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1374 bool fDebugPcInitialized;
1375# endif
1376#endif
1377
1378#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1379 /** Allocation bitmap for aHstSimdRegs. */
1380 uint32_t bmHstSimdRegs;
1381
1382 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1383 * This is used during register allocation to try preserve copies. */
1384 uint32_t bmHstSimdRegsWithGstShadow;
1385 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1386 uint64_t bmGstSimdRegShadows;
1387 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1388 uint64_t bmGstSimdRegShadowDirtyLo128;
1389 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1390 uint64_t bmGstSimdRegShadowDirtyHi128;
1391#endif
1392
1393 union
1394 {
1395 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1396 uint8_t aidxArgVars[8];
1397 /** For more efficient resetting. */
1398 uint64_t u64ArgVars;
1399 };
1400
1401 /** Allocation bitmap for the stack. */
1402 uint32_t bmStack;
1403 /** Allocation bitmap for aVars. */
1404 uint32_t bmVars;
1405
1406 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1407 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1408 * (A shadow copy of a guest register can only be held in a one host register,
1409 * there are no duplicate copies or ambiguities like that). */
1410 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1411#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1412 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1413 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1414 * (A shadow copy of a guest register can only be held in a one host register,
1415 * there are no duplicate copies or ambiguities like that). */
1416 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1417#endif
1418
1419 /** Host register allocation tracking. */
1420 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1421#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1422 /** Host SIMD register allocation tracking. */
1423 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1424#endif
1425
1426 /** Variables and arguments. */
1427 IEMNATIVEVAR aVars[9];
1428} IEMNATIVECORESTATE;
1429/** Pointer to core state. */
1430typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1431/** Pointer to const core state. */
1432typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1433
1434/** @def IEMNATIVE_VAR_IDX_UNPACK
1435 * @returns Index into IEMNATIVECORESTATE::aVars.
1436 * @param a_idxVar Variable index w/ magic (in strict builds).
1437 */
1438/** @def IEMNATIVE_VAR_IDX_PACK
1439 * @returns Variable index w/ magic (in strict builds).
1440 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1441 */
1442#ifdef VBOX_STRICT
1443# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1444# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1445# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1446# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1447# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1448#else
1449# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1450# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1451#endif
1452
1453
1454#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1455/** Clear the dirty state of the given guest SIMD register. */
1456# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1457 do { \
1458 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1459 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1460 } while (0)
1461
1462/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1463# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1464 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1465/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1466# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1467 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1468/** Returns whether the given guest SIMD register is dirty. */
1469# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1470 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1471
1472/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1473# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1474 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1475/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1476# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1477 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1478
1479/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1480# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1481 /** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1482# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1483/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1484# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1485/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1486# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1487# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1488/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1489# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1490/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1491# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1492# endif
1493#endif
1494
1495
1496/**
1497 * Conditional stack entry.
1498 */
1499typedef struct IEMNATIVECOND
1500{
1501 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1502 bool fInElse;
1503 union
1504 {
1505 RT_GCC_EXTENSION struct
1506 {
1507 /** Set if the if-block unconditionally exited the TB. */
1508 bool fIfExitTb;
1509 /** Set if the else-block unconditionally exited the TB. */
1510 bool fElseExitTb;
1511 };
1512 /** Indexed by fInElse. */
1513 bool afExitTb[2];
1514 };
1515 bool afPadding[5];
1516 /** The label for the IEM_MC_ELSE. */
1517 uint32_t idxLabelElse;
1518 /** The label for the IEM_MC_ENDIF. */
1519 uint32_t idxLabelEndIf;
1520 /** The initial state snapshot as the if-block starts executing. */
1521 IEMNATIVECORESTATE InitialState;
1522 /** The state snapshot at the end of the if-block. */
1523 IEMNATIVECORESTATE IfFinalState;
1524} IEMNATIVECOND;
1525/** Pointer to a condition stack entry. */
1526typedef IEMNATIVECOND *PIEMNATIVECOND;
1527
1528
1529/**
1530 * Native recompiler state.
1531 */
1532typedef struct IEMRECOMPILERSTATE
1533{
1534 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1535 * IEMNATIVEINSTR units. */
1536 uint32_t cInstrBufAlloc;
1537#ifdef VBOX_STRICT
1538 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1539 uint32_t offInstrBufChecked;
1540#else
1541 uint32_t uPadding1; /* We don't keep track of the size here... */
1542#endif
1543 /** Fixed temporary code buffer for native recompilation. */
1544 PIEMNATIVEINSTR pInstrBuf;
1545
1546 /** Bitmaps with the label types used. */
1547 uint64_t bmLabelTypes;
1548 /** Actual number of labels in paLabels. */
1549 uint32_t cLabels;
1550 /** Max number of entries allowed in paLabels before reallocating it. */
1551 uint32_t cLabelsAlloc;
1552 /** Labels defined while recompiling (referenced by fixups). */
1553 PIEMNATIVELABEL paLabels;
1554 /** Array with indexes of unique labels (uData always 0). */
1555 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1556
1557 /** Actual number of fixups paFixups. */
1558 uint32_t cFixups;
1559 /** Max number of entries allowed in paFixups before reallocating it. */
1560 uint32_t cFixupsAlloc;
1561 /** Buffer used by the recompiler for recording fixups when generating code. */
1562 PIEMNATIVEFIXUP paFixups;
1563
1564#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1565 /** Actual number of fixups in paTbExitFixups. */
1566 uint32_t cTbExitFixups;
1567 /** Max number of entries allowed in paTbExitFixups before reallocating it. */
1568 uint32_t cTbExitFixupsAlloc;
1569 /** Buffer used by the recompiler for recording fixups when generating code. */
1570 PIEMNATIVEEXITFIXUP paTbExitFixups;
1571#endif
1572
1573#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) || defined(VBOX_WITH_STATISTICS)
1574 /** Statistics: The idxInstr+1 value at the last PC update. */
1575 uint8_t idxInstrPlusOneOfLastPcUpdate;
1576#endif
1577
1578#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1579 /** Number of debug info entries allocated for pDbgInfo. */
1580 uint32_t cDbgInfoAlloc;
1581 /** Debug info. */
1582 PIEMTBDBG pDbgInfo;
1583#endif
1584
1585#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1586 /** The current call index (liveness array and threaded calls in TB). */
1587 uint32_t idxCurCall;
1588 /** Number of liveness entries allocated. */
1589 uint32_t cLivenessEntriesAlloc;
1590 /** Liveness entries for all the calls in the TB begin recompiled.
1591 * The entry for idxCurCall contains the info for what the next call will
1592 * require wrt registers. (Which means the last entry is the initial liveness
1593 * state.) */
1594 PIEMLIVENESSENTRY paLivenessEntries;
1595#endif
1596
1597 /** The translation block being recompiled. */
1598 PCIEMTB pTbOrg;
1599 /** The VMCPU structure of the EMT. */
1600 PVMCPUCC pVCpu;
1601
1602 /** Condition sequence number (for generating unique labels). */
1603 uint16_t uCondSeqNo;
1604 /** Check IRQ sequence number (for generating unique labels). */
1605 uint16_t uCheckIrqSeqNo;
1606 /** TLB load sequence number (for generating unique labels). */
1607 uint16_t uTlbSeqNo;
1608 /** The current condition stack depth (aCondStack). */
1609 uint8_t cCondDepth;
1610
1611 /** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1612 uint8_t cArgsX;
1613 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1614 uint32_t fCImpl;
1615 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1616 uint32_t fMc;
1617 /** The expected IEMCPU::fExec value for the current call/instruction. */
1618 uint32_t fExec;
1619#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1620 /** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1621 * we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1622 *
1623 * This is an optimization because these control registers can only be changed from
1624 * by calling a C helper we can catch. Should reduce the number of instructions in a TB
1625 * consisting of multiple SIMD instructions.
1626 */
1627 uint32_t fSimdRaiseXcptChecksEmitted;
1628#endif
1629 /** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1630 uint32_t idxLastCheckIrqCallNo;
1631
1632 /** Core state requiring care with branches. */
1633 IEMNATIVECORESTATE Core;
1634
1635 /** The condition nesting stack. */
1636 IEMNATIVECOND aCondStack[2];
1637
1638#ifndef IEM_WITH_THROW_CATCH
1639 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1640 * for recompilation error handling. */
1641 jmp_buf JmpBuf;
1642#endif
1643} IEMRECOMPILERSTATE;
1644/** Pointer to a native recompiler state. */
1645typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1646
1647
1648/** @def IEMNATIVE_TRY_SETJMP
1649 * Wrapper around setjmp / try, hiding all the ugly differences.
1650 *
1651 * @note Use with extreme care as this is a fragile macro.
1652 * @param a_pReNative The native recompile state.
1653 * @param a_rcTarget The variable that should receive the status code in case
1654 * of a longjmp/throw.
1655 */
1656/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1657 * Start wrapper for catch / setjmp-else.
1658 *
1659 * This will set up a scope.
1660 *
1661 * @note Use with extreme care as this is a fragile macro.
1662 * @param a_pReNative The native recompile state.
1663 * @param a_rcTarget The variable that should receive the status code in case
1664 * of a longjmp/throw.
1665 */
1666/** @def IEMNATIVE_CATCH_LONGJMP_END
1667 * End wrapper for catch / setjmp-else.
1668 *
1669 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1670 * up the state.
1671 *
1672 * @note Use with extreme care as this is a fragile macro.
1673 * @param a_pReNative The native recompile state.
1674 */
1675/** @def IEMNATIVE_DO_LONGJMP
1676 *
1677 * Wrapper around longjmp / throw.
1678 *
1679 * @param a_pReNative The native recompile state.
1680 * @param a_rc The status code jump back with / throw.
1681 */
1682#ifdef IEM_WITH_THROW_CATCH
1683# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1684 a_rcTarget = VINF_SUCCESS; \
1685 try
1686# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1687 catch (int rcThrown) \
1688 { \
1689 a_rcTarget = rcThrown
1690# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1691 } \
1692 ((void)0)
1693# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1694#else /* !IEM_WITH_THROW_CATCH */
1695# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1696 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1697# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1698 else \
1699 { \
1700 ((void)0)
1701# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1702 }
1703# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1704#endif /* !IEM_WITH_THROW_CATCH */
1705
1706
1707/**
1708 * Native recompiler worker for a threaded function.
1709 *
1710 * @returns New code buffer offset; throws VBox status code in case of a failure.
1711 * @param pReNative The native recompiler state.
1712 * @param off The current code buffer offset.
1713 * @param pCallEntry The threaded call entry.
1714 *
1715 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1716 */
1717typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1718/** Pointer to a native recompiler worker for a threaded function. */
1719typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1720
1721/** Defines a native recompiler worker for a threaded function.
1722 * @see FNIEMNATIVERECOMPFUNC */
1723#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1724 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1725
1726/** Prototypes a native recompiler function for a threaded function.
1727 * @see FNIEMNATIVERECOMPFUNC */
1728#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1729
1730
1731/**
1732 * Native recompiler liveness analysis worker for a threaded function.
1733 *
1734 * @param pCallEntry The threaded call entry.
1735 * @param pIncoming The incoming liveness state entry.
1736 * @param pOutgoing The outgoing liveness state entry.
1737 */
1738typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1739 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1740/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1741typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1742
1743/** Defines a native recompiler liveness analysis worker for a threaded function.
1744 * @see FNIEMNATIVELIVENESSFUNC */
1745#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1746 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1747
1748/** Prototypes a native recompiler liveness analysis function for a threaded function.
1749 * @see FNIEMNATIVELIVENESSFUNC */
1750#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1751
1752
1753/** Define a native recompiler helper function, safe to call from the TB code. */
1754#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1755 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1756/** Prototype a native recompiler helper function, safe to call from the TB code. */
1757#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1758 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1759/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1760#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1761 a_RetType (VBOXCALL *a_Name) a_ArgList
1762
1763
1764#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1765DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1766DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1767 uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1768# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1769DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1770 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1771 uint8_t idxHstSimdReg = UINT8_MAX,
1772 uint8_t idxHstSimdRegPrev = UINT8_MAX);
1773# endif
1774# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1775DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1776 uint8_t idxGstReg, uint8_t idxHstReg);
1777DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1778 uint64_t fGstReg);
1779# endif
1780DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1781 uint64_t offPc, uint32_t cInstrSkipped);
1782#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1783
1784DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1785 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1786DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1787DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1788 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1789DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1790 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1791#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1792DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1793#endif
1794DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1795
1796DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1797DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1798 bool fPreferVolatile = true);
1799DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1800 bool fPreferVolatile = true);
1801DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1802 IEMNATIVEGSTREG enmGstReg,
1803 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1804 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1805DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1806 IEMNATIVEGSTREG enmGstReg);
1807
1808DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1809DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1810#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) || (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1811DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1812 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1813# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1814DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1815 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1816# endif
1817#endif
1818DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1819DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1820DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1821DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1822#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1823DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1824# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1825DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1826# endif
1827#endif
1828DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1829DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1830 uint32_t fKeepVars = 0);
1831DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1832DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1833DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1834 uint32_t fHstRegsActiveShadows);
1835#ifdef VBOX_STRICT
1836DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1837#endif
1838DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1839 uint64_t fGstSimdShwExcept);
1840#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1841# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1842DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1843DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
1844# endif
1845DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1846#endif
1847#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1848DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1849DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1850 PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
1851DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1852 uint64_t fFlushGstReg = UINT64_MAX);
1853DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
1854 uint32_t off, uint8_t idxHstReg);
1855#endif
1856
1857
1858#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1859DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1860DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1861 bool fPreferVolatile = true);
1862DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1863 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1864 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1865 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1866 bool fNoVolatileRegs = false);
1867DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1868DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1869DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1870 IEMNATIVEGSTSIMDREG enmGstSimdReg);
1871DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1872 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1873 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1874#endif
1875
1876DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1877DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1878DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1879DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1880DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1881DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1882DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1883DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1884DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1885 IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1886DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1887DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1888 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1889#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1890DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1891 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1892#endif
1893DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1894 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1895DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1896 uint32_t fHstRegsNotToSave);
1897DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1898 uint32_t fHstRegsNotToSave);
1899DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1900DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1901
1902DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1903 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1904#ifdef VBOX_STRICT
1905DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1906DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1907 IEMNATIVEGSTREG enmGstReg);
1908# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1909DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1910 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1911 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1912# endif
1913DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1914#endif
1915#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1916DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1917#endif
1918DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1919DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1920DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1921 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1922 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1923DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1924 PCIEMTHRDEDCALLENTRY pCallEntry);
1925IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(iemNativeLivenessFunc_ThreadedCall);
1926DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1927 IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1928
1929
1930IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1931IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1932IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1933IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1934IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1935IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1936IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1937IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1938IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1939IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1940IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1941IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1942IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1943
1944IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1945IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1946IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1947IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1948IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1949IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1950IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1951IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1952IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1953IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1954#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1955IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1956IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1957IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1958IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1959IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1960#endif
1961IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1962IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1963IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1964IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1965#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1966IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1967IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1968IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1969IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1970#endif
1971IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1972IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1973IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1974IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1975IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1976IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1977IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1978
1979IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1980IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1981IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1982IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1983IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1984IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1985IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1986IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1987IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1988IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1989#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1990IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1991IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1992IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1993IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1994IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1995#endif
1996IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1997IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1998IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1999IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2000#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2001IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2002IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2003IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2004IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2005#endif
2006IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2007IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2008IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2009IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2010IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2011IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2012IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2013
2014IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2015IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2016IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2017IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2018IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2019IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2020IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2021IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2022IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2023IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2024IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2025IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2026IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2027IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2028IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2029IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2030IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2031IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2032IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2033IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2034IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2035IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2036
2037IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2038IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2039IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2040IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2041IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2042IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2043IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2044IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2045IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2046IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2047IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2048IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2049IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2050IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2051IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2052IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2053IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2054IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2055IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2056IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2057IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2058IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
2059
2060IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2061IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2062IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2063IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2064
2065
2066/**
2067 * Info about shadowed guest register values.
2068 * @see IEMNATIVEGSTREG
2069 */
2070typedef struct IEMANTIVEGSTREGINFO
2071{
2072 /** Offset in VMCPU. */
2073 uint32_t off;
2074 /** The field size. */
2075 uint8_t cb;
2076 /** Name (for logging). */
2077 const char *pszName;
2078} IEMANTIVEGSTREGINFO;
2079extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2080extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2081extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2082extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2083extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2084
2085
2086
2087/**
2088 * Ensures that there is sufficient space in the instruction output buffer.
2089 *
2090 * This will reallocate the buffer if needed and allowed.
2091 *
2092 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2093 * allocation size.
2094 *
2095 * @returns Pointer to the instruction output buffer on success; throws VBox
2096 * status code on failure, so no need to check it.
2097 * @param pReNative The native recompile state.
2098 * @param off Current instruction offset. Works safely for UINT32_MAX
2099 * as well.
2100 * @param cInstrReq Number of instruction about to be added. It's okay to
2101 * overestimate this a bit.
2102 */
2103DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2104iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2105{
2106 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2107 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2108 {
2109#ifdef VBOX_STRICT
2110 pReNative->offInstrBufChecked = offChecked;
2111#endif
2112 return pReNative->pInstrBuf;
2113 }
2114 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2115}
2116
2117/**
2118 * Checks that we didn't exceed the space requested in the last
2119 * iemNativeInstrBufEnsure() call.
2120 */
2121#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2122 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2123 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2124
2125/**
2126 * Checks that a variable index is valid.
2127 */
2128#ifdef IEMNATIVE_VAR_IDX_MAGIC
2129# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2130 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2131 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2132 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2133 ("%s=%#x\n", #a_idxVar, a_idxVar))
2134#else
2135# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2136 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2137 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2138#endif
2139
2140/**
2141 * Checks that a variable index is valid and that the variable is assigned the
2142 * correct argument number.
2143 * This also adds a RT_NOREF of a_idxVar.
2144 */
2145#ifdef IEMNATIVE_VAR_IDX_MAGIC
2146# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2147 RT_NOREF_PV(a_idxVar); \
2148 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2149 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2150 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2151 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2152 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2153 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2154 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2155 a_uArgNo)); \
2156 } while (0)
2157#else
2158# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2159 RT_NOREF_PV(a_idxVar); \
2160 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2161 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2162 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2163 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2164 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2165 } while (0)
2166#endif
2167
2168
2169/**
2170 * Checks that a variable has the expected size.
2171 */
2172#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2173 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2174 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2175 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2176
2177
2178/**
2179 * Calculates the stack address of a variable as a [r]BP displacement value.
2180 */
2181DECL_FORCE_INLINE(int32_t)
2182iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2183{
2184 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2185 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2186}
2187
2188
2189/**
2190 * Releases the variable's register.
2191 *
2192 * The register must have been previously acquired calling
2193 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2194 * iemNativeVarRegisterSetAndAcquire().
2195 */
2196DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2197{
2198 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2199 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2200 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2201}
2202
2203
2204#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2205DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2206{
2207 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2208 iemNativeVarRegisterRelease(pReNative, idxVar);
2209}
2210#endif
2211
2212
2213/**
2214 * Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2215 *
2216 * @returns The flush mask.
2217 * @param fCImpl The IEM_CIMPL_F_XXX flags.
2218 * @param fGstShwFlush The starting flush mask.
2219 */
2220DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2221{
2222 if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2223 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2224 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2225 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2226 if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2227 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2228 | RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2229 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2230 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2231 else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2232 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2233 if (fCImpl & (IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_STATUS_FLAGS | IEM_CIMPL_F_INHIBIT_SHADOW))
2234 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
2235 return fGstShwFlush;
2236}
2237
2238
2239/** Number of hidden arguments for CIMPL calls.
2240 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2241#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2242# define IEM_CIMPL_HIDDEN_ARGS 3
2243#else
2244# define IEM_CIMPL_HIDDEN_ARGS 2
2245#endif
2246
2247
2248#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2249/** Number of hidden arguments for SSE_AIMPL calls. */
2250# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2251/** Number of hidden arguments for AVX_AIMPL calls. */
2252# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2253#endif
2254
2255
2256#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2257
2258# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2259/**
2260 * Helper for iemNativeLivenessGetStateByGstReg.
2261 *
2262 * @returns IEMLIVENESS_STATE_XXX
2263 * @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2264 * ORed together.
2265 */
2266DECL_FORCE_INLINE(uint32_t)
2267iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2268{
2269 /* INPUT trumps anything else. */
2270 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2271 return IEMLIVENESS_STATE_INPUT;
2272
2273 /* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2274 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2275 {
2276 /* If not all sub-fields are clobbered they must be considered INPUT. */
2277 if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) | RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2278 return IEMLIVENESS_STATE_INPUT;
2279 return IEMLIVENESS_STATE_CLOBBERED;
2280 }
2281
2282 /* XCPT_OR_CALL trumps UNUSED. */
2283 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2284 return IEMLIVENESS_STATE_XCPT_OR_CALL;
2285
2286 return IEMLIVENESS_STATE_UNUSED;
2287}
2288# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2289
2290
2291DECL_FORCE_INLINE(uint32_t)
2292iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2293{
2294# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2295 return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2296 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2297# else
2298 return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2299 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2300 | (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2301 | (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2302# endif
2303}
2304
2305
2306DECL_FORCE_INLINE(uint32_t)
2307iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2308{
2309 uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2310 if (enmGstReg == kIemNativeGstReg_EFlags)
2311 {
2312 /* Merge the eflags states to one. */
2313# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2314 uRet = RT_BIT_32(uRet);
2315 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflCf | (pLivenessEntry->Bit1.fEflCf << 1));
2316 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflPf | (pLivenessEntry->Bit1.fEflPf << 1));
2317 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflAf | (pLivenessEntry->Bit1.fEflAf << 1));
2318 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflZf | (pLivenessEntry->Bit1.fEflZf << 1));
2319 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflSf | (pLivenessEntry->Bit1.fEflSf << 1));
2320 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflOf | (pLivenessEntry->Bit1.fEflOf << 1));
2321 uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2322# else
2323 AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2324 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2325 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2326 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2327 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2328 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2329 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2330# endif
2331 }
2332 return uRet;
2333}
2334
2335
2336# ifdef VBOX_STRICT
2337/** For assertions only - caller checks that idxCurCall isn't zero. */
2338DECL_FORCE_INLINE(uint32_t)
2339iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2340{
2341 return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2342}
2343# endif /* VBOX_STRICT */
2344
2345#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2346
2347
2348/**
2349 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2350 */
2351DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2352{
2353 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2354 return IEM_CIMPL_HIDDEN_ARGS;
2355 if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE | IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2356 return 1;
2357 return 0;
2358}
2359
2360
2361DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2362 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2363{
2364 pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
2365
2366 pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2367 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2368 pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2369 return (uint8_t)idxReg;
2370}
2371
2372
2373
2374/*********************************************************************************************************************************
2375* Register Allocator (GPR) *
2376*********************************************************************************************************************************/
2377
2378/**
2379 * Marks host register @a idxHstReg as containing a shadow copy of guest
2380 * register @a enmGstReg.
2381 *
2382 * ASSUMES that caller has made sure @a enmGstReg is not associated with any
2383 * host register before calling.
2384 */
2385DECL_FORCE_INLINE(void)
2386iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2387{
2388 Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2389 Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2390 Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2391
2392 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2393 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2394 pReNative->Core.bmGstRegShadows |= RT_BIT_64(enmGstReg);
2395 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxHstReg);
2396#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2397 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2398 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2399#else
2400 RT_NOREF(off);
2401#endif
2402}
2403
2404
2405/**
2406 * Clear any guest register shadow claims from @a idxHstReg.
2407 *
2408 * The register does not need to be shadowing any guest registers.
2409 */
2410DECL_FORCE_INLINE(void)
2411iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2412{
2413 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2414 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2415 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2416 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2417 == RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2418#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2419 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2420#endif
2421
2422#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2423 uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2424 if (fGstRegs)
2425 {
2426 Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2427 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2428 while (fGstRegs)
2429 {
2430 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2431 fGstRegs &= ~RT_BIT_64(iGstReg);
2432 iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2433 }
2434 }
2435#else
2436 RT_NOREF(off);
2437#endif
2438
2439 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2440 pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2441 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2442}
2443
2444
2445/**
2446 * Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2447 * and global overview flags.
2448 */
2449DECL_FORCE_INLINE(void)
2450iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2451{
2452 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2453 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2454 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2455 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2456 Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2457 Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2458 Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2459#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2460 Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2461#endif
2462
2463#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2464 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2465 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2466#else
2467 RT_NOREF(off);
2468#endif
2469
2470 uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2471 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2472 if (!fGstRegShadowsNew)
2473 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2474 pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2475}
2476
2477
2478#if 0 /* unused */
2479/**
2480 * Clear any guest register shadow claim for @a enmGstReg.
2481 */
2482DECL_FORCE_INLINE(void)
2483iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2484{
2485 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2486 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2487 {
2488 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2489 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2490 }
2491}
2492#endif
2493
2494
2495/**
2496 * Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2497 * as the new shadow of it.
2498 *
2499 * Unlike the other guest reg shadow helpers, this does the logging for you.
2500 * However, it is the liveness state is not asserted here, the caller must do
2501 * that.
2502 */
2503DECL_FORCE_INLINE(void)
2504iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2505 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2506{
2507 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2508 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2509 {
2510 uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2511 Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2512 if (idxHstRegOld == idxHstRegNew)
2513 return;
2514 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2515 g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2516 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2517 }
2518 else
2519 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2520 g_aGstShadowInfo[enmGstReg].pszName));
2521 iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2522}
2523
2524
2525/**
2526 * Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2527 * to @a idxRegTo.
2528 */
2529DECL_FORCE_INLINE(void)
2530iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2531 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2532{
2533 Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2534 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2535 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2536 == pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2537 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2538 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2539 == pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2540 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2541 == RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2542
2543 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2544 pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2545 if (!fGstRegShadowsFrom)
2546 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2547 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxRegTo);
2548 pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows |= RT_BIT_64(enmGstReg);
2549 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2550#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2551 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2552 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2553#else
2554 RT_NOREF(off);
2555#endif
2556}
2557
2558
2559/**
2560 * Flushes any delayed guest register writes.
2561 *
2562 * This must be called prior to calling CImpl functions and any helpers that use
2563 * the guest state (like raising exceptions) and such.
2564 *
2565 * This optimization has not yet been implemented. The first target would be
2566 * RIP updates, since these are the most common ones.
2567 *
2568 * @note This function does not flush any shadowing information for guest
2569 * registers. This needs to be done by the caller if it wishes to do so.
2570 */
2571DECL_INLINE_THROW(uint32_t)
2572iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
2573 uint64_t fGstSimdShwExcept = 0)
2574{
2575#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2576 uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
2577#else
2578 uint64_t const fWritebackPc = 0;
2579#endif
2580#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2581 uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2582#else
2583 uint64_t const bmGstRegShadowDirty = 0;
2584#endif
2585#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2586 uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
2587 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2588 & ~fGstSimdShwExcept;
2589#else
2590 uint64_t const bmGstSimdRegShadowDirty = 0;
2591#endif
2592 if (bmGstRegShadowDirty | bmGstSimdRegShadowDirty | fWritebackPc)
2593 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2594
2595 return off;
2596}
2597
2598
2599
2600/*********************************************************************************************************************************
2601* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2602*********************************************************************************************************************************/
2603
2604#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2605
2606DECL_FORCE_INLINE(uint8_t)
2607iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2608 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2609{
2610 pReNative->Core.bmHstSimdRegs |= RT_BIT_32(idxSimdReg);
2611
2612 pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2613 pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2614 pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2615 return idxSimdReg;
2616}
2617
2618
2619/**
2620 * Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2621 * SIMD register @a enmGstSimdReg.
2622 *
2623 * ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2624 * host register before calling.
2625 */
2626DECL_FORCE_INLINE(void)
2627iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2628 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2629{
2630 Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2631 Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2632 Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2633
2634 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2635 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2636 pReNative->Core.bmGstSimdRegShadows |= RT_BIT_64(enmGstSimdReg);
2637 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxHstSimdReg);
2638#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2639 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2640 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2641#else
2642 RT_NOREF(off);
2643#endif
2644}
2645
2646
2647/**
2648 * Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2649 * to @a idxSimdRegTo.
2650 */
2651DECL_FORCE_INLINE(void)
2652iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2653 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2654{
2655 Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2656 Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2657 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2658 == pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2659 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2660 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2661 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2662 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2663 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2664 Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2665 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2666
2667 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2668 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2669 if (!fGstRegShadowsFrom)
2670 {
2671 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2672 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2673 }
2674 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxSimdRegTo);
2675 pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2676 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2677#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2678 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2679 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2680#else
2681 RT_NOREF(off);
2682#endif
2683}
2684
2685
2686/**
2687 * Clear any guest register shadow claims from @a idxHstSimdReg.
2688 *
2689 * The register does not need to be shadowing any guest registers.
2690 */
2691DECL_FORCE_INLINE(void)
2692iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2693{
2694 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2695 == pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2696 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2697 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2698 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2699 Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2700 && !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2701
2702#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2703 uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2704 if (fGstRegs)
2705 {
2706 Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2707 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2708 while (fGstRegs)
2709 {
2710 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2711 fGstRegs &= ~RT_BIT_64(iGstReg);
2712 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2713 }
2714 }
2715#else
2716 RT_NOREF(off);
2717#endif
2718
2719 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2720 pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2721 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2722 pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2723}
2724
2725#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2726
2727
2728#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2729/**
2730 * Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2731 */
2732DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2733{
2734 if (pReNative->Core.offPc)
2735 return iemNativeEmitPcWritebackSlow(pReNative, off);
2736 return off;
2737}
2738#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2739
2740
2741#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2742/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2743 * it saves us the trouble of a hidden parameter on MSC/amd64. */
2744# ifdef RT_ARCH_AMD64
2745extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2746# elif defined(RT_ARCH_ARM64)
2747extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2748# endif
2749#endif
2750
2751#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2752extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2753#endif
2754
2755#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2756
2757/** @} */
2758
2759#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2760
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette