VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106078

Last change on this file since 106078 was 106078, checked in by vboxsync, 5 months ago

VMM/IEM: Liveness work for bugref:10720. bugref:10372
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 133.2 KB
Line 
1/* $Id: IEMN8veRecompiler.h 106078 2024-09-17 19:41:52Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43 * Enables generating internal debug info for better TB disassembly dumping. */
44#if defined(DEBUG) || defined(DOXYGEN_RUNNING) || 0
45# define IEMNATIVE_WITH_TB_DEBUG_INFO
46#endif
47
48/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49 * Enables liveness analysis. */
50#if 1 || defined(DOXYGEN_RUNNING)
51# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
53#endif
54
55/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56 * Enables skipping EFLAGS calculations/updating based on liveness info. */
57#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(DOXYGEN_RUNNING)
58# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59#endif
60
61/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62 * Enables strict consistency checks around EFLAGS skipping.
63 * @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65# ifdef VBOX_STRICT
66# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67# endif
68#elif defined(DOXYGEN_RUNNING)
69# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70#endif
71
72#ifdef VBOX_WITH_STATISTICS
73/** Always count instructions for now. */
74# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75#endif
76
77/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78 * Enables having only a single prologue for native TBs. */
79#if 1 || defined(DOXYGEN_RUNNING)
80# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81#endif
82
83/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84 * Enable this to use common epilogue and tail code for all TBs in a chunk. */
85#if 1 || defined(DOXYGEN_RUNNING)
86# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87#endif
88
89
90/** @name Stack Frame Layout
91 *
92 * @{ */
93/** The size of the area for stack variables and spills and stuff.
94 * @note This limit is duplicated in the python script(s). We add 0x40 for
95 * alignment padding. */
96#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101#ifdef RT_ARCH_AMD64
102/** An stack alignment adjustment (between non-volatile register pushes and
103 * the stack variable area, so the latter better aligned). */
104# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106/** Number of stack arguments slots for calls made from the frame. */
107# ifdef RT_OS_WINDOWS
108# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109# else
110# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111# endif
112/** Number of any shadow arguments (spill area) for calls we make. */
113# ifdef RT_OS_WINDOWS
114# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115# else
116# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117# endif
118
119/** Frame pointer (RBP) relative offset of the last push. */
120# ifdef RT_OS_WINDOWS
121# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122# else
123# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124# endif
125/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126 * address for it). */
127# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132# ifdef RT_OS_WINDOWS
133/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137# endif
138
139# ifdef RT_OS_WINDOWS
140/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148# endif
149
150#elif RT_ARCH_ARM64
151/** No alignment padding needed for arm64. */
152# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153/** No stack argument slots, got 8 registers for arguments will suffice. */
154# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155/** There are no argument spill area. */
156# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158/** Number of saved registers at the top of our stack frame.
159 * This includes the return address and old frame pointer, so x19 thru x30. */
160# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164/** Frame pointer (BP) relative offset of the last push. */
165# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168 * address for it). */
169# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171#else
172# error "port me"
173#endif
174/** @} */
175
176
177/** @name Fixed Register Allocation(s)
178 * @{ */
179/** @def IEMNATIVE_REG_FIXED_PVMCPU
180 * The number of the register holding the pVCpu pointer. */
181/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183 * @note This not available on AMD64, only ARM64. */
184/** @def IEMNATIVE_REG_FIXED_TMP0
185 * Dedicated temporary register.
186 * @note This has extremely short lifetime, must be used with great care to make
187 * sure any calling code or code being called is making use of it.
188 * It will definitely not survive a call or anything of that nature.
189 * @todo replace this by a register allocator and content tracker. */
190/** @def IEMNATIVE_REG_FIXED_MASK
191 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
192 * architecture. */
193#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
196 * architecture. */
197/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
198 * Dedicated temporary SIMD register. */
199#endif
200#if defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
201# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
202# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
203# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
204# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
205# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
206# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
207# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
208# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
209# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
210 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
211# else
212# define IEMNATIVE_REG_FIXED_MASK_ADD 0
213# endif
214# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
215 | RT_BIT_32(ARMV8_A64_REG_LR) \
216 | RT_BIT_32(ARMV8_A64_REG_BP) \
217 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
218 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
219 | RT_BIT_32(ARMV8_A64_REG_X18) \
220 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
221 | IEMNATIVE_REG_FIXED_MASK_ADD)
222
223# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
224# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
225# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
226# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
227# else
228/** @note
229 * ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
230 * support emulating 256-bit registers we pair two real registers statically to
231 * one virtual for now, leaving us with only 16 256-bit registers. We always
232 * pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
233 * the register allocator assumes that it will be always free when the lower is
234 * picked.
235 *
236 * Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
237 * touch them in order to avoid having to save and restore them in the
238 * prologue/epilogue.
239 */
240# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
241 | RT_BIT_32(ARMV8_A64_REG_Q31) \
242 | RT_BIT_32(ARMV8_A64_REG_Q30) \
243 | RT_BIT_32(ARMV8_A64_REG_Q29) \
244 | RT_BIT_32(ARMV8_A64_REG_Q27) \
245 | RT_BIT_32(ARMV8_A64_REG_Q25) \
246 | RT_BIT_32(ARMV8_A64_REG_Q23) \
247 | RT_BIT_32(ARMV8_A64_REG_Q21) \
248 | RT_BIT_32(ARMV8_A64_REG_Q19) \
249 | RT_BIT_32(ARMV8_A64_REG_Q17) \
250 | RT_BIT_32(ARMV8_A64_REG_Q15) \
251 | RT_BIT_32(ARMV8_A64_REG_Q13) \
252 | RT_BIT_32(ARMV8_A64_REG_Q11) \
253 | RT_BIT_32(ARMV8_A64_REG_Q9) \
254 | RT_BIT_32(ARMV8_A64_REG_Q7) \
255 | RT_BIT_32(ARMV8_A64_REG_Q5) \
256 | RT_BIT_32(ARMV8_A64_REG_Q3) \
257 | RT_BIT_32(ARMV8_A64_REG_Q1))
258# endif
259# endif
260
261#elif defined(RT_ARCH_AMD64)
262# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
263# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
264# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
265# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
266 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
267 | RT_BIT_32(X86_GREG_xSP) \
268 | RT_BIT_32(X86_GREG_xBP) )
269
270# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
271# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
272# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
273# ifndef _MSC_VER
274# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275# endif
276# endif
277# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
278# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
279# else
280/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
281# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
282 | RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
283# endif
284# endif
285
286#else
287# error "port me"
288#endif
289/** @} */
290
291/** @name Call related registers.
292 * @{ */
293/** @def IEMNATIVE_CALL_RET_GREG
294 * The return value register. */
295/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
296 * Number of arguments in registers. */
297/** @def IEMNATIVE_CALL_ARG0_GREG
298 * The general purpose register carrying argument \#0. */
299/** @def IEMNATIVE_CALL_ARG1_GREG
300 * The general purpose register carrying argument \#1. */
301/** @def IEMNATIVE_CALL_ARG2_GREG
302 * The general purpose register carrying argument \#2. */
303/** @def IEMNATIVE_CALL_ARG3_GREG
304 * The general purpose register carrying argument \#3. */
305/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
306 * Mask of registers the callee will not save and may trash. */
307#ifdef RT_ARCH_AMD64
308# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
309
310# ifdef RT_OS_WINDOWS
311# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
312# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
313# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
314# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
315# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
316# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
317 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
318 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
319 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
320# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
321 | RT_BIT_32(X86_GREG_xCX) \
322 | RT_BIT_32(X86_GREG_xDX) \
323 | RT_BIT_32(X86_GREG_x8) \
324 | RT_BIT_32(X86_GREG_x9) \
325 | RT_BIT_32(X86_GREG_x10) \
326 | RT_BIT_32(X86_GREG_x11) )
327# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
328/* xmm0 - xmm5 are marked as volatile. */
329# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
330# endif
331
332# else /* !RT_OS_WINDOWS */
333# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
334# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
335# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
336# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
337# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
338# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
339# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
340# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
341 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
342 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
343 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
344 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
345 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
346# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
347 | RT_BIT_32(X86_GREG_xCX) \
348 | RT_BIT_32(X86_GREG_xDX) \
349 | RT_BIT_32(X86_GREG_xDI) \
350 | RT_BIT_32(X86_GREG_xSI) \
351 | RT_BIT_32(X86_GREG_x8) \
352 | RT_BIT_32(X86_GREG_x9) \
353 | RT_BIT_32(X86_GREG_x10) \
354 | RT_BIT_32(X86_GREG_x11) )
355# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
356/* xmm0 - xmm15 are marked as volatile. */
357# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
358# endif
359# endif /* !RT_OS_WINDOWS */
360
361#elif defined(RT_ARCH_ARM64)
362# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
363# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
364# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
365# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
366# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
367# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
368# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
369# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
370# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
371# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
372# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373 | RT_BIT_32(ARMV8_A64_REG_X1) \
374 | RT_BIT_32(ARMV8_A64_REG_X2) \
375 | RT_BIT_32(ARMV8_A64_REG_X3) \
376 | RT_BIT_32(ARMV8_A64_REG_X4) \
377 | RT_BIT_32(ARMV8_A64_REG_X5) \
378 | RT_BIT_32(ARMV8_A64_REG_X6) \
379 | RT_BIT_32(ARMV8_A64_REG_X7) )
380# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
381 | RT_BIT_32(ARMV8_A64_REG_X1) \
382 | RT_BIT_32(ARMV8_A64_REG_X2) \
383 | RT_BIT_32(ARMV8_A64_REG_X3) \
384 | RT_BIT_32(ARMV8_A64_REG_X4) \
385 | RT_BIT_32(ARMV8_A64_REG_X5) \
386 | RT_BIT_32(ARMV8_A64_REG_X6) \
387 | RT_BIT_32(ARMV8_A64_REG_X7) \
388 | RT_BIT_32(ARMV8_A64_REG_X8) \
389 | RT_BIT_32(ARMV8_A64_REG_X9) \
390 | RT_BIT_32(ARMV8_A64_REG_X10) \
391 | RT_BIT_32(ARMV8_A64_REG_X11) \
392 | RT_BIT_32(ARMV8_A64_REG_X12) \
393 | RT_BIT_32(ARMV8_A64_REG_X13) \
394 | RT_BIT_32(ARMV8_A64_REG_X14) \
395 | RT_BIT_32(ARMV8_A64_REG_X15) \
396 | RT_BIT_32(ARMV8_A64_REG_X16) \
397 | RT_BIT_32(ARMV8_A64_REG_X17) )
398# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
399/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
400 * so to simplify our life a bit we just mark everything as volatile. */
401# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
402# endif
403
404#endif
405
406/** This is the maximum argument count we'll ever be needing. */
407#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
408#ifdef RT_OS_WINDOWS
409# ifdef VBOXSTRICTRC_STRICT_ENABLED
410# undef IEMNATIVE_CALL_MAX_ARG_COUNT
411# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
412# endif
413#endif
414
415/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
416 * Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
417 * IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
418#ifdef IEMNATIVE_REG_FIXED_TMP0
419# ifdef IEMNATIVE_REG_FIXED_TMP1
420# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
421 & ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
422 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
423# else
424# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
425# endif
426#else
427# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
428#endif
429/** @} */
430
431
432/** @def IEMNATIVE_HST_GREG_COUNT
433 * Number of host general purpose registers we tracker. */
434/** @def IEMNATIVE_HST_GREG_MASK
435 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
436 * inverted register masks and such to get down to a correct set of regs. */
437#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
438/** @def IEMNATIVE_HST_SIMD_REG_COUNT
439 * Number of host SIMD registers we track. */
440/** @def IEMNATIVE_HST_SIMD_REG_MASK
441 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
442 * inverted register masks and such to get down to a correct set of regs. */
443#endif
444#ifdef RT_ARCH_AMD64
445# define IEMNATIVE_HST_GREG_COUNT 16
446# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
447
448# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
449# define IEMNATIVE_HST_SIMD_REG_COUNT 16
450# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
451# endif
452
453#elif defined(RT_ARCH_ARM64)
454# define IEMNATIVE_HST_GREG_COUNT 32
455# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
456
457# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
458# define IEMNATIVE_HST_SIMD_REG_COUNT 32
459# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
460# endif
461
462#else
463# error "Port me!"
464#endif
465
466
467#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
468
469
470/** Native code generator label types. */
471typedef enum
472{
473 kIemNativeLabelType_Invalid = 0,
474 /*
475 * Labels w/o data, only once instance per TB - aka exit reasons.
476 *
477 * Note! Jumps to these requires instructions that are capable of spanning
478 * the max TB length.
479 */
480 /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
481 kIemNativeLabelType_RaiseDe, /**< Raise (throw) X86_XCPT_DE (00h). */
482 kIemNativeLabelType_RaiseUd, /**< Raise (throw) X86_XCPT_UD (06h). */
483 kIemNativeLabelType_RaiseSseRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
484 kIemNativeLabelType_RaiseAvxRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
485 kIemNativeLabelType_RaiseSseAvxFpRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
486 kIemNativeLabelType_RaiseNm, /**< Raise (throw) X86_XCPT_NM (07h). */
487 kIemNativeLabelType_RaiseGp0, /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
488 kIemNativeLabelType_RaiseMf, /**< Raise (throw) X86_XCPT_MF (10h). */
489 kIemNativeLabelType_RaiseXf, /**< Raise (throw) X86_XCPT_XF (13h). */
490 kIemNativeLabelType_ObsoleteTb,
491 kIemNativeLabelType_NeedCsLimChecking,
492 kIemNativeLabelType_CheckBranchMiss,
493 kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
494 /* Manually defined labels. */
495 kIemNativeLabelType_ReturnBreak,
496 kIemNativeLabelType_ReturnBreakFF,
497 kIemNativeLabelType_ReturnBreakViaLookup,
498 kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
499 kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
500 kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
501 kIemNativeLabelType_ReturnWithFlags,
502 kIemNativeLabelType_NonZeroRetOrPassUp,
503#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
504 kIemNativeLabelType_ReturnSuccess, /**< Sets eax/w0 to zero and returns. */
505#else
506 kIemNativeLabelType_Return,
507#endif
508 /** The last fixup for branches that can span almost the whole TB length.
509 * @note Whether kIemNativeLabelType_Return needs to be one of these is
510 * a bit questionable, since nobody jumps to it except other tail code. */
511#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
512 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
513#else
514 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
515#endif
516 /** The last fixup for branches that exits the TB. */
517#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
518 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_ReturnSuccess,
519#else
520 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
521#endif
522
523 /** Loop-jump target. */
524 kIemNativeLabelType_LoopJumpTarget,
525
526 /*
527 * Labels with data, potentially multiple instances per TB:
528 *
529 * These are localized labels, so no fixed jump type restrictions here.
530 */
531 kIemNativeLabelType_FirstWithMultipleInstances,
532 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
533 kIemNativeLabelType_Else,
534 kIemNativeLabelType_Endif,
535 kIemNativeLabelType_CheckIrq,
536 kIemNativeLabelType_TlbLookup,
537 kIemNativeLabelType_TlbMiss,
538 kIemNativeLabelType_TlbDone,
539 kIemNativeLabelType_End
540} IEMNATIVELABELTYPE;
541
542#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
543 ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
544
545
546/** Native code generator label definition. */
547typedef struct IEMNATIVELABEL
548{
549 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
550 * the epilog. */
551 uint32_t off;
552 /** The type of label (IEMNATIVELABELTYPE). */
553 uint16_t enmType;
554 /** Additional label data, type specific. */
555 uint16_t uData;
556} IEMNATIVELABEL;
557/** Pointer to a label. */
558typedef IEMNATIVELABEL *PIEMNATIVELABEL;
559
560
561/** Native code generator fixup types. */
562typedef enum
563{
564 kIemNativeFixupType_Invalid = 0,
565#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
566 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
567 kIemNativeFixupType_Rel32,
568#elif defined(RT_ARCH_ARM64)
569 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
570 kIemNativeFixupType_RelImm26At0,
571 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
572 kIemNativeFixupType_RelImm19At5,
573 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
574 kIemNativeFixupType_RelImm14At5,
575#endif
576 kIemNativeFixupType_End
577} IEMNATIVEFIXUPTYPE;
578
579/** Native code generator fixup. */
580typedef struct IEMNATIVEFIXUP
581{
582 /** Code offset of the fixup location. */
583 uint32_t off;
584 /** The IEMNATIVELABEL this is a fixup for. */
585 uint16_t idxLabel;
586 /** The fixup type (IEMNATIVEFIXUPTYPE). */
587 uint8_t enmType;
588 /** Addend or other data. */
589 int8_t offAddend;
590} IEMNATIVEFIXUP;
591/** Pointer to a native code generator fixup. */
592typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
593
594#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
595
596/** Native code generator fixup to per chunk TB tail code. */
597typedef struct IEMNATIVEEXITFIXUP
598{
599 /** Code offset of the fixup location. */
600 uint32_t off;
601 /** The exit reason. */
602 IEMNATIVELABELTYPE enmExitReason;
603} IEMNATIVEEXITFIXUP;
604/** Pointer to a native code generator TB exit fixup. */
605typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
606
607/**
608 * Per executable memory chunk context with addresses for common code.
609 */
610typedef struct IEMNATIVEPERCHUNKCTX
611{
612 /** Pointers to the exit labels */
613 PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
614} IEMNATIVEPERCHUNKCTX;
615/** Pointer to per-chunk recompiler context. */
616typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
617/** Pointer to const per-chunk recompiler context. */
618typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
619
620#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
621
622
623/**
624 * One bit of the state.
625 *
626 * Each register state takes up two bits. We keep the two bits in two separate
627 * 64-bit words to simplify applying them to the guest shadow register mask in
628 * the register allocator.
629 */
630typedef union IEMLIVENESSBIT
631{
632 uint64_t bm64;
633 RT_GCC_EXTENSION struct
634 { /* bit no */
635 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
636 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
637 uint64_t fCr0 : 1; /**< 0x11 / 17: */
638 uint64_t fFcw : 1; /**< 0x12 / 18: */
639 uint64_t fFsw : 1; /**< 0x13 / 19: */
640 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
641 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
642 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
643 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
644 uint64_t fCr4 : 1; /**< 0x2c / 44: */
645 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
646 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
647 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
648 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
649 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
650 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
651 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
652 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
653 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
654 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
655 };
656} IEMLIVENESSBIT;
657AssertCompileSize(IEMLIVENESSBIT, 8);
658
659#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
660#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
661#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
662#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
663#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
664#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
665#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
666
667
668/**
669 * A liveness state entry.
670 *
671 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
672 * Once we add a SSE register shadowing, we'll add another 64-bit element for
673 * that.
674 */
675typedef union IEMLIVENESSENTRY
676{
677#ifndef IEMLIVENESS_EXTENDED_LAYOUT
678 uint64_t bm64[16 / 8];
679 uint16_t bm32[16 / 4];
680 uint16_t bm16[16 / 2];
681 uint8_t bm8[ 16 / 1];
682 IEMLIVENESSBIT aBits[2];
683#else
684 uint64_t bm64[32 / 8];
685 uint16_t bm32[32 / 4];
686 uint16_t bm16[32 / 2];
687 uint8_t bm8[ 32 / 1];
688 IEMLIVENESSBIT aBits[4];
689#endif
690 RT_GCC_EXTENSION struct
691 {
692 /** Bit \#0 of the register states. */
693 IEMLIVENESSBIT Bit0;
694 /** Bit \#1 of the register states. */
695 IEMLIVENESSBIT Bit1;
696#ifdef IEMLIVENESS_EXTENDED_LAYOUT
697 /** Bit \#2 of the register states. */
698 IEMLIVENESSBIT Bit2;
699 /** Bit \#3 of the register states. */
700 IEMLIVENESSBIT Bit3;
701#endif
702 };
703} IEMLIVENESSENTRY;
704#ifndef IEMLIVENESS_EXTENDED_LAYOUT
705AssertCompileSize(IEMLIVENESSENTRY, 16);
706#else
707AssertCompileSize(IEMLIVENESSENTRY, 32);
708#endif
709/** Pointer to a liveness state entry. */
710typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
711/** Pointer to a const liveness state entry. */
712typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
713
714/** @name 64-bit value masks for IEMLIVENESSENTRY.
715 * @{ */ /* 0xzzzzyyyyxxxxwwww */
716#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
717
718#ifndef IEMLIVENESS_EXTENDED_LAYOUT
719# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
720# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
721
722# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
723# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
724#endif
725
726#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
727#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
728
729#ifndef IEMLIVENESS_EXTENDED_LAYOUT
730# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
731# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
732#endif
733/** @} */
734
735
736/** @name The liveness state for a register.
737 *
738 * The state values have been picked to with state accumulation in mind (what
739 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
740 * performance critical work done with the values.
741 *
742 * This is a compressed state that only requires 2 bits per register.
743 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
744 * 1. the incoming state from the following call,
745 * 2. the outgoing state for this call,
746 * 3. mask of the entries set in the 2nd.
747 *
748 * The mask entry (3rd one above) will be used both when updating the outgoing
749 * state and when merging in incoming state for registers not touched by the
750 * current call.
751 *
752 * @{ */
753#ifndef IEMLIVENESS_EXTENDED_LAYOUT
754/** The register will be clobbered and the current value thrown away.
755 *
756 * When this is applied to the state (2) we'll simply be AND'ing it with the
757 * (old) mask (3) and adding the register to the mask. This way we'll
758 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
759 * IEMLIVENESS_STATE_INPUT states. */
760# define IEMLIVENESS_STATE_CLOBBERED 0
761/** The register is unused in the remainder of the TB.
762 *
763 * This is an initial state and can not be set by any of the
764 * iemNativeLivenessFunc_xxxx callbacks. */
765# define IEMLIVENESS_STATE_UNUSED 1
766/** The register value is required in a potential call or exception.
767 *
768 * This means that the register value must be calculated and is best written to
769 * the state, but that any shadowing registers can be flushed thereafter as it's
770 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
771 *
772 * It is typically applied across the board, but we preserve incoming
773 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
774 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
775 * 1. r0 = old & ~mask;
776 * 2. r0 = t1 & (t1 >> 1);
777 * 3. state |= r0 | 0b10;
778 * 4. mask = ~0;
779 */
780# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
781/** The register value is used as input.
782 *
783 * This means that the register value must be calculated and it is best to keep
784 * it in a register. It does not need to be writtent out as such. This is the
785 * highest priority state.
786 *
787 * Whether the call modifies the register or not isn't relevant to earlier
788 * calls, so that's not recorded.
789 *
790 * When applying this state we just or in the value in the outgoing state and
791 * mask. */
792# define IEMLIVENESS_STATE_INPUT 3
793/** Mask of the state bits. */
794# define IEMLIVENESS_STATE_MASK 3
795/** The number of bits per state. */
796# define IEMLIVENESS_STATE_BIT_COUNT 2
797
798/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
799# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
800/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
801# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
802/** Check if a register clobbering is expected given the (previous) liveness state.
803 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
804 * include INPUT if the register is used in more than one place. */
805# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
806
807/** Check if all status flags are going to be clobbered and doesn't need
808 * calculating in the current step.
809 * @param a_pCurEntry The current liveness entry. */
810# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
811 ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
812
813#else /* IEMLIVENESS_EXTENDED_LAYOUT */
814/** The register is not used any more. */
815# define IEMLIVENESS_STATE_UNUSED 0
816/** Flag: The register is required in a potential call or/and exception. */
817# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
818# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
819/** Flag: The register is read. */
820# define IEMLIVENESS_STATE_READ 2
821# define IEMLIVENESS_BIT_READ 1
822/** Flag: The register is written. */
823# define IEMLIVENESS_STATE_WRITE 4
824# define IEMLIVENESS_BIT_WRITE 2
825/** Flag: Unconditional call. */
826# define IEMLIVENESS_STATE_CALL 8
827# define IEMLIVENESS_BIT_CALL 3
828
829# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
830 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
831# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
832# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
833
834# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
835 ( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
836 && !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
837 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
838 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
839 & IEMLIVENESSBIT_STATUS_EFL_MASK) )
840
841#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
842/** @} */
843
844/** @name Liveness helpers for builtin functions and similar.
845 *
846 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
847 * own set of manipulator macros for those.
848 *
849 * @{ */
850/** Initializing the state as all unused. */
851#ifndef IEMLIVENESS_EXTENDED_LAYOUT
852# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
853 do { \
854 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
855 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
856 } while (0)
857#else
858# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
859 do { \
860 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
861 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
862 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
863 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
864 } while (0)
865#endif
866
867/** Initializing the outgoing state with a potential xcpt or call state.
868 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
869#ifndef IEMLIVENESS_EXTENDED_LAYOUT
870# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
871 do { \
872 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
873 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
874 } while (0)
875#else
876# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
877 do { \
878 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
879 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
880 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
881 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
882 } while (0)
883#endif
884
885/** Initializing the outgoing state with an unconditional call state.
886 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
887#ifndef IEMLIVENESS_EXTENDED_LAYOUT
888# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
889 do { \
890 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
891 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
892 } while (0)
893#else
894# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
895 do { \
896 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
897 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
898 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
899 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
900 } while (0)
901#endif
902
903/** Initializing the outgoing state with an unconditional call state as well as
904 * an potential call/exception preceeding it.
905 * This should only be used alone, really. */
906#ifndef IEMLIVENESS_EXTENDED_LAYOUT
907# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
908 do { \
909 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
910 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
911 } while (0)
912#else
913# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
914 do { \
915 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
916 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
917 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
918 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
919 } while (0)
920#endif
921
922/** Adds a segment base register as input to the outgoing state. */
923#ifndef IEMLIVENESS_EXTENDED_LAYOUT
924# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
925 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
926 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
927 } while (0)
928#else
929# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
930 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
931 } while (0)
932#endif
933
934/** Adds a segment attribute register as input to the outgoing state. */
935#ifndef IEMLIVENESS_EXTENDED_LAYOUT
936# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
937 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
938 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
939 } while (0)
940#else
941# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
942 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
943 } while (0)
944#endif
945
946/** Adds a segment limit register as input to the outgoing state. */
947#ifndef IEMLIVENESS_EXTENDED_LAYOUT
948# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
949 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
950 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
951 } while (0)
952#else
953# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
954 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
955 } while (0)
956#endif
957
958/** Adds a segment limit register as input to the outgoing state. */
959#ifndef IEMLIVENESS_EXTENDED_LAYOUT
960# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
961 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
962 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
963 } while (0)
964#else
965# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
966 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
967 } while (0)
968#endif
969/** @} */
970
971/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
972 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
973 * calculated and up to date. This is to double check that we haven't skipped
974 * EFLAGS calculations when we actually need them. NOP in non-strict builds.
975 * @note has to be placed in
976 */
977#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
978# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
979 do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
980#else
981# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
982#endif
983
984
985/**
986 * Guest registers that can be shadowed in GPRs.
987 *
988 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
989 * must be placed last, as the liveness state tracks it as 7 subcomponents and
990 * we don't want to waste space here.
991 *
992 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
993 * friends as well as IEMAllN8veLiveness.cpp.
994 */
995typedef enum IEMNATIVEGSTREG : uint8_t
996{
997 kIemNativeGstReg_GprFirst = 0,
998 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
999 kIemNativeGstReg_Pc,
1000 kIemNativeGstReg_Cr0,
1001 kIemNativeGstReg_FpuFcw,
1002 kIemNativeGstReg_FpuFsw,
1003 kIemNativeGstReg_SegBaseFirst,
1004 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1005 kIemNativeGstReg_SegAttribFirst,
1006 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1007 kIemNativeGstReg_SegLimitFirst,
1008 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1009 kIemNativeGstReg_SegSelFirst,
1010 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1011 kIemNativeGstReg_Cr4,
1012 kIemNativeGstReg_Xcr0,
1013 kIemNativeGstReg_MxCsr,
1014 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
1015 kIemNativeGstReg_End
1016} IEMNATIVEGSTREG;
1017AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1018AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1019
1020/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1021 * @{ */
1022#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1023#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1024#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1025#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1026#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1027/** @} */
1028
1029#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1030
1031/**
1032 * Guest registers that can be shadowed in host SIMD registers.
1033 *
1034 * @todo r=aeichner Liveness tracking
1035 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1036 */
1037typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1038{
1039 kIemNativeGstSimdReg_SimdRegFirst = 0,
1040 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1041 kIemNativeGstSimdReg_End
1042} IEMNATIVEGSTSIMDREG;
1043
1044/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1045 * @{ */
1046#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1047/** @} */
1048
1049/**
1050 * The Load/store size for a SIMD guest register.
1051 */
1052typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1053{
1054 /** Invalid size. */
1055 kIemNativeGstSimdRegLdStSz_Invalid = 0,
1056 /** Loads the low 128-bit of a guest SIMD register. */
1057 kIemNativeGstSimdRegLdStSz_Low128,
1058 /** Loads the high 128-bit of a guest SIMD register. */
1059 kIemNativeGstSimdRegLdStSz_High128,
1060 /** Loads the whole 256-bits of a guest SIMD register. */
1061 kIemNativeGstSimdRegLdStSz_256,
1062 /** End value. */
1063 kIemNativeGstSimdRegLdStSz_End
1064} IEMNATIVEGSTSIMDREGLDSTSZ;
1065
1066#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1067
1068/**
1069 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
1070 */
1071typedef enum IEMNATIVEGSTREGUSE
1072{
1073 /** The usage is read-only, the register holding the guest register
1074 * shadow copy will not be modified by the caller. */
1075 kIemNativeGstRegUse_ReadOnly = 0,
1076 /** The caller will update the guest register (think: PC += cbInstr).
1077 * The guest shadow copy will follow the returned register. */
1078 kIemNativeGstRegUse_ForUpdate,
1079 /** The call will put an entirely new value in the guest register, so
1080 * if new register is allocate it will be returned uninitialized. */
1081 kIemNativeGstRegUse_ForFullWrite,
1082 /** The caller will use the guest register value as input in a calculation
1083 * and the host register will be modified.
1084 * This means that the returned host register will not be marked as a shadow
1085 * copy of the guest register. */
1086 kIemNativeGstRegUse_Calculation
1087} IEMNATIVEGSTREGUSE;
1088
1089/**
1090 * Guest registers (classes) that can be referenced.
1091 */
1092typedef enum IEMNATIVEGSTREGREF : uint8_t
1093{
1094 kIemNativeGstRegRef_Invalid = 0,
1095 kIemNativeGstRegRef_Gpr,
1096 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
1097 kIemNativeGstRegRef_EFlags,
1098 kIemNativeGstRegRef_MxCsr,
1099 kIemNativeGstRegRef_FpuReg,
1100 kIemNativeGstRegRef_MReg,
1101 kIemNativeGstRegRef_XReg,
1102 kIemNativeGstRegRef_X87,
1103 kIemNativeGstRegRef_XState,
1104 //kIemNativeGstRegRef_YReg, - doesn't work.
1105 kIemNativeGstRegRef_End
1106} IEMNATIVEGSTREGREF;
1107
1108
1109/** Variable kinds. */
1110typedef enum IEMNATIVEVARKIND : uint8_t
1111{
1112 /** Customary invalid zero value. */
1113 kIemNativeVarKind_Invalid = 0,
1114 /** This is either in a register or on the stack. */
1115 kIemNativeVarKind_Stack,
1116 /** Immediate value - loaded into register when needed, or can live on the
1117 * stack if referenced (in theory). */
1118 kIemNativeVarKind_Immediate,
1119 /** Variable reference - loaded into register when needed, never stack. */
1120 kIemNativeVarKind_VarRef,
1121 /** Guest register reference - loaded into register when needed, never stack. */
1122 kIemNativeVarKind_GstRegRef,
1123 /** End of valid values. */
1124 kIemNativeVarKind_End
1125} IEMNATIVEVARKIND;
1126
1127
1128/** Variable or argument. */
1129typedef struct IEMNATIVEVAR
1130{
1131 /** The kind of variable. */
1132 IEMNATIVEVARKIND enmKind;
1133 /** The variable size in bytes. */
1134 uint8_t cbVar;
1135 /** The first stack slot (uint64_t), except for immediate and references
1136 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1137 * has a stack slot it has been initialized and has a value. Unused variables
1138 * has neither a stack slot nor a host register assignment. */
1139 uint8_t idxStackSlot;
1140 /** The host register allocated for the variable, UINT8_MAX if not. */
1141 uint8_t idxReg;
1142 /** The argument number if argument, UINT8_MAX if regular variable. */
1143 uint8_t uArgNo;
1144 /** If referenced, the index (unpacked) of the variable referencing this one,
1145 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1146 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1147 uint8_t idxReferrerVar;
1148 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1149 * @todo not sure what this really is for... */
1150 IEMNATIVEGSTREG enmGstReg;
1151#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1152 /** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1153 * only valid when idxReg is not UINT8_MAX. */
1154 bool fSimdReg : 1;
1155 /** Set if the registered is currently used exclusively, false if the
1156 * variable is idle and the register can be grabbed. */
1157 bool fRegAcquired : 1;
1158#else
1159 /** Set if the registered is currently used exclusively, false if the
1160 * variable is idle and the register can be grabbed. */
1161 bool fRegAcquired;
1162#endif
1163
1164 union
1165 {
1166 /** kIemNativeVarKind_Immediate: The immediate value. */
1167 uint64_t uValue;
1168 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1169 uint8_t idxRefVar;
1170 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1171 struct
1172 {
1173 /** The class of register. */
1174 IEMNATIVEGSTREGREF enmClass;
1175 /** Index within the class. */
1176 uint8_t idx;
1177 } GstRegRef;
1178 } u;
1179} IEMNATIVEVAR;
1180/** Pointer to a variable or argument. */
1181typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1182/** Pointer to a const variable or argument. */
1183typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1184
1185/** What is being kept in a host register. */
1186typedef enum IEMNATIVEWHAT : uint8_t
1187{
1188 /** The traditional invalid zero value. */
1189 kIemNativeWhat_Invalid = 0,
1190 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1191 kIemNativeWhat_Var,
1192 /** Temporary register, this is typically freed when a MC completes. */
1193 kIemNativeWhat_Tmp,
1194 /** Call argument w/o a variable mapping. This is free (via
1195 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1196 kIemNativeWhat_Arg,
1197 /** Return status code.
1198 * @todo not sure if we need this... */
1199 kIemNativeWhat_rc,
1200 /** The fixed pVCpu (PVMCPUCC) register.
1201 * @todo consider offsetting this on amd64 to use negative offsets to access
1202 * more members using 8-byte disp. */
1203 kIemNativeWhat_pVCpuFixed,
1204 /** The fixed pCtx (PCPUMCTX) register.
1205 * @todo consider offsetting this on amd64 to use negative offsets to access
1206 * more members using 8-byte disp. */
1207 kIemNativeWhat_pCtxFixed,
1208 /** Fixed temporary register. */
1209 kIemNativeWhat_FixedTmp,
1210#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1211 /** Shadow RIP for the delayed RIP updating debugging. */
1212 kIemNativeWhat_PcShadow,
1213#endif
1214 /** Register reserved by the CPU or OS architecture. */
1215 kIemNativeWhat_FixedReserved,
1216 /** End of valid values. */
1217 kIemNativeWhat_End
1218} IEMNATIVEWHAT;
1219
1220/**
1221 * Host general register entry.
1222 *
1223 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1224 *
1225 * @todo Track immediate values in host registers similarlly to how we track the
1226 * guest register shadow copies. For it to be real helpful, though,
1227 * we probably need to know which will be reused and put them into
1228 * non-volatile registers, otherwise it's going to be more or less
1229 * restricted to an instruction or two.
1230 */
1231typedef struct IEMNATIVEHSTREG
1232{
1233 /** Set of guest registers this one shadows.
1234 *
1235 * Using a bitmap here so we can designate the same host register as a copy
1236 * for more than one guest register. This is expected to be useful in
1237 * situations where one value is copied to several registers in a sequence.
1238 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1239 * sequence we'd want to let this register follow to be a copy of and there
1240 * will always be places where we'd be picking the wrong one.
1241 */
1242 uint64_t fGstRegShadows;
1243 /** What is being kept in this register. */
1244 IEMNATIVEWHAT enmWhat;
1245 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1246 uint8_t idxVar;
1247 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1248 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1249 * that scope. */
1250 uint8_t idxStackSlot;
1251 /** Alignment padding. */
1252 uint8_t abAlign[5];
1253} IEMNATIVEHSTREG;
1254
1255
1256#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1257/**
1258 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1259 * halves, on architectures where there is no 256-bit register available this entry will track
1260 * two adjacent 128-bit host registers.
1261 *
1262 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1263 */
1264typedef struct IEMNATIVEHSTSIMDREG
1265{
1266 /** Set of guest registers this one shadows.
1267 *
1268 * Using a bitmap here so we can designate the same host register as a copy
1269 * for more than one guest register. This is expected to be useful in
1270 * situations where one value is copied to several registers in a sequence.
1271 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1272 * sequence we'd want to let this register follow to be a copy of and there
1273 * will always be places where we'd be picking the wrong one.
1274 */
1275 uint64_t fGstRegShadows;
1276 /** What is being kept in this register. */
1277 IEMNATIVEWHAT enmWhat;
1278 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1279 uint8_t idxVar;
1280 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1281 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1282 /** Alignment padding. */
1283 uint8_t abAlign[5];
1284} IEMNATIVEHSTSIMDREG;
1285#endif
1286
1287
1288/**
1289 * Core state for the native recompiler, that is, things that needs careful
1290 * handling when dealing with branches.
1291 */
1292typedef struct IEMNATIVECORESTATE
1293{
1294 /** Allocation bitmap for aHstRegs. */
1295 uint32_t bmHstRegs;
1296
1297 /** Bitmap marking which host register contains guest register shadow copies.
1298 * This is used during register allocation to try preserve copies. */
1299 uint32_t bmHstRegsWithGstShadow;
1300 /** Bitmap marking valid entries in aidxGstRegShadows. */
1301 uint64_t bmGstRegShadows;
1302#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1303 /** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1304 uint64_t bmGstRegShadowDirty;
1305#endif
1306
1307#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1308 /** The current instruction offset in bytes from when the guest program counter
1309 * was updated last. Used for delaying the write to the guest context program counter
1310 * as long as possible. */
1311 int64_t offPc;
1312# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1313 /** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1314 bool fDebugPcInitialized;
1315# endif
1316#endif
1317
1318#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1319 /** Allocation bitmap for aHstSimdRegs. */
1320 uint32_t bmHstSimdRegs;
1321
1322 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1323 * This is used during register allocation to try preserve copies. */
1324 uint32_t bmHstSimdRegsWithGstShadow;
1325 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1326 uint64_t bmGstSimdRegShadows;
1327 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1328 uint64_t bmGstSimdRegShadowDirtyLo128;
1329 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1330 uint64_t bmGstSimdRegShadowDirtyHi128;
1331#endif
1332
1333 union
1334 {
1335 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1336 uint8_t aidxArgVars[8];
1337 /** For more efficient resetting. */
1338 uint64_t u64ArgVars;
1339 };
1340
1341 /** Allocation bitmap for the stack. */
1342 uint32_t bmStack;
1343 /** Allocation bitmap for aVars. */
1344 uint32_t bmVars;
1345
1346 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1347 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1348 * (A shadow copy of a guest register can only be held in a one host register,
1349 * there are no duplicate copies or ambiguities like that). */
1350 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1351#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1352 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1353 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1354 * (A shadow copy of a guest register can only be held in a one host register,
1355 * there are no duplicate copies or ambiguities like that). */
1356 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1357#endif
1358
1359 /** Host register allocation tracking. */
1360 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1361#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1362 /** Host SIMD register allocation tracking. */
1363 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1364#endif
1365
1366 /** Variables and arguments. */
1367 IEMNATIVEVAR aVars[9];
1368} IEMNATIVECORESTATE;
1369/** Pointer to core state. */
1370typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1371/** Pointer to const core state. */
1372typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1373
1374/** @def IEMNATIVE_VAR_IDX_UNPACK
1375 * @returns Index into IEMNATIVECORESTATE::aVars.
1376 * @param a_idxVar Variable index w/ magic (in strict builds).
1377 */
1378/** @def IEMNATIVE_VAR_IDX_PACK
1379 * @returns Variable index w/ magic (in strict builds).
1380 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1381 */
1382#ifdef VBOX_STRICT
1383# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1384# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1385# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1386# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1387# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1388#else
1389# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1390# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1391#endif
1392
1393
1394#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1395/** Clear the dirty state of the given guest SIMD register. */
1396# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1397 do { \
1398 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1399 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1400 } while (0)
1401
1402/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1403# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1404 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1405/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1406# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1407 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1408/** Returns whether the given guest SIMD register is dirty. */
1409# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1410 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1411
1412/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1413# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1414 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1415/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1416# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1417 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1418
1419/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1420# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1421 /** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1422# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1423/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1424# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1425/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1426# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1427# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1428/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1429# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1430/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1431# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1432# endif
1433#endif
1434
1435
1436/**
1437 * Conditional stack entry.
1438 */
1439typedef struct IEMNATIVECOND
1440{
1441 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1442 bool fInElse;
1443 union
1444 {
1445 RT_GCC_EXTENSION struct
1446 {
1447 /** Set if the if-block unconditionally exited the TB. */
1448 bool fIfExitTb;
1449 /** Set if the else-block unconditionally exited the TB. */
1450 bool fElseExitTb;
1451 };
1452 /** Indexed by fInElse. */
1453 bool afExitTb[2];
1454 };
1455 bool afPadding[5];
1456 /** The label for the IEM_MC_ELSE. */
1457 uint32_t idxLabelElse;
1458 /** The label for the IEM_MC_ENDIF. */
1459 uint32_t idxLabelEndIf;
1460 /** The initial state snapshot as the if-block starts executing. */
1461 IEMNATIVECORESTATE InitialState;
1462 /** The state snapshot at the end of the if-block. */
1463 IEMNATIVECORESTATE IfFinalState;
1464} IEMNATIVECOND;
1465/** Pointer to a condition stack entry. */
1466typedef IEMNATIVECOND *PIEMNATIVECOND;
1467
1468
1469/**
1470 * Native recompiler state.
1471 */
1472typedef struct IEMRECOMPILERSTATE
1473{
1474 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1475 * IEMNATIVEINSTR units. */
1476 uint32_t cInstrBufAlloc;
1477#ifdef VBOX_STRICT
1478 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1479 uint32_t offInstrBufChecked;
1480#else
1481 uint32_t uPadding1; /* We don't keep track of the size here... */
1482#endif
1483 /** Fixed temporary code buffer for native recompilation. */
1484 PIEMNATIVEINSTR pInstrBuf;
1485
1486 /** Bitmaps with the label types used. */
1487 uint64_t bmLabelTypes;
1488 /** Actual number of labels in paLabels. */
1489 uint32_t cLabels;
1490 /** Max number of entries allowed in paLabels before reallocating it. */
1491 uint32_t cLabelsAlloc;
1492 /** Labels defined while recompiling (referenced by fixups). */
1493 PIEMNATIVELABEL paLabels;
1494 /** Array with indexes of unique labels (uData always 0). */
1495 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1496
1497 /** Actual number of fixups paFixups. */
1498 uint32_t cFixups;
1499 /** Max number of entries allowed in paFixups before reallocating it. */
1500 uint32_t cFixupsAlloc;
1501 /** Buffer used by the recompiler for recording fixups when generating code. */
1502 PIEMNATIVEFIXUP paFixups;
1503
1504#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1505 /** Actual number of fixups in paTbExitFixups. */
1506 uint32_t cTbExitFixups;
1507 /** Max number of entries allowed in paTbExitFixups before reallocating it. */
1508 uint32_t cTbExitFixupsAlloc;
1509 /** Buffer used by the recompiler for recording fixups when generating code. */
1510 PIEMNATIVEEXITFIXUP paTbExitFixups;
1511#endif
1512
1513#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) || defined(VBOX_WITH_STATISTICS)
1514 /** Statistics: The idxInstr+1 value at the last PC update. */
1515 uint8_t idxInstrPlusOneOfLastPcUpdate;
1516#endif
1517
1518#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1519 /** Number of debug info entries allocated for pDbgInfo. */
1520 uint32_t cDbgInfoAlloc;
1521 /** Debug info. */
1522 PIEMTBDBG pDbgInfo;
1523#endif
1524
1525#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1526 /** The current call index (liveness array and threaded calls in TB). */
1527 uint32_t idxCurCall;
1528 /** Number of liveness entries allocated. */
1529 uint32_t cLivenessEntriesAlloc;
1530 /** Liveness entries for all the calls in the TB begin recompiled.
1531 * The entry for idxCurCall contains the info for what the next call will
1532 * require wrt registers. (Which means the last entry is the initial liveness
1533 * state.) */
1534 PIEMLIVENESSENTRY paLivenessEntries;
1535#endif
1536
1537 /** The translation block being recompiled. */
1538 PCIEMTB pTbOrg;
1539 /** The VMCPU structure of the EMT. */
1540 PVMCPUCC pVCpu;
1541
1542 /** Condition sequence number (for generating unique labels). */
1543 uint16_t uCondSeqNo;
1544 /** Check IRQ sequence number (for generating unique labels). */
1545 uint16_t uCheckIrqSeqNo;
1546 /** TLB load sequence number (for generating unique labels). */
1547 uint16_t uTlbSeqNo;
1548 /** The current condition stack depth (aCondStack). */
1549 uint8_t cCondDepth;
1550
1551 /** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1552 uint8_t cArgsX;
1553 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1554 uint32_t fCImpl;
1555 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1556 uint32_t fMc;
1557 /** The expected IEMCPU::fExec value for the current call/instruction. */
1558 uint32_t fExec;
1559#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1560 /** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1561 * we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1562 *
1563 * This is an optimization because these control registers can only be changed from
1564 * by calling a C helper we can catch. Should reduce the number of instructions in a TB
1565 * consisting of multiple SIMD instructions.
1566 */
1567 uint32_t fSimdRaiseXcptChecksEmitted;
1568#endif
1569 /** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1570 uint32_t idxLastCheckIrqCallNo;
1571
1572 /** Core state requiring care with branches. */
1573 IEMNATIVECORESTATE Core;
1574
1575 /** The condition nesting stack. */
1576 IEMNATIVECOND aCondStack[2];
1577
1578#ifndef IEM_WITH_THROW_CATCH
1579 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1580 * for recompilation error handling. */
1581 jmp_buf JmpBuf;
1582#endif
1583} IEMRECOMPILERSTATE;
1584/** Pointer to a native recompiler state. */
1585typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1586
1587
1588/** @def IEMNATIVE_TRY_SETJMP
1589 * Wrapper around setjmp / try, hiding all the ugly differences.
1590 *
1591 * @note Use with extreme care as this is a fragile macro.
1592 * @param a_pReNative The native recompile state.
1593 * @param a_rcTarget The variable that should receive the status code in case
1594 * of a longjmp/throw.
1595 */
1596/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1597 * Start wrapper for catch / setjmp-else.
1598 *
1599 * This will set up a scope.
1600 *
1601 * @note Use with extreme care as this is a fragile macro.
1602 * @param a_pReNative The native recompile state.
1603 * @param a_rcTarget The variable that should receive the status code in case
1604 * of a longjmp/throw.
1605 */
1606/** @def IEMNATIVE_CATCH_LONGJMP_END
1607 * End wrapper for catch / setjmp-else.
1608 *
1609 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1610 * up the state.
1611 *
1612 * @note Use with extreme care as this is a fragile macro.
1613 * @param a_pReNative The native recompile state.
1614 */
1615/** @def IEMNATIVE_DO_LONGJMP
1616 *
1617 * Wrapper around longjmp / throw.
1618 *
1619 * @param a_pReNative The native recompile state.
1620 * @param a_rc The status code jump back with / throw.
1621 */
1622#ifdef IEM_WITH_THROW_CATCH
1623# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1624 a_rcTarget = VINF_SUCCESS; \
1625 try
1626# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1627 catch (int rcThrown) \
1628 { \
1629 a_rcTarget = rcThrown
1630# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1631 } \
1632 ((void)0)
1633# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1634#else /* !IEM_WITH_THROW_CATCH */
1635# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1636 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1637# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1638 else \
1639 { \
1640 ((void)0)
1641# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1642 }
1643# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1644#endif /* !IEM_WITH_THROW_CATCH */
1645
1646
1647/**
1648 * Native recompiler worker for a threaded function.
1649 *
1650 * @returns New code buffer offset; throws VBox status code in case of a failure.
1651 * @param pReNative The native recompiler state.
1652 * @param off The current code buffer offset.
1653 * @param pCallEntry The threaded call entry.
1654 *
1655 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1656 */
1657typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1658/** Pointer to a native recompiler worker for a threaded function. */
1659typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1660
1661/** Defines a native recompiler worker for a threaded function.
1662 * @see FNIEMNATIVERECOMPFUNC */
1663#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1664 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1665
1666/** Prototypes a native recompiler function for a threaded function.
1667 * @see FNIEMNATIVERECOMPFUNC */
1668#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1669
1670
1671/**
1672 * Native recompiler liveness analysis worker for a threaded function.
1673 *
1674 * @param pCallEntry The threaded call entry.
1675 * @param pIncoming The incoming liveness state entry.
1676 * @param pOutgoing The outgoing liveness state entry.
1677 */
1678typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1679 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1680/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1681typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1682
1683/** Defines a native recompiler liveness analysis worker for a threaded function.
1684 * @see FNIEMNATIVELIVENESSFUNC */
1685#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1686 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1687
1688/** Prototypes a native recompiler liveness analysis function for a threaded function.
1689 * @see FNIEMNATIVELIVENESSFUNC */
1690#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1691
1692
1693/** Define a native recompiler helper function, safe to call from the TB code. */
1694#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1695 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1696/** Prototype a native recompiler helper function, safe to call from the TB code. */
1697#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1698 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1699/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1700#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1701 a_RetType (VBOXCALL *a_Name) a_ArgList
1702
1703
1704#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1705DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1706DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1707 uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1708# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1709DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1710 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1711 uint8_t idxHstSimdReg = UINT8_MAX,
1712 uint8_t idxHstSimdRegPrev = UINT8_MAX);
1713# endif
1714# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1715DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1716 uint8_t idxGstReg, uint8_t idxHstReg);
1717DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1718 uint64_t fGstReg);
1719# endif
1720DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1721 uint64_t offPc, uint32_t cInstrSkipped);
1722#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1723
1724DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1725 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1726DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1727DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1728 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1729DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1730 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1731#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1732DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1733#endif
1734DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1735
1736DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1737DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1738 bool fPreferVolatile = true);
1739DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1740 bool fPreferVolatile = true);
1741DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1742 IEMNATIVEGSTREG enmGstReg,
1743 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1744 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1745DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1746 IEMNATIVEGSTREG enmGstReg);
1747
1748DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1749DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1750#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) || (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1751DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1752 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1753# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1754DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1755 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1756# endif
1757#endif
1758DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1759DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1760DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1761DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1762#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1763DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1764# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1765DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1766# endif
1767#endif
1768DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1769DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1770 uint32_t fKeepVars = 0);
1771DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1772DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1773DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1774 uint32_t fHstRegsActiveShadows);
1775#ifdef VBOX_STRICT
1776DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1777#endif
1778DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1779 uint64_t fGstSimdShwExcept);
1780#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1781# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1782DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1783DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
1784# endif
1785DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1786#endif
1787#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1788DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1789DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1790 PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
1791DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1792 uint64_t fFlushGstReg = UINT64_MAX);
1793DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
1794 uint32_t off, uint8_t idxHstReg);
1795#endif
1796
1797
1798#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1799DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1800DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1801 bool fPreferVolatile = true);
1802DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1803 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1804 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1805 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1806 bool fNoVolatileRegs = false);
1807DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1808DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1809DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1810 IEMNATIVEGSTSIMDREG enmGstSimdReg);
1811DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1812 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1813 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1814#endif
1815
1816DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1817DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1818DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1819DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1820DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1821DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1822DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1823DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1824DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1825 IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1826DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1827DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1828 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1829#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1830DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1831 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1832#endif
1833DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1834 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1835DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1836 uint32_t fHstRegsNotToSave);
1837DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1838 uint32_t fHstRegsNotToSave);
1839DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1840DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1841
1842DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1843 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1844#ifdef VBOX_STRICT
1845DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1846DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1847 IEMNATIVEGSTREG enmGstReg);
1848# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1849DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1850 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1851 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1852# endif
1853DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1854#endif
1855#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1856DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1857#endif
1858DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1859DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1860DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1861 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1862 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1863DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1864 PCIEMTHRDEDCALLENTRY pCallEntry);
1865DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1866 IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1867
1868
1869IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1870IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1871IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1872IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1873IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1874IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1875IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1876IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1877IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1878IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1879IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1880IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1881IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1882
1883IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1884IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1885IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1886IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1887IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1888IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1889IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1890IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1891IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1892IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1893#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1894IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1895IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1896IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1897IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1898IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1899#endif
1900IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1901IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1902IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1903IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1904#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1905IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1906IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1907IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1908IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1909#endif
1910IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1911IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1912IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1913IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1914IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1915IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1916IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1917
1918IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1919IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1920IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1921IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1922IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1923IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1924IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1925IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1926IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1927IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1928#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1929IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1930IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1931IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1932IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1933IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1934#endif
1935IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1936IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1937IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1938IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1939#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1940IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1941IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1942IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1943IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1944#endif
1945IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1946IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1947IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1948IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1949IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1950IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1951IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1952
1953IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1954IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1955IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1956IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1957IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1958IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1959IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1960IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1961IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1962IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1963IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1964IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1965IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1966IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1967IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1968IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1969IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1970IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1971IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1972IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1973IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1974IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1975
1976IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1977IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1978IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1979IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1980IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1981IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1982IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1983IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1984IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1985IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1986IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1987IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1988IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1989IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1990IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1991IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1992IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1993IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1994IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1995IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1996IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1997IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1998
1999IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2000IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2001IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2002IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2003
2004
2005/**
2006 * Info about shadowed guest register values.
2007 * @see IEMNATIVEGSTREG
2008 */
2009typedef struct IEMANTIVEGSTREGINFO
2010{
2011 /** Offset in VMCPU. */
2012 uint32_t off;
2013 /** The field size. */
2014 uint8_t cb;
2015 /** Name (for logging). */
2016 const char *pszName;
2017} IEMANTIVEGSTREGINFO;
2018extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2019extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2020extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2021extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2022extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2023
2024
2025
2026/**
2027 * Ensures that there is sufficient space in the instruction output buffer.
2028 *
2029 * This will reallocate the buffer if needed and allowed.
2030 *
2031 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2032 * allocation size.
2033 *
2034 * @returns Pointer to the instruction output buffer on success; throws VBox
2035 * status code on failure, so no need to check it.
2036 * @param pReNative The native recompile state.
2037 * @param off Current instruction offset. Works safely for UINT32_MAX
2038 * as well.
2039 * @param cInstrReq Number of instruction about to be added. It's okay to
2040 * overestimate this a bit.
2041 */
2042DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2043iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2044{
2045 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2046 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2047 {
2048#ifdef VBOX_STRICT
2049 pReNative->offInstrBufChecked = offChecked;
2050#endif
2051 return pReNative->pInstrBuf;
2052 }
2053 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2054}
2055
2056/**
2057 * Checks that we didn't exceed the space requested in the last
2058 * iemNativeInstrBufEnsure() call.
2059 */
2060#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2061 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2062 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2063
2064/**
2065 * Checks that a variable index is valid.
2066 */
2067#ifdef IEMNATIVE_VAR_IDX_MAGIC
2068# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2069 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2070 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2071 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2072 ("%s=%#x\n", #a_idxVar, a_idxVar))
2073#else
2074# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2075 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2076 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2077#endif
2078
2079/**
2080 * Checks that a variable index is valid and that the variable is assigned the
2081 * correct argument number.
2082 * This also adds a RT_NOREF of a_idxVar.
2083 */
2084#ifdef IEMNATIVE_VAR_IDX_MAGIC
2085# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2086 RT_NOREF_PV(a_idxVar); \
2087 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2088 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2089 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2090 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2091 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2092 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2093 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2094 a_uArgNo)); \
2095 } while (0)
2096#else
2097# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2098 RT_NOREF_PV(a_idxVar); \
2099 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2100 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2101 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2102 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2103 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2104 } while (0)
2105#endif
2106
2107
2108/**
2109 * Checks that a variable has the expected size.
2110 */
2111#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2112 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2113 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2114 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2115
2116
2117/**
2118 * Calculates the stack address of a variable as a [r]BP displacement value.
2119 */
2120DECL_FORCE_INLINE(int32_t)
2121iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2122{
2123 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2124 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2125}
2126
2127
2128/**
2129 * Releases the variable's register.
2130 *
2131 * The register must have been previously acquired calling
2132 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2133 * iemNativeVarRegisterSetAndAcquire().
2134 */
2135DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2136{
2137 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2138 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2139 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2140}
2141
2142
2143#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2144DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2145{
2146 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2147 iemNativeVarRegisterRelease(pReNative, idxVar);
2148}
2149#endif
2150
2151
2152/**
2153 * Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2154 *
2155 * @returns The flush mask.
2156 * @param fCImpl The IEM_CIMPL_F_XXX flags.
2157 * @param fGstShwFlush The starting flush mask.
2158 */
2159DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2160{
2161 if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2162 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2163 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2164 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2165 if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2166 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2167 | RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2168 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2169 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2170 else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2171 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2172 if (fCImpl & (IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_STATUS_FLAGS | IEM_CIMPL_F_INHIBIT_SHADOW))
2173 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
2174 return fGstShwFlush;
2175}
2176
2177
2178/** Number of hidden arguments for CIMPL calls.
2179 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2180#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2181# define IEM_CIMPL_HIDDEN_ARGS 3
2182#else
2183# define IEM_CIMPL_HIDDEN_ARGS 2
2184#endif
2185
2186
2187#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2188/** Number of hidden arguments for SSE_AIMPL calls. */
2189# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2190/** Number of hidden arguments for AVX_AIMPL calls. */
2191# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2192#endif
2193
2194
2195#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2196
2197# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2198/**
2199 * Helper for iemNativeLivenessGetStateByGstReg.
2200 *
2201 * @returns IEMLIVENESS_STATE_XXX
2202 * @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2203 * ORed together.
2204 */
2205DECL_FORCE_INLINE(uint32_t)
2206iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2207{
2208 /* INPUT trumps anything else. */
2209 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2210 return IEMLIVENESS_STATE_INPUT;
2211
2212 /* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2213 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2214 {
2215 /* If not all sub-fields are clobbered they must be considered INPUT. */
2216 if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) | RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2217 return IEMLIVENESS_STATE_INPUT;
2218 return IEMLIVENESS_STATE_CLOBBERED;
2219 }
2220
2221 /* XCPT_OR_CALL trumps UNUSED. */
2222 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2223 return IEMLIVENESS_STATE_XCPT_OR_CALL;
2224
2225 return IEMLIVENESS_STATE_UNUSED;
2226}
2227# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2228
2229
2230DECL_FORCE_INLINE(uint32_t)
2231iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2232{
2233# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2234 return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2235 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2236# else
2237 return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2238 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2239 | (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2240 | (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2241# endif
2242}
2243
2244
2245DECL_FORCE_INLINE(uint32_t)
2246iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2247{
2248 uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2249 if (enmGstReg == kIemNativeGstReg_EFlags)
2250 {
2251 /* Merge the eflags states to one. */
2252# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2253 uRet = RT_BIT_32(uRet);
2254 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflCf | (pLivenessEntry->Bit1.fEflCf << 1));
2255 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflPf | (pLivenessEntry->Bit1.fEflPf << 1));
2256 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflAf | (pLivenessEntry->Bit1.fEflAf << 1));
2257 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflZf | (pLivenessEntry->Bit1.fEflZf << 1));
2258 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflSf | (pLivenessEntry->Bit1.fEflSf << 1));
2259 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflOf | (pLivenessEntry->Bit1.fEflOf << 1));
2260 uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2261# else
2262 AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2263 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2264 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2265 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2266 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2267 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2268 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2269# endif
2270 }
2271 return uRet;
2272}
2273
2274
2275# ifdef VBOX_STRICT
2276/** For assertions only - caller checks that idxCurCall isn't zero. */
2277DECL_FORCE_INLINE(uint32_t)
2278iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2279{
2280 return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2281}
2282# endif /* VBOX_STRICT */
2283
2284#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2285
2286
2287/**
2288 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2289 */
2290DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2291{
2292 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2293 return IEM_CIMPL_HIDDEN_ARGS;
2294 if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE | IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2295 return 1;
2296 return 0;
2297}
2298
2299
2300DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2301 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2302{
2303 pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
2304
2305 pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2306 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2307 pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2308 return (uint8_t)idxReg;
2309}
2310
2311
2312
2313/*********************************************************************************************************************************
2314* Register Allocator (GPR) *
2315*********************************************************************************************************************************/
2316
2317/**
2318 * Marks host register @a idxHstReg as containing a shadow copy of guest
2319 * register @a enmGstReg.
2320 *
2321 * ASSUMES that caller has made sure @a enmGstReg is not associated with any
2322 * host register before calling.
2323 */
2324DECL_FORCE_INLINE(void)
2325iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2326{
2327 Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2328 Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2329 Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2330
2331 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2332 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2333 pReNative->Core.bmGstRegShadows |= RT_BIT_64(enmGstReg);
2334 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxHstReg);
2335#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2336 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2337 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2338#else
2339 RT_NOREF(off);
2340#endif
2341}
2342
2343
2344/**
2345 * Clear any guest register shadow claims from @a idxHstReg.
2346 *
2347 * The register does not need to be shadowing any guest registers.
2348 */
2349DECL_FORCE_INLINE(void)
2350iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2351{
2352 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2353 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2354 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2355 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2356 == RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2357#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2358 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2359#endif
2360
2361#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2362 uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2363 if (fGstRegs)
2364 {
2365 Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2366 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2367 while (fGstRegs)
2368 {
2369 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2370 fGstRegs &= ~RT_BIT_64(iGstReg);
2371 iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2372 }
2373 }
2374#else
2375 RT_NOREF(off);
2376#endif
2377
2378 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2379 pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2380 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2381}
2382
2383
2384/**
2385 * Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2386 * and global overview flags.
2387 */
2388DECL_FORCE_INLINE(void)
2389iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2390{
2391 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2392 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2393 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2394 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2395 Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2396 Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2397 Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2398#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2399 Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2400#endif
2401
2402#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2403 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2404 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2405#else
2406 RT_NOREF(off);
2407#endif
2408
2409 uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2410 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2411 if (!fGstRegShadowsNew)
2412 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2413 pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2414}
2415
2416
2417#if 0 /* unused */
2418/**
2419 * Clear any guest register shadow claim for @a enmGstReg.
2420 */
2421DECL_FORCE_INLINE(void)
2422iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2423{
2424 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2425 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2426 {
2427 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2428 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2429 }
2430}
2431#endif
2432
2433
2434/**
2435 * Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2436 * as the new shadow of it.
2437 *
2438 * Unlike the other guest reg shadow helpers, this does the logging for you.
2439 * However, it is the liveness state is not asserted here, the caller must do
2440 * that.
2441 */
2442DECL_FORCE_INLINE(void)
2443iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2444 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2445{
2446 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2447 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2448 {
2449 uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2450 Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2451 if (idxHstRegOld == idxHstRegNew)
2452 return;
2453 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2454 g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2455 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2456 }
2457 else
2458 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2459 g_aGstShadowInfo[enmGstReg].pszName));
2460 iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2461}
2462
2463
2464/**
2465 * Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2466 * to @a idxRegTo.
2467 */
2468DECL_FORCE_INLINE(void)
2469iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2470 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2471{
2472 Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2473 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2474 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2475 == pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2476 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2477 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2478 == pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2479 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2480 == RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2481
2482 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2483 pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2484 if (!fGstRegShadowsFrom)
2485 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2486 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxRegTo);
2487 pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows |= RT_BIT_64(enmGstReg);
2488 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2489#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2490 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2491 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2492#else
2493 RT_NOREF(off);
2494#endif
2495}
2496
2497
2498/**
2499 * Flushes any delayed guest register writes.
2500 *
2501 * This must be called prior to calling CImpl functions and any helpers that use
2502 * the guest state (like raising exceptions) and such.
2503 *
2504 * This optimization has not yet been implemented. The first target would be
2505 * RIP updates, since these are the most common ones.
2506 *
2507 * @note This function does not flush any shadowing information for guest
2508 * registers. This needs to be done by the caller if it wishes to do so.
2509 */
2510DECL_INLINE_THROW(uint32_t)
2511iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
2512 uint64_t fGstSimdShwExcept = 0)
2513{
2514#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2515 uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
2516#else
2517 uint64_t const fWritebackPc = 0;
2518#endif
2519#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2520 uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2521#else
2522 uint64_t const bmGstRegShadowDirty = 0;
2523#endif
2524#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2525 uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
2526 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2527 & ~fGstSimdShwExcept;
2528#else
2529 uint64_t const bmGstSimdRegShadowDirty = 0;
2530#endif
2531 if (bmGstRegShadowDirty | bmGstSimdRegShadowDirty | fWritebackPc)
2532 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2533
2534 return off;
2535}
2536
2537
2538
2539/*********************************************************************************************************************************
2540* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2541*********************************************************************************************************************************/
2542
2543#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2544
2545DECL_FORCE_INLINE(uint8_t)
2546iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2547 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2548{
2549 pReNative->Core.bmHstSimdRegs |= RT_BIT_32(idxSimdReg);
2550
2551 pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2552 pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2553 pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2554 return idxSimdReg;
2555}
2556
2557
2558/**
2559 * Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2560 * SIMD register @a enmGstSimdReg.
2561 *
2562 * ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2563 * host register before calling.
2564 */
2565DECL_FORCE_INLINE(void)
2566iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2567 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2568{
2569 Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2570 Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2571 Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2572
2573 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2574 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2575 pReNative->Core.bmGstSimdRegShadows |= RT_BIT_64(enmGstSimdReg);
2576 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxHstSimdReg);
2577#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2578 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2579 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2580#else
2581 RT_NOREF(off);
2582#endif
2583}
2584
2585
2586/**
2587 * Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2588 * to @a idxSimdRegTo.
2589 */
2590DECL_FORCE_INLINE(void)
2591iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2592 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2593{
2594 Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2595 Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2596 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2597 == pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2598 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2599 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2600 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2601 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2602 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2603 Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2604 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2605
2606 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2607 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2608 if (!fGstRegShadowsFrom)
2609 {
2610 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2611 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2612 }
2613 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxSimdRegTo);
2614 pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2615 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2616#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2617 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2618 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2619#else
2620 RT_NOREF(off);
2621#endif
2622}
2623
2624
2625/**
2626 * Clear any guest register shadow claims from @a idxHstSimdReg.
2627 *
2628 * The register does not need to be shadowing any guest registers.
2629 */
2630DECL_FORCE_INLINE(void)
2631iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2632{
2633 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2634 == pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2635 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2636 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2637 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2638 Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2639 && !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2640
2641#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2642 uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2643 if (fGstRegs)
2644 {
2645 Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2646 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2647 while (fGstRegs)
2648 {
2649 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2650 fGstRegs &= ~RT_BIT_64(iGstReg);
2651 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2652 }
2653 }
2654#else
2655 RT_NOREF(off);
2656#endif
2657
2658 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2659 pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2660 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2661 pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2662}
2663
2664#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2665
2666
2667#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2668/**
2669 * Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2670 */
2671DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2672{
2673 if (pReNative->Core.offPc)
2674 return iemNativeEmitPcWritebackSlow(pReNative, off);
2675 return off;
2676}
2677#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2678
2679
2680#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2681/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2682 * it saves us the trouble of a hidden parameter on MSC/amd64. */
2683# ifdef RT_ARCH_AMD64
2684extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2685# elif defined(RT_ARCH_ARM64)
2686extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2687# endif
2688#endif
2689
2690#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2691extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2692#endif
2693
2694#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2695
2696/** @} */
2697
2698#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2699
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette