VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 105307

Last change on this file since 105307 was 105271, checked in by vboxsync, 8 months ago

VMM/IEM: Replaced IEMNATIVEEXITREASON with IEMNATIVELABELTYPE, since it's always been a super set of it. Some source code width adjustments. bugref:10677
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 128.3 KB
Line 
1/* $Id: IEMN8veRecompiler.h 105271 2024-07-11 10:30:56Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43 * Enables generating internal debug info for better TB disassembly dumping. */
44#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
45# define IEMNATIVE_WITH_TB_DEBUG_INFO
46#endif
47
48/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49 * Enables liveness analysis. */
50#if 1 || defined(DOXYGEN_RUNNING)
51# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
53#endif
54
55/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56 * Enables skipping EFLAGS calculations/updating based on liveness info. */
57#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(DOXYGEN_RUNNING)
58# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59#endif
60
61/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62 * Enables strict consistency checks around EFLAGS skipping.
63 * @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65# ifdef VBOX_STRICT
66# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67# endif
68#elif defined(DOXYGEN_RUNNING)
69# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70#endif
71
72#ifdef VBOX_WITH_STATISTICS
73/** Always count instructions for now. */
74# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75#endif
76
77/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78 * Enables having only a single prologue for native TBs. */
79#if 1 || defined(DOXYGEN_RUNNING)
80# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81#endif
82
83/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84 * Enable this to use common epilogue and tail code for all TBs in a chunk. */
85#if 1 || defined(DOXYGEN_RUNNING)
86# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87#endif
88
89
90/** @name Stack Frame Layout
91 *
92 * @{ */
93/** The size of the area for stack variables and spills and stuff.
94 * @note This limit is duplicated in the python script(s). We add 0x40 for
95 * alignment padding. */
96#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101#ifdef RT_ARCH_AMD64
102/** An stack alignment adjustment (between non-volatile register pushes and
103 * the stack variable area, so the latter better aligned). */
104# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106/** Number of stack arguments slots for calls made from the frame. */
107# ifdef RT_OS_WINDOWS
108# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109# else
110# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111# endif
112/** Number of any shadow arguments (spill area) for calls we make. */
113# ifdef RT_OS_WINDOWS
114# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115# else
116# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117# endif
118
119/** Frame pointer (RBP) relative offset of the last push. */
120# ifdef RT_OS_WINDOWS
121# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122# else
123# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124# endif
125/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126 * address for it). */
127# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132# ifdef RT_OS_WINDOWS
133/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137# endif
138
139# ifdef RT_OS_WINDOWS
140/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148# endif
149
150#elif RT_ARCH_ARM64
151/** No alignment padding needed for arm64. */
152# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153/** No stack argument slots, got 8 registers for arguments will suffice. */
154# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155/** There are no argument spill area. */
156# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158/** Number of saved registers at the top of our stack frame.
159 * This includes the return address and old frame pointer, so x19 thru x30. */
160# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164/** Frame pointer (BP) relative offset of the last push. */
165# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168 * address for it). */
169# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171#else
172# error "port me"
173#endif
174/** @} */
175
176
177/** @name Fixed Register Allocation(s)
178 * @{ */
179/** @def IEMNATIVE_REG_FIXED_PVMCPU
180 * The number of the register holding the pVCpu pointer. */
181/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183 * @note This not available on AMD64, only ARM64. */
184/** @def IEMNATIVE_REG_FIXED_TMP0
185 * Dedicated temporary register.
186 * @todo replace this by a register allocator and content tracker. */
187/** @def IEMNATIVE_REG_FIXED_MASK
188 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
189 * architecture. */
190#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
191/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
192 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
193 * architecture. */
194/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195 * Dedicated temporary SIMD register. */
196#endif
197#if defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
198# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
199# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
200# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
201# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
202# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208# else
209# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210# endif
211# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212 | RT_BIT_32(ARMV8_A64_REG_LR) \
213 | RT_BIT_32(ARMV8_A64_REG_BP) \
214 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216 | RT_BIT_32(ARMV8_A64_REG_X18) \
217 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218 | IEMNATIVE_REG_FIXED_MASK_ADD)
219
220# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224# else
225/** @note
226 * ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
227 * support emulating 256-bit registers we pair two real registers statically to
228 * one virtual for now, leaving us with only 16 256-bit registers. We always
229 * pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
230 * the register allocator assumes that it will be always free when the lower is
231 * picked.
232 *
233 * Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
234 * touch them in order to avoid having to save and restore them in the
235 * prologue/epilogue.
236 */
237# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
238 | RT_BIT_32(ARMV8_A64_REG_Q31) \
239 | RT_BIT_32(ARMV8_A64_REG_Q30) \
240 | RT_BIT_32(ARMV8_A64_REG_Q29) \
241 | RT_BIT_32(ARMV8_A64_REG_Q27) \
242 | RT_BIT_32(ARMV8_A64_REG_Q25) \
243 | RT_BIT_32(ARMV8_A64_REG_Q23) \
244 | RT_BIT_32(ARMV8_A64_REG_Q21) \
245 | RT_BIT_32(ARMV8_A64_REG_Q19) \
246 | RT_BIT_32(ARMV8_A64_REG_Q17) \
247 | RT_BIT_32(ARMV8_A64_REG_Q15) \
248 | RT_BIT_32(ARMV8_A64_REG_Q13) \
249 | RT_BIT_32(ARMV8_A64_REG_Q11) \
250 | RT_BIT_32(ARMV8_A64_REG_Q9) \
251 | RT_BIT_32(ARMV8_A64_REG_Q7) \
252 | RT_BIT_32(ARMV8_A64_REG_Q5) \
253 | RT_BIT_32(ARMV8_A64_REG_Q3) \
254 | RT_BIT_32(ARMV8_A64_REG_Q1))
255# endif
256# endif
257
258#elif defined(RT_ARCH_AMD64)
259# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
260# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
261# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
262# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
263 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
264 | RT_BIT_32(X86_GREG_xSP) \
265 | RT_BIT_32(X86_GREG_xBP) )
266
267# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
268# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
269# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
270# ifndef _MSC_VER
271# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
272# endif
273# endif
274# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
276# else
277/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
278# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
279 | RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
280# endif
281# endif
282
283#else
284# error "port me"
285#endif
286/** @} */
287
288/** @name Call related registers.
289 * @{ */
290/** @def IEMNATIVE_CALL_RET_GREG
291 * The return value register. */
292/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
293 * Number of arguments in registers. */
294/** @def IEMNATIVE_CALL_ARG0_GREG
295 * The general purpose register carrying argument \#0. */
296/** @def IEMNATIVE_CALL_ARG1_GREG
297 * The general purpose register carrying argument \#1. */
298/** @def IEMNATIVE_CALL_ARG2_GREG
299 * The general purpose register carrying argument \#2. */
300/** @def IEMNATIVE_CALL_ARG3_GREG
301 * The general purpose register carrying argument \#3. */
302/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
303 * Mask of registers the callee will not save and may trash. */
304#ifdef RT_ARCH_AMD64
305# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
306
307# ifdef RT_OS_WINDOWS
308# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
309# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
310# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
311# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
312# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
313# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
314 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
315 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
316 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
317# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
318 | RT_BIT_32(X86_GREG_xCX) \
319 | RT_BIT_32(X86_GREG_xDX) \
320 | RT_BIT_32(X86_GREG_x8) \
321 | RT_BIT_32(X86_GREG_x9) \
322 | RT_BIT_32(X86_GREG_x10) \
323 | RT_BIT_32(X86_GREG_x11) )
324# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
325/* xmm0 - xmm5 are marked as volatile. */
326# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
327# endif
328
329# else /* !RT_OS_WINDOWS */
330# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
331# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
332# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
333# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
334# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
335# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
336# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
337# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
338 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
339 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
340 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
341 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
342 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
343# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
344 | RT_BIT_32(X86_GREG_xCX) \
345 | RT_BIT_32(X86_GREG_xDX) \
346 | RT_BIT_32(X86_GREG_xDI) \
347 | RT_BIT_32(X86_GREG_xSI) \
348 | RT_BIT_32(X86_GREG_x8) \
349 | RT_BIT_32(X86_GREG_x9) \
350 | RT_BIT_32(X86_GREG_x10) \
351 | RT_BIT_32(X86_GREG_x11) )
352# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
353/* xmm0 - xmm15 are marked as volatile. */
354# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
355# endif
356# endif /* !RT_OS_WINDOWS */
357
358#elif defined(RT_ARCH_ARM64)
359# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
360# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
361# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
362# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
363# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
364# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
365# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
366# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
367# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
368# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
369# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
370 | RT_BIT_32(ARMV8_A64_REG_X1) \
371 | RT_BIT_32(ARMV8_A64_REG_X2) \
372 | RT_BIT_32(ARMV8_A64_REG_X3) \
373 | RT_BIT_32(ARMV8_A64_REG_X4) \
374 | RT_BIT_32(ARMV8_A64_REG_X5) \
375 | RT_BIT_32(ARMV8_A64_REG_X6) \
376 | RT_BIT_32(ARMV8_A64_REG_X7) )
377# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
378 | RT_BIT_32(ARMV8_A64_REG_X1) \
379 | RT_BIT_32(ARMV8_A64_REG_X2) \
380 | RT_BIT_32(ARMV8_A64_REG_X3) \
381 | RT_BIT_32(ARMV8_A64_REG_X4) \
382 | RT_BIT_32(ARMV8_A64_REG_X5) \
383 | RT_BIT_32(ARMV8_A64_REG_X6) \
384 | RT_BIT_32(ARMV8_A64_REG_X7) \
385 | RT_BIT_32(ARMV8_A64_REG_X8) \
386 | RT_BIT_32(ARMV8_A64_REG_X9) \
387 | RT_BIT_32(ARMV8_A64_REG_X10) \
388 | RT_BIT_32(ARMV8_A64_REG_X11) \
389 | RT_BIT_32(ARMV8_A64_REG_X12) \
390 | RT_BIT_32(ARMV8_A64_REG_X13) \
391 | RT_BIT_32(ARMV8_A64_REG_X14) \
392 | RT_BIT_32(ARMV8_A64_REG_X15) \
393 | RT_BIT_32(ARMV8_A64_REG_X16) \
394 | RT_BIT_32(ARMV8_A64_REG_X17) )
395# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
396/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
397 * so to simplify our life a bit we just mark everything as volatile. */
398# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
399# endif
400
401#endif
402
403/** This is the maximum argument count we'll ever be needing. */
404#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
405#ifdef RT_OS_WINDOWS
406# ifdef VBOXSTRICTRC_STRICT_ENABLED
407# undef IEMNATIVE_CALL_MAX_ARG_COUNT
408# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
409# endif
410#endif
411/** @} */
412
413
414/** @def IEMNATIVE_HST_GREG_COUNT
415 * Number of host general purpose registers we tracker. */
416/** @def IEMNATIVE_HST_GREG_MASK
417 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
418 * inverted register masks and such to get down to a correct set of regs. */
419#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
420/** @def IEMNATIVE_HST_SIMD_REG_COUNT
421 * Number of host SIMD registers we track. */
422/** @def IEMNATIVE_HST_SIMD_REG_MASK
423 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
424 * inverted register masks and such to get down to a correct set of regs. */
425#endif
426#ifdef RT_ARCH_AMD64
427# define IEMNATIVE_HST_GREG_COUNT 16
428# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
429
430# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
431# define IEMNATIVE_HST_SIMD_REG_COUNT 16
432# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
433# endif
434
435#elif defined(RT_ARCH_ARM64)
436# define IEMNATIVE_HST_GREG_COUNT 32
437# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
438
439# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
440# define IEMNATIVE_HST_SIMD_REG_COUNT 32
441# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
442# endif
443
444#else
445# error "Port me!"
446#endif
447
448
449#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
450
451
452/** Native code generator label types. */
453typedef enum
454{
455 kIemNativeLabelType_Invalid = 0,
456 /*
457 * Labels w/o data, only once instance per TB - aka exit reasons.
458 *
459 * Note! Jumps to these requires instructions that are capable of spanning
460 * the max TB length.
461 */
462 /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
463 kIemNativeLabelType_RaiseDe, /**< Raise (throw) X86_XCPT_DE (00h). */
464 kIemNativeLabelType_RaiseUd, /**< Raise (throw) X86_XCPT_UD (06h). */
465 kIemNativeLabelType_RaiseSseRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
466 kIemNativeLabelType_RaiseAvxRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
467 kIemNativeLabelType_RaiseSseAvxFpRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
468 kIemNativeLabelType_RaiseNm, /**< Raise (throw) X86_XCPT_NM (07h). */
469 kIemNativeLabelType_RaiseGp0, /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
470 kIemNativeLabelType_RaiseMf, /**< Raise (throw) X86_XCPT_MF (10h). */
471 kIemNativeLabelType_RaiseXf, /**< Raise (throw) X86_XCPT_XF (13h). */
472 kIemNativeLabelType_ObsoleteTb,
473 kIemNativeLabelType_NeedCsLimChecking,
474 kIemNativeLabelType_CheckBranchMiss,
475 kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
476 /* Manually defined labels. */
477 kIemNativeLabelType_ReturnBreak,
478 kIemNativeLabelType_ReturnBreakFF,
479 kIemNativeLabelType_ReturnBreakViaLookup,
480 kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
481 kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
482 kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
483 kIemNativeLabelType_ReturnWithFlags,
484 kIemNativeLabelType_NonZeroRetOrPassUp,
485 kIemNativeLabelType_Return,
486 /** The last fixup for branches that can span almost the whole TB length.
487 * @note Whether kIemNativeLabelType_Return needs to be one of these is
488 * a bit questionable, since nobody jumps to it except other tail code. */
489 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
490 /** The last fixup for branches that exits the TB. */
491 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
492
493 /*
494 * Labels with data, potentially multiple instances per TB:
495 *
496 * These are localized labels, so no fixed jump type restrictions here.
497 */
498 kIemNativeLabelType_FirstWithMultipleInstances,
499 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
500 kIemNativeLabelType_Else,
501 kIemNativeLabelType_Endif,
502 kIemNativeLabelType_CheckIrq,
503 kIemNativeLabelType_TlbLookup,
504 kIemNativeLabelType_TlbMiss,
505 kIemNativeLabelType_TlbDone,
506 kIemNativeLabelType_End
507} IEMNATIVELABELTYPE;
508
509#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
510 ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
511
512
513/** Native code generator label definition. */
514typedef struct IEMNATIVELABEL
515{
516 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
517 * the epilog. */
518 uint32_t off;
519 /** The type of label (IEMNATIVELABELTYPE). */
520 uint16_t enmType;
521 /** Additional label data, type specific. */
522 uint16_t uData;
523} IEMNATIVELABEL;
524/** Pointer to a label. */
525typedef IEMNATIVELABEL *PIEMNATIVELABEL;
526
527
528/** Native code generator fixup types. */
529typedef enum
530{
531 kIemNativeFixupType_Invalid = 0,
532#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
533 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
534 kIemNativeFixupType_Rel32,
535#elif defined(RT_ARCH_ARM64)
536 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
537 kIemNativeFixupType_RelImm26At0,
538 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
539 kIemNativeFixupType_RelImm19At5,
540 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
541 kIemNativeFixupType_RelImm14At5,
542#endif
543 kIemNativeFixupType_End
544} IEMNATIVEFIXUPTYPE;
545
546/** Native code generator fixup. */
547typedef struct IEMNATIVEFIXUP
548{
549 /** Code offset of the fixup location. */
550 uint32_t off;
551 /** The IEMNATIVELABEL this is a fixup for. */
552 uint16_t idxLabel;
553 /** The fixup type (IEMNATIVEFIXUPTYPE). */
554 uint8_t enmType;
555 /** Addend or other data. */
556 int8_t offAddend;
557} IEMNATIVEFIXUP;
558/** Pointer to a native code generator fixup. */
559typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
560
561#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
562
563/** Native code generator fixup to per chunk TB tail code. */
564typedef struct IEMNATIVEEXITFIXUP
565{
566 /** Code offset of the fixup location. */
567 uint32_t off;
568 /** The exit reason. */
569 IEMNATIVELABELTYPE enmExitReason;
570} IEMNATIVEEXITFIXUP;
571/** Pointer to a native code generator TB exit fixup. */
572typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
573
574/**
575 * Per executable memory chunk context with addresses for common code.
576 */
577typedef struct IEMNATIVEPERCHUNKCTX
578{
579 /** Pointers to the exit labels */
580 PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
581} IEMNATIVEPERCHUNKCTX;
582/** Pointer to per-chunk recompiler context. */
583typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
584/** Pointer to const per-chunk recompiler context. */
585typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
586
587#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
588
589
590/**
591 * One bit of the state.
592 *
593 * Each register state takes up two bits. We keep the two bits in two separate
594 * 64-bit words to simplify applying them to the guest shadow register mask in
595 * the register allocator.
596 */
597typedef union IEMLIVENESSBIT
598{
599 uint64_t bm64;
600 RT_GCC_EXTENSION struct
601 { /* bit no */
602 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
603 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
604 uint64_t fCr0 : 1; /**< 0x11 / 17: */
605 uint64_t fFcw : 1; /**< 0x12 / 18: */
606 uint64_t fFsw : 1; /**< 0x13 / 19: */
607 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
608 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
609 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
610 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
611 uint64_t fCr4 : 1; /**< 0x2c / 44: */
612 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
613 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
614 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
615 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
616 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
617 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
618 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
619 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
620 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
621 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
622 };
623} IEMLIVENESSBIT;
624AssertCompileSize(IEMLIVENESSBIT, 8);
625
626#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
627#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
628#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
629#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
630#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
631#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
632#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
633
634
635/**
636 * A liveness state entry.
637 *
638 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
639 * Once we add a SSE register shadowing, we'll add another 64-bit element for
640 * that.
641 */
642typedef union IEMLIVENESSENTRY
643{
644#ifndef IEMLIVENESS_EXTENDED_LAYOUT
645 uint64_t bm64[16 / 8];
646 uint16_t bm32[16 / 4];
647 uint16_t bm16[16 / 2];
648 uint8_t bm8[ 16 / 1];
649 IEMLIVENESSBIT aBits[2];
650#else
651 uint64_t bm64[32 / 8];
652 uint16_t bm32[32 / 4];
653 uint16_t bm16[32 / 2];
654 uint8_t bm8[ 32 / 1];
655 IEMLIVENESSBIT aBits[4];
656#endif
657 RT_GCC_EXTENSION struct
658 {
659 /** Bit \#0 of the register states. */
660 IEMLIVENESSBIT Bit0;
661 /** Bit \#1 of the register states. */
662 IEMLIVENESSBIT Bit1;
663#ifdef IEMLIVENESS_EXTENDED_LAYOUT
664 /** Bit \#2 of the register states. */
665 IEMLIVENESSBIT Bit2;
666 /** Bit \#3 of the register states. */
667 IEMLIVENESSBIT Bit3;
668#endif
669 };
670} IEMLIVENESSENTRY;
671#ifndef IEMLIVENESS_EXTENDED_LAYOUT
672AssertCompileSize(IEMLIVENESSENTRY, 16);
673#else
674AssertCompileSize(IEMLIVENESSENTRY, 32);
675#endif
676/** Pointer to a liveness state entry. */
677typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
678/** Pointer to a const liveness state entry. */
679typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
680
681/** @name 64-bit value masks for IEMLIVENESSENTRY.
682 * @{ */ /* 0xzzzzyyyyxxxxwwww */
683#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
684
685#ifndef IEMLIVENESS_EXTENDED_LAYOUT
686# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
687# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
688
689# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
690# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
691#endif
692
693#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
694#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
695
696#ifndef IEMLIVENESS_EXTENDED_LAYOUT
697# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
698# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
699#endif
700/** @} */
701
702
703/** @name The liveness state for a register.
704 *
705 * The state values have been picked to with state accumulation in mind (what
706 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
707 * performance critical work done with the values.
708 *
709 * This is a compressed state that only requires 2 bits per register.
710 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
711 * 1. the incoming state from the following call,
712 * 2. the outgoing state for this call,
713 * 3. mask of the entries set in the 2nd.
714 *
715 * The mask entry (3rd one above) will be used both when updating the outgoing
716 * state and when merging in incoming state for registers not touched by the
717 * current call.
718 *
719 * @{ */
720#ifndef IEMLIVENESS_EXTENDED_LAYOUT
721/** The register will be clobbered and the current value thrown away.
722 *
723 * When this is applied to the state (2) we'll simply be AND'ing it with the
724 * (old) mask (3) and adding the register to the mask. This way we'll
725 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
726 * IEMLIVENESS_STATE_INPUT states. */
727# define IEMLIVENESS_STATE_CLOBBERED 0
728/** The register is unused in the remainder of the TB.
729 *
730 * This is an initial state and can not be set by any of the
731 * iemNativeLivenessFunc_xxxx callbacks. */
732# define IEMLIVENESS_STATE_UNUSED 1
733/** The register value is required in a potential call or exception.
734 *
735 * This means that the register value must be calculated and is best written to
736 * the state, but that any shadowing registers can be flushed thereafter as it's
737 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
738 *
739 * It is typically applied across the board, but we preserve incoming
740 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
741 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
742 * 1. r0 = old & ~mask;
743 * 2. r0 = t1 & (t1 >> 1)'
744 * 3. state |= r0 | 0b10;
745 * 4. mask = ~0;
746 */
747# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
748/** The register value is used as input.
749 *
750 * This means that the register value must be calculated and it is best to keep
751 * it in a register. It does not need to be writtent out as such. This is the
752 * highest priority state.
753 *
754 * Whether the call modifies the register or not isn't relevant to earlier
755 * calls, so that's not recorded.
756 *
757 * When applying this state we just or in the value in the outgoing state and
758 * mask. */
759# define IEMLIVENESS_STATE_INPUT 3
760/** Mask of the state bits. */
761# define IEMLIVENESS_STATE_MASK 3
762/** The number of bits per state. */
763# define IEMLIVENESS_STATE_BIT_COUNT 2
764/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
765# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
766/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
767# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
768/** Check if a register clobbering is expected given the (previous) liveness state.
769 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
770 * include INPUT if the register is used in more than one place. */
771# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
772
773/** Check if all status flags are going to be clobbered and doesn't need
774 * calculating in the current step.
775 * @param a_pCurEntry The current liveness entry. */
776# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
777 ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
778
779#else /* IEMLIVENESS_EXTENDED_LAYOUT */
780/** The register is not used any more. */
781# define IEMLIVENESS_STATE_UNUSED 0
782/** Flag: The register is required in a potential exception or call. */
783# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
784# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
785/** Flag: The register is read. */
786# define IEMLIVENESS_STATE_READ 2
787# define IEMLIVENESS_BIT_READ 1
788/** Flag: The register is written. */
789# define IEMLIVENESS_STATE_WRITE 4
790# define IEMLIVENESS_BIT_WRITE 2
791/** Flag: Unconditional call (not needed, can be redefined for research). */
792# define IEMLIVENESS_STATE_CALL 8
793# define IEMLIVENESS_BIT_CALL 3
794# define IEMLIVENESS_BIT_OTHER 3 /**< More convenient name for this one. */
795# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
796 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
797# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
798# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
799
800# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
801 ( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
802 && !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
803 & IEMLIVENESSBIT_STATUS_EFL_MASK) )
804
805#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
806/** @} */
807
808/** @name Liveness helpers for builtin functions and similar.
809 *
810 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
811 * own set of manimulator macros for those.
812 *
813 * @{ */
814/** Initializing the state as all unused. */
815#ifndef IEMLIVENESS_EXTENDED_LAYOUT
816# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
817 do { \
818 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
819 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
820 } while (0)
821#else
822# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
823 do { \
824 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
825 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
826 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
827 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
828 } while (0)
829#endif
830
831/** Initializing the outgoing state with a potential xcpt or call state.
832 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
833#ifndef IEMLIVENESS_EXTENDED_LAYOUT
834# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
835 do { \
836 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
837 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
838 } while (0)
839#else
840# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
841 do { \
842 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
843 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
844 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
845 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
846 } while (0)
847#endif
848
849/** Adds a segment base register as input to the outgoing state. */
850#ifndef IEMLIVENESS_EXTENDED_LAYOUT
851# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
852 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
853 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
854 } while (0)
855#else
856# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
857 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
858 } while (0)
859#endif
860
861/** Adds a segment attribute register as input to the outgoing state. */
862#ifndef IEMLIVENESS_EXTENDED_LAYOUT
863# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
864 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
865 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
866 } while (0)
867#else
868# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
869 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
870 } while (0)
871#endif
872
873/** Adds a segment limit register as input to the outgoing state. */
874#ifndef IEMLIVENESS_EXTENDED_LAYOUT
875# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
876 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
877 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
878 } while (0)
879#else
880# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
881 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
882 } while (0)
883#endif
884
885/** Adds a segment limit register as input to the outgoing state. */
886#ifndef IEMLIVENESS_EXTENDED_LAYOUT
887# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
888 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
889 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
890 } while (0)
891#else
892# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
893 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
894 } while (0)
895#endif
896/** @} */
897
898/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
899 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
900 * calculated and up to date. This is to double check that we haven't skipped
901 * EFLAGS calculations when we actually need them. NOP in non-strict builds.
902 * @note has to be placed in
903 */
904#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
905# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
906 do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
907#else
908# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
909#endif
910
911
912/**
913 * Guest registers that can be shadowed in GPRs.
914 *
915 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
916 * must be placed last, as the liveness state tracks it as 7 subcomponents and
917 * we don't want to waste space here.
918 *
919 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
920 * friends as well as IEMAllN8veLiveness.cpp.
921 */
922typedef enum IEMNATIVEGSTREG : uint8_t
923{
924 kIemNativeGstReg_GprFirst = 0,
925 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
926 kIemNativeGstReg_Pc,
927 kIemNativeGstReg_Cr0,
928 kIemNativeGstReg_FpuFcw,
929 kIemNativeGstReg_FpuFsw,
930 kIemNativeGstReg_SegBaseFirst,
931 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
932 kIemNativeGstReg_SegAttribFirst,
933 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
934 kIemNativeGstReg_SegLimitFirst,
935 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
936 kIemNativeGstReg_SegSelFirst,
937 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
938 kIemNativeGstReg_Cr4,
939 kIemNativeGstReg_Xcr0,
940 kIemNativeGstReg_MxCsr,
941 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
942 kIemNativeGstReg_End
943} IEMNATIVEGSTREG;
944AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
945AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
946
947/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
948 * @{ */
949#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
950#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
951#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
952#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
953#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
954/** @} */
955
956#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
957
958/**
959 * Guest registers that can be shadowed in host SIMD registers.
960 *
961 * @todo r=aeichner Liveness tracking
962 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
963 */
964typedef enum IEMNATIVEGSTSIMDREG : uint8_t
965{
966 kIemNativeGstSimdReg_SimdRegFirst = 0,
967 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
968 kIemNativeGstSimdReg_End
969} IEMNATIVEGSTSIMDREG;
970
971/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
972 * @{ */
973#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
974/** @} */
975
976/**
977 * The Load/store size for a SIMD guest register.
978 */
979typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
980{
981 /** Invalid size. */
982 kIemNativeGstSimdRegLdStSz_Invalid = 0,
983 /** Loads the low 128-bit of a guest SIMD register. */
984 kIemNativeGstSimdRegLdStSz_Low128,
985 /** Loads the high 128-bit of a guest SIMD register. */
986 kIemNativeGstSimdRegLdStSz_High128,
987 /** Loads the whole 256-bits of a guest SIMD register. */
988 kIemNativeGstSimdRegLdStSz_256,
989 /** End value. */
990 kIemNativeGstSimdRegLdStSz_End
991} IEMNATIVEGSTSIMDREGLDSTSZ;
992
993#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
994
995/**
996 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
997 */
998typedef enum IEMNATIVEGSTREGUSE
999{
1000 /** The usage is read-only, the register holding the guest register
1001 * shadow copy will not be modified by the caller. */
1002 kIemNativeGstRegUse_ReadOnly = 0,
1003 /** The caller will update the guest register (think: PC += cbInstr).
1004 * The guest shadow copy will follow the returned register. */
1005 kIemNativeGstRegUse_ForUpdate,
1006 /** The call will put an entirely new value in the guest register, so
1007 * if new register is allocate it will be returned uninitialized. */
1008 kIemNativeGstRegUse_ForFullWrite,
1009 /** The caller will use the guest register value as input in a calculation
1010 * and the host register will be modified.
1011 * This means that the returned host register will not be marked as a shadow
1012 * copy of the guest register. */
1013 kIemNativeGstRegUse_Calculation
1014} IEMNATIVEGSTREGUSE;
1015
1016/**
1017 * Guest registers (classes) that can be referenced.
1018 */
1019typedef enum IEMNATIVEGSTREGREF : uint8_t
1020{
1021 kIemNativeGstRegRef_Invalid = 0,
1022 kIemNativeGstRegRef_Gpr,
1023 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
1024 kIemNativeGstRegRef_EFlags,
1025 kIemNativeGstRegRef_MxCsr,
1026 kIemNativeGstRegRef_FpuReg,
1027 kIemNativeGstRegRef_MReg,
1028 kIemNativeGstRegRef_XReg,
1029 kIemNativeGstRegRef_X87,
1030 kIemNativeGstRegRef_XState,
1031 //kIemNativeGstRegRef_YReg, - doesn't work.
1032 kIemNativeGstRegRef_End
1033} IEMNATIVEGSTREGREF;
1034
1035
1036/** Variable kinds. */
1037typedef enum IEMNATIVEVARKIND : uint8_t
1038{
1039 /** Customary invalid zero value. */
1040 kIemNativeVarKind_Invalid = 0,
1041 /** This is either in a register or on the stack. */
1042 kIemNativeVarKind_Stack,
1043 /** Immediate value - loaded into register when needed, or can live on the
1044 * stack if referenced (in theory). */
1045 kIemNativeVarKind_Immediate,
1046 /** Variable reference - loaded into register when needed, never stack. */
1047 kIemNativeVarKind_VarRef,
1048 /** Guest register reference - loaded into register when needed, never stack. */
1049 kIemNativeVarKind_GstRegRef,
1050 /** End of valid values. */
1051 kIemNativeVarKind_End
1052} IEMNATIVEVARKIND;
1053
1054
1055/** Variable or argument. */
1056typedef struct IEMNATIVEVAR
1057{
1058 /** The kind of variable. */
1059 IEMNATIVEVARKIND enmKind;
1060 /** The variable size in bytes. */
1061 uint8_t cbVar;
1062 /** The first stack slot (uint64_t), except for immediate and references
1063 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1064 * has a stack slot it has been initialized and has a value. Unused variables
1065 * has neither a stack slot nor a host register assignment. */
1066 uint8_t idxStackSlot;
1067 /** The host register allocated for the variable, UINT8_MAX if not. */
1068 uint8_t idxReg;
1069 /** The argument number if argument, UINT8_MAX if regular variable. */
1070 uint8_t uArgNo;
1071 /** If referenced, the index (unpacked) of the variable referencing this one,
1072 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1073 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1074 uint8_t idxReferrerVar;
1075 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1076 * @todo not sure what this really is for... */
1077 IEMNATIVEGSTREG enmGstReg;
1078#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1079 /** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1080 * only valid when idxReg is not UINT8_MAX. */
1081 bool fSimdReg : 1;
1082 /** Set if the registered is currently used exclusively, false if the
1083 * variable is idle and the register can be grabbed. */
1084 bool fRegAcquired : 1;
1085#else
1086 /** Set if the registered is currently used exclusively, false if the
1087 * variable is idle and the register can be grabbed. */
1088 bool fRegAcquired;
1089#endif
1090
1091 union
1092 {
1093 /** kIemNativeVarKind_Immediate: The immediate value. */
1094 uint64_t uValue;
1095 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1096 uint8_t idxRefVar;
1097 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1098 struct
1099 {
1100 /** The class of register. */
1101 IEMNATIVEGSTREGREF enmClass;
1102 /** Index within the class. */
1103 uint8_t idx;
1104 } GstRegRef;
1105 } u;
1106} IEMNATIVEVAR;
1107/** Pointer to a variable or argument. */
1108typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1109/** Pointer to a const variable or argument. */
1110typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1111
1112/** What is being kept in a host register. */
1113typedef enum IEMNATIVEWHAT : uint8_t
1114{
1115 /** The traditional invalid zero value. */
1116 kIemNativeWhat_Invalid = 0,
1117 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1118 kIemNativeWhat_Var,
1119 /** Temporary register, this is typically freed when a MC completes. */
1120 kIemNativeWhat_Tmp,
1121 /** Call argument w/o a variable mapping. This is free (via
1122 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1123 kIemNativeWhat_Arg,
1124 /** Return status code.
1125 * @todo not sure if we need this... */
1126 kIemNativeWhat_rc,
1127 /** The fixed pVCpu (PVMCPUCC) register.
1128 * @todo consider offsetting this on amd64 to use negative offsets to access
1129 * more members using 8-byte disp. */
1130 kIemNativeWhat_pVCpuFixed,
1131 /** The fixed pCtx (PCPUMCTX) register.
1132 * @todo consider offsetting this on amd64 to use negative offsets to access
1133 * more members using 8-byte disp. */
1134 kIemNativeWhat_pCtxFixed,
1135 /** Fixed temporary register. */
1136 kIemNativeWhat_FixedTmp,
1137#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1138 /** Shadow RIP for the delayed RIP updating debugging. */
1139 kIemNativeWhat_PcShadow,
1140#endif
1141 /** Register reserved by the CPU or OS architecture. */
1142 kIemNativeWhat_FixedReserved,
1143 /** End of valid values. */
1144 kIemNativeWhat_End
1145} IEMNATIVEWHAT;
1146
1147/**
1148 * Host general register entry.
1149 *
1150 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1151 *
1152 * @todo Track immediate values in host registers similarlly to how we track the
1153 * guest register shadow copies. For it to be real helpful, though,
1154 * we probably need to know which will be reused and put them into
1155 * non-volatile registers, otherwise it's going to be more or less
1156 * restricted to an instruction or two.
1157 */
1158typedef struct IEMNATIVEHSTREG
1159{
1160 /** Set of guest registers this one shadows.
1161 *
1162 * Using a bitmap here so we can designate the same host register as a copy
1163 * for more than one guest register. This is expected to be useful in
1164 * situations where one value is copied to several registers in a sequence.
1165 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1166 * sequence we'd want to let this register follow to be a copy of and there
1167 * will always be places where we'd be picking the wrong one.
1168 */
1169 uint64_t fGstRegShadows;
1170 /** What is being kept in this register. */
1171 IEMNATIVEWHAT enmWhat;
1172 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1173 uint8_t idxVar;
1174 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1175 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1176 * that scope. */
1177 uint8_t idxStackSlot;
1178 /** Alignment padding. */
1179 uint8_t abAlign[5];
1180} IEMNATIVEHSTREG;
1181
1182
1183#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1184/**
1185 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1186 * halves, on architectures where there is no 256-bit register available this entry will track
1187 * two adjacent 128-bit host registers.
1188 *
1189 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1190 */
1191typedef struct IEMNATIVEHSTSIMDREG
1192{
1193 /** Set of guest registers this one shadows.
1194 *
1195 * Using a bitmap here so we can designate the same host register as a copy
1196 * for more than one guest register. This is expected to be useful in
1197 * situations where one value is copied to several registers in a sequence.
1198 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1199 * sequence we'd want to let this register follow to be a copy of and there
1200 * will always be places where we'd be picking the wrong one.
1201 */
1202 uint64_t fGstRegShadows;
1203 /** What is being kept in this register. */
1204 IEMNATIVEWHAT enmWhat;
1205 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1206 uint8_t idxVar;
1207 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1208 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1209 /** Alignment padding. */
1210 uint8_t abAlign[5];
1211} IEMNATIVEHSTSIMDREG;
1212#endif
1213
1214
1215/**
1216 * Core state for the native recompiler, that is, things that needs careful
1217 * handling when dealing with branches.
1218 */
1219typedef struct IEMNATIVECORESTATE
1220{
1221#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1222 /** The current instruction offset in bytes from when the guest program counter
1223 * was updated last. Used for delaying the write to the guest context program counter
1224 * as long as possible. */
1225 uint32_t offPc;
1226 /** Number of instructions where we could skip the updating. */
1227 uint32_t cInstrPcUpdateSkipped;
1228#endif
1229 /** Allocation bitmap for aHstRegs. */
1230 uint32_t bmHstRegs;
1231
1232 /** Bitmap marking which host register contains guest register shadow copies.
1233 * This is used during register allocation to try preserve copies. */
1234 uint32_t bmHstRegsWithGstShadow;
1235 /** Bitmap marking valid entries in aidxGstRegShadows. */
1236 uint64_t bmGstRegShadows;
1237#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1238 /** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1239 uint64_t bmGstRegShadowDirty;
1240#endif
1241
1242#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1243 /** Allocation bitmap for aHstSimdRegs. */
1244 uint32_t bmHstSimdRegs;
1245
1246 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1247 * This is used during register allocation to try preserve copies. */
1248 uint32_t bmHstSimdRegsWithGstShadow;
1249 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1250 uint64_t bmGstSimdRegShadows;
1251 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1252 uint64_t bmGstSimdRegShadowDirtyLo128;
1253 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1254 uint64_t bmGstSimdRegShadowDirtyHi128;
1255#endif
1256
1257 union
1258 {
1259 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1260 uint8_t aidxArgVars[8];
1261 /** For more efficient resetting. */
1262 uint64_t u64ArgVars;
1263 };
1264
1265 /** Allocation bitmap for the stack. */
1266 uint32_t bmStack;
1267 /** Allocation bitmap for aVars. */
1268 uint32_t bmVars;
1269
1270 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1271 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1272 * (A shadow copy of a guest register can only be held in a one host register,
1273 * there are no duplicate copies or ambiguities like that). */
1274 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1275#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1276 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1277 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1278 * (A shadow copy of a guest register can only be held in a one host register,
1279 * there are no duplicate copies or ambiguities like that). */
1280 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1281#endif
1282
1283 /** Host register allocation tracking. */
1284 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1285#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1286 /** Host SIMD register allocation tracking. */
1287 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1288#endif
1289
1290 /** Variables and arguments. */
1291 IEMNATIVEVAR aVars[9];
1292} IEMNATIVECORESTATE;
1293/** Pointer to core state. */
1294typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1295/** Pointer to const core state. */
1296typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1297
1298/** @def IEMNATIVE_VAR_IDX_UNPACK
1299 * @returns Index into IEMNATIVECORESTATE::aVars.
1300 * @param a_idxVar Variable index w/ magic (in strict builds).
1301 */
1302/** @def IEMNATIVE_VAR_IDX_PACK
1303 * @returns Variable index w/ magic (in strict builds).
1304 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1305 */
1306#ifdef VBOX_STRICT
1307# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1308# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1309# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1310# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1311# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1312#else
1313# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1314# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1315#endif
1316
1317
1318#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1319/** Clear the dirty state of the given guest SIMD register. */
1320# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1321 do { \
1322 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1323 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1324 } while (0)
1325
1326/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1327# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1328 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1329/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1330# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1331 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1332/** Returns whether the given guest SIMD register is dirty. */
1333# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1334 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1335
1336/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1337# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1338 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1339/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1340# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1341 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1342
1343/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1344# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1345 /** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1346# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1347/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1348# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1349/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1350# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1351#endif
1352
1353
1354/**
1355 * Conditional stack entry.
1356 */
1357typedef struct IEMNATIVECOND
1358{
1359 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1360 bool fInElse;
1361 /** The label for the IEM_MC_ELSE. */
1362 uint32_t idxLabelElse;
1363 /** The label for the IEM_MC_ENDIF. */
1364 uint32_t idxLabelEndIf;
1365 /** The initial state snapshot as the if-block starts executing. */
1366 IEMNATIVECORESTATE InitialState;
1367 /** The state snapshot at the end of the if-block. */
1368 IEMNATIVECORESTATE IfFinalState;
1369} IEMNATIVECOND;
1370/** Pointer to a condition stack entry. */
1371typedef IEMNATIVECOND *PIEMNATIVECOND;
1372
1373
1374/**
1375 * Native recompiler state.
1376 */
1377typedef struct IEMRECOMPILERSTATE
1378{
1379 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1380 * IEMNATIVEINSTR units. */
1381 uint32_t cInstrBufAlloc;
1382#ifdef VBOX_STRICT
1383 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1384 uint32_t offInstrBufChecked;
1385#else
1386 uint32_t uPadding1; /* We don't keep track of the size here... */
1387#endif
1388 /** Fixed temporary code buffer for native recompilation. */
1389 PIEMNATIVEINSTR pInstrBuf;
1390
1391 /** Bitmaps with the label types used. */
1392 uint64_t bmLabelTypes;
1393 /** Actual number of labels in paLabels. */
1394 uint32_t cLabels;
1395 /** Max number of entries allowed in paLabels before reallocating it. */
1396 uint32_t cLabelsAlloc;
1397 /** Labels defined while recompiling (referenced by fixups). */
1398 PIEMNATIVELABEL paLabels;
1399 /** Array with indexes of unique labels (uData always 0). */
1400 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1401
1402 /** Actual number of fixups paFixups. */
1403 uint32_t cFixups;
1404 /** Max number of entries allowed in paFixups before reallocating it. */
1405 uint32_t cFixupsAlloc;
1406 /** Buffer used by the recompiler for recording fixups when generating code. */
1407 PIEMNATIVEFIXUP paFixups;
1408
1409#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1410 /** Actual number of fixups in paTbExitFixups. */
1411 uint32_t cTbExitFixups;
1412 /** Max number of entries allowed in paTbExitFixups before reallocating it. */
1413 uint32_t cTbExitFixupsAlloc;
1414 /** Buffer used by the recompiler for recording fixups when generating code. */
1415 PIEMNATIVEEXITFIXUP paTbExitFixups;
1416#endif
1417
1418#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1419 /** Number of debug info entries allocated for pDbgInfo. */
1420 uint32_t cDbgInfoAlloc;
1421 uint32_t uPadding;
1422 /** Debug info. */
1423 PIEMTBDBG pDbgInfo;
1424#endif
1425
1426#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1427 /** The current call index (liveness array and threaded calls in TB). */
1428 uint32_t idxCurCall;
1429 /** Number of liveness entries allocated. */
1430 uint32_t cLivenessEntriesAlloc;
1431 /** Liveness entries for all the calls in the TB begin recompiled.
1432 * The entry for idxCurCall contains the info for what the next call will
1433 * require wrt registers. (Which means the last entry is the initial liveness
1434 * state.) */
1435 PIEMLIVENESSENTRY paLivenessEntries;
1436#endif
1437
1438 /** The translation block being recompiled. */
1439 PCIEMTB pTbOrg;
1440 /** The VMCPU structure of the EMT. */
1441 PVMCPUCC pVCpu;
1442
1443 /** Condition sequence number (for generating unique labels). */
1444 uint16_t uCondSeqNo;
1445 /** Check IRQ seqeunce number (for generating unique labels). */
1446 uint16_t uCheckIrqSeqNo;
1447 /** TLB load sequence number (for generating unique labels). */
1448 uint16_t uTlbSeqNo;
1449 /** The current condition stack depth (aCondStack). */
1450 uint8_t cCondDepth;
1451
1452 /** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1453 uint8_t cArgsX;
1454 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1455 uint32_t fCImpl;
1456 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1457 uint32_t fMc;
1458 /** The expected IEMCPU::fExec value for the current call/instruction. */
1459 uint32_t fExec;
1460#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1461 /** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1462 * we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1463 *
1464 * This is an optimization because these control registers can only be changed from
1465 * by calling a C helper we can catch. Should reduce the number of instructions in a TB
1466 * consisting of multiple SIMD instructions.
1467 */
1468 uint32_t fSimdRaiseXcptChecksEmitted;
1469#endif
1470 /** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1471 uint32_t idxLastCheckIrqCallNo;
1472
1473 /** Core state requiring care with branches. */
1474 IEMNATIVECORESTATE Core;
1475
1476 /** The condition nesting stack. */
1477 IEMNATIVECOND aCondStack[2];
1478
1479#ifndef IEM_WITH_THROW_CATCH
1480 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1481 * for recompilation error handling. */
1482 jmp_buf JmpBuf;
1483#endif
1484} IEMRECOMPILERSTATE;
1485/** Pointer to a native recompiler state. */
1486typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1487
1488
1489/** @def IEMNATIVE_TRY_SETJMP
1490 * Wrapper around setjmp / try, hiding all the ugly differences.
1491 *
1492 * @note Use with extreme care as this is a fragile macro.
1493 * @param a_pReNative The native recompile state.
1494 * @param a_rcTarget The variable that should receive the status code in case
1495 * of a longjmp/throw.
1496 */
1497/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1498 * Start wrapper for catch / setjmp-else.
1499 *
1500 * This will set up a scope.
1501 *
1502 * @note Use with extreme care as this is a fragile macro.
1503 * @param a_pReNative The native recompile state.
1504 * @param a_rcTarget The variable that should receive the status code in case
1505 * of a longjmp/throw.
1506 */
1507/** @def IEMNATIVE_CATCH_LONGJMP_END
1508 * End wrapper for catch / setjmp-else.
1509 *
1510 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1511 * up the state.
1512 *
1513 * @note Use with extreme care as this is a fragile macro.
1514 * @param a_pReNative The native recompile state.
1515 */
1516/** @def IEMNATIVE_DO_LONGJMP
1517 *
1518 * Wrapper around longjmp / throw.
1519 *
1520 * @param a_pReNative The native recompile state.
1521 * @param a_rc The status code jump back with / throw.
1522 */
1523#ifdef IEM_WITH_THROW_CATCH
1524# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1525 a_rcTarget = VINF_SUCCESS; \
1526 try
1527# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1528 catch (int rcThrown) \
1529 { \
1530 a_rcTarget = rcThrown
1531# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1532 } \
1533 ((void)0)
1534# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1535#else /* !IEM_WITH_THROW_CATCH */
1536# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1537 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1538# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1539 else \
1540 { \
1541 ((void)0)
1542# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1543 }
1544# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1545#endif /* !IEM_WITH_THROW_CATCH */
1546
1547
1548/**
1549 * Native recompiler worker for a threaded function.
1550 *
1551 * @returns New code buffer offset; throws VBox status code in case of a failure.
1552 * @param pReNative The native recompiler state.
1553 * @param off The current code buffer offset.
1554 * @param pCallEntry The threaded call entry.
1555 *
1556 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1557 */
1558typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1559/** Pointer to a native recompiler worker for a threaded function. */
1560typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1561
1562/** Defines a native recompiler worker for a threaded function.
1563 * @see FNIEMNATIVERECOMPFUNC */
1564#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1565 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1566
1567/** Prototypes a native recompiler function for a threaded function.
1568 * @see FNIEMNATIVERECOMPFUNC */
1569#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1570
1571
1572/**
1573 * Native recompiler liveness analysis worker for a threaded function.
1574 *
1575 * @param pCallEntry The threaded call entry.
1576 * @param pIncoming The incoming liveness state entry.
1577 * @param pOutgoing The outgoing liveness state entry.
1578 */
1579typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1580 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1581/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1582typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1583
1584/** Defines a native recompiler liveness analysis worker for a threaded function.
1585 * @see FNIEMNATIVELIVENESSFUNC */
1586#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1587 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1588
1589/** Prototypes a native recompiler liveness analysis function for a threaded function.
1590 * @see FNIEMNATIVELIVENESSFUNC */
1591#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1592
1593
1594/** Define a native recompiler helper function, safe to call from the TB code. */
1595#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1596 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1597/** Prototype a native recompiler helper function, safe to call from the TB code. */
1598#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1599 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1600/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1601#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1602 a_RetType (VBOXCALL *a_Name) a_ArgList
1603
1604
1605#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1606DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1607DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1608 uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1609# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1610DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1611 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1612 uint8_t idxHstSimdReg = UINT8_MAX,
1613 uint8_t idxHstSimdRegPrev = UINT8_MAX);
1614# endif
1615# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1616DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1617 uint8_t idxGstReg, uint8_t idxHstReg);
1618DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1619 uint64_t fGstReg);
1620# endif
1621DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1622 uint32_t offPc, uint32_t cInstrSkipped);
1623#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1624
1625DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1626 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1627DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1628DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1629 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1630#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1631DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1632#endif
1633DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1634
1635DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1636DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1637 bool fPreferVolatile = true);
1638DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1639 bool fPreferVolatile = true);
1640DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1641 IEMNATIVEGSTREG enmGstReg,
1642 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1643 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1644DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1645 IEMNATIVEGSTREG enmGstReg);
1646
1647DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1648DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1649#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) || (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1650DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1651 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1652# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1653DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1654 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1655# endif
1656#endif
1657DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1658DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1659DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1660DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1661#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1662DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1663# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1664DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1665# endif
1666#endif
1667DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1668DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1669 uint32_t fKeepVars = 0);
1670DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1671DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1672DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1673 uint32_t fHstRegsActiveShadows);
1674#ifdef VBOX_STRICT
1675DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1676#endif
1677DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1678 uint64_t fGstSimdShwExcept);
1679#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1680DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1681#endif
1682#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1683DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1684DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1685DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1686#endif
1687
1688
1689#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1690DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1691DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1692 bool fPreferVolatile = true);
1693DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1694 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1695 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1696 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1697 bool fNoVolatileRegs = false);
1698DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1699DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1700DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1701 IEMNATIVEGSTSIMDREG enmGstSimdReg);
1702DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1703 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1704 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1705#endif
1706
1707DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1708DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1709DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1710DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1711DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1712DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1713DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1714DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1715DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1716 IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1717DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1718DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1719 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1720#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1721DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1722 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1723#endif
1724DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1725 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1726DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1727 uint32_t fHstRegsNotToSave);
1728DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1729 uint32_t fHstRegsNotToSave);
1730DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1731DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1732
1733DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1734 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1735#ifdef VBOX_STRICT
1736DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1737DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1738 IEMNATIVEGSTREG enmGstReg);
1739# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1740DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1741 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1742 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1743# endif
1744DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1745#endif
1746#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1747DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1748#endif
1749DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1750DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1751DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1752 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1753 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1754DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1755 PCIEMTHRDEDCALLENTRY pCallEntry);
1756DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1757 uint8_t idxAddrReg, uint8_t idxInstr);
1758DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1759 uint8_t idxAddrReg, uint8_t idxInstr);
1760DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1761 IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1762
1763
1764IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1765IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1766IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1767IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1768IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1769IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1770IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1771IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1772IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1773IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1774IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1775IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1776IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1777
1778IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1779IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1780IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1781IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1782IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1783IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1784IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1785IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1786IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1787IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1788#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1789IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1790IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1791IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1792IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1793IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1794#endif
1795IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1796IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1797IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1798IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1799#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1800IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1801IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1802IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1803IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1804#endif
1805IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1806IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1807IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1808IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1809IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1810IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1811IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1812
1813IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1814IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1815IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1816IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1817IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1818IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1819IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1820IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1821IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1822IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1823#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1824IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1825IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1826IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1827IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1828IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1829#endif
1830IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1831IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1832IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1833IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1834#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1835IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1836IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1837IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1838IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1839#endif
1840IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1841IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1842IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1843IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1844IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1845IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1846IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1847
1848IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1849IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1850IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1851IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1852IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1853IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1854IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1855IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1856IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1857IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1858IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1859IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1860IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1861IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1862IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1863IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1864IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1865IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1866IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1867IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1868IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1869IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1870
1871IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1872IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1873IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1874IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1875IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1876IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1877IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1878IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1879IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1880IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1881IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1882IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1883IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1884IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1885IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1886IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1887IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1888IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1889IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1890IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1891IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1892IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1893
1894IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1895IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1896IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1897IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1898
1899
1900/**
1901 * Info about shadowed guest register values.
1902 * @see IEMNATIVEGSTREG
1903 */
1904typedef struct IEMANTIVEGSTREGINFO
1905{
1906 /** Offset in VMCPU. */
1907 uint32_t off;
1908 /** The field size. */
1909 uint8_t cb;
1910 /** Name (for logging). */
1911 const char *pszName;
1912} IEMANTIVEGSTREGINFO;
1913extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1914extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1915extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1916extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1917extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1918
1919
1920
1921/**
1922 * Ensures that there is sufficient space in the instruction output buffer.
1923 *
1924 * This will reallocate the buffer if needed and allowed.
1925 *
1926 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1927 * allocation size.
1928 *
1929 * @returns Pointer to the instruction output buffer on success; throws VBox
1930 * status code on failure, so no need to check it.
1931 * @param pReNative The native recompile state.
1932 * @param off Current instruction offset. Works safely for UINT32_MAX
1933 * as well.
1934 * @param cInstrReq Number of instruction about to be added. It's okay to
1935 * overestimate this a bit.
1936 */
1937DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1938iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1939{
1940 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1941 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1942 {
1943#ifdef VBOX_STRICT
1944 pReNative->offInstrBufChecked = offChecked;
1945#endif
1946 return pReNative->pInstrBuf;
1947 }
1948 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1949}
1950
1951/**
1952 * Checks that we didn't exceed the space requested in the last
1953 * iemNativeInstrBufEnsure() call.
1954 */
1955#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1956 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1957 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1958
1959/**
1960 * Checks that a variable index is valid.
1961 */
1962#ifdef IEMNATIVE_VAR_IDX_MAGIC
1963# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1964 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1965 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1966 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1967 ("%s=%#x\n", #a_idxVar, a_idxVar))
1968#else
1969# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1970 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1971 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1972#endif
1973
1974/**
1975 * Checks that a variable index is valid and that the variable is assigned the
1976 * correct argument number.
1977 * This also adds a RT_NOREF of a_idxVar.
1978 */
1979#ifdef IEMNATIVE_VAR_IDX_MAGIC
1980# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1981 RT_NOREF_PV(a_idxVar); \
1982 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1983 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1984 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1985 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1986 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1987 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1988 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1989 a_uArgNo)); \
1990 } while (0)
1991#else
1992# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1993 RT_NOREF_PV(a_idxVar); \
1994 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1995 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1996 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1997 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1998 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1999 } while (0)
2000#endif
2001
2002
2003/**
2004 * Checks that a variable has the expected size.
2005 */
2006#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2007 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2008 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2009 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
2010
2011
2012/**
2013 * Calculates the stack address of a variable as a [r]BP displacement value.
2014 */
2015DECL_FORCE_INLINE(int32_t)
2016iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2017{
2018 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2019 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2020}
2021
2022
2023/**
2024 * Releases the variable's register.
2025 *
2026 * The register must have been previously acquired calling
2027 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2028 * iemNativeVarRegisterSetAndAcquire().
2029 */
2030DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2031{
2032 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2033 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2034 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2035}
2036
2037
2038#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2039DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2040{
2041 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2042 iemNativeVarRegisterRelease(pReNative, idxVar);
2043}
2044#endif
2045
2046
2047/**
2048 * Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2049 *
2050 * @returns The flush mask.
2051 * @param fCImpl The IEM_CIMPL_F_XXX flags.
2052 * @param fGstShwFlush The starting flush mask.
2053 */
2054DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2055{
2056 if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2057 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2058 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2059 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2060 if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2061 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2062 | RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2063 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2064 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2065 else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2066 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2067 if (fCImpl & (IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_STATUS_FLAGS | IEM_CIMPL_F_INHIBIT_SHADOW))
2068 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
2069 return fGstShwFlush;
2070}
2071
2072
2073/** Number of hidden arguments for CIMPL calls.
2074 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2075#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2076# define IEM_CIMPL_HIDDEN_ARGS 3
2077#else
2078# define IEM_CIMPL_HIDDEN_ARGS 2
2079#endif
2080
2081
2082#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2083/** Number of hidden arguments for SSE_AIMPL calls. */
2084# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2085/** Number of hidden arguments for AVX_AIMPL calls. */
2086# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2087#endif
2088
2089
2090#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2091
2092# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2093/**
2094 * Helper for iemNativeLivenessGetStateByGstReg.
2095 *
2096 * @returns IEMLIVENESS_STATE_XXX
2097 * @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2098 * ORed together.
2099 */
2100DECL_FORCE_INLINE(uint32_t)
2101iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2102{
2103 /* INPUT trumps anything else. */
2104 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2105 return IEMLIVENESS_STATE_INPUT;
2106
2107 /* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2108 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2109 {
2110 /* If not all sub-fields are clobbered they must be considered INPUT. */
2111 if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) | RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2112 return IEMLIVENESS_STATE_INPUT;
2113 return IEMLIVENESS_STATE_CLOBBERED;
2114 }
2115
2116 /* XCPT_OR_CALL trumps UNUSED. */
2117 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2118 return IEMLIVENESS_STATE_XCPT_OR_CALL;
2119
2120 return IEMLIVENESS_STATE_UNUSED;
2121}
2122# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2123
2124
2125DECL_FORCE_INLINE(uint32_t)
2126iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2127{
2128# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2129 return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2130 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2131# else
2132 return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2133 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2134 | (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2135 | (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2136# endif
2137}
2138
2139
2140DECL_FORCE_INLINE(uint32_t)
2141iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2142{
2143 uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2144 if (enmGstReg == kIemNativeGstReg_EFlags)
2145 {
2146 /* Merge the eflags states to one. */
2147# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2148 uRet = RT_BIT_32(uRet);
2149 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflCf | (pLivenessEntry->Bit1.fEflCf << 1));
2150 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflPf | (pLivenessEntry->Bit1.fEflPf << 1));
2151 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflAf | (pLivenessEntry->Bit1.fEflAf << 1));
2152 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflZf | (pLivenessEntry->Bit1.fEflZf << 1));
2153 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflSf | (pLivenessEntry->Bit1.fEflSf << 1));
2154 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflOf | (pLivenessEntry->Bit1.fEflOf << 1));
2155 uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2156# else
2157 AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2158 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2159 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2160 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2161 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2162 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2163 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2164# endif
2165 }
2166 return uRet;
2167}
2168
2169
2170# ifdef VBOX_STRICT
2171/** For assertions only, user checks that idxCurCall isn't zerow. */
2172DECL_FORCE_INLINE(uint32_t)
2173iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2174{
2175 return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2176}
2177# endif /* VBOX_STRICT */
2178
2179#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2180
2181
2182/**
2183 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2184 */
2185DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2186{
2187 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2188 return IEM_CIMPL_HIDDEN_ARGS;
2189 if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE | IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2190 return 1;
2191 return 0;
2192}
2193
2194
2195DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2196 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2197{
2198 pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
2199
2200 pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2201 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2202 pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2203 return (uint8_t)idxReg;
2204}
2205
2206
2207
2208/*********************************************************************************************************************************
2209* Register Allocator (GPR) *
2210*********************************************************************************************************************************/
2211
2212/**
2213 * Marks host register @a idxHstReg as containing a shadow copy of guest
2214 * register @a enmGstReg.
2215 *
2216 * ASSUMES that caller has made sure @a enmGstReg is not associated with any
2217 * host register before calling.
2218 */
2219DECL_FORCE_INLINE(void)
2220iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2221{
2222 Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2223 Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2224 Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2225
2226 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2227 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2228 pReNative->Core.bmGstRegShadows |= RT_BIT_64(enmGstReg);
2229 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxHstReg);
2230#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2231 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2232 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2233#else
2234 RT_NOREF(off);
2235#endif
2236}
2237
2238
2239/**
2240 * Clear any guest register shadow claims from @a idxHstReg.
2241 *
2242 * The register does not need to be shadowing any guest registers.
2243 */
2244DECL_FORCE_INLINE(void)
2245iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2246{
2247 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2248 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2249 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2250 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2251 == RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2252#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2253 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2254#endif
2255
2256#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2257 uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2258 if (fGstRegs)
2259 {
2260 Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2261 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2262 while (fGstRegs)
2263 {
2264 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2265 fGstRegs &= ~RT_BIT_64(iGstReg);
2266 iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2267 }
2268 }
2269#else
2270 RT_NOREF(off);
2271#endif
2272
2273 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2274 pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2275 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2276}
2277
2278
2279/**
2280 * Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2281 * and global overview flags.
2282 */
2283DECL_FORCE_INLINE(void)
2284iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2285{
2286 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2287 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2288 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2289 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2290 Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2291 Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2292 Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2293#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2294 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2295#endif
2296
2297#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2298 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2299 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2300#else
2301 RT_NOREF(off);
2302#endif
2303
2304 uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2305 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2306 if (!fGstRegShadowsNew)
2307 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2308 pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2309}
2310
2311
2312#if 0 /* unused */
2313/**
2314 * Clear any guest register shadow claim for @a enmGstReg.
2315 */
2316DECL_FORCE_INLINE(void)
2317iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2318{
2319 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2320 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2321 {
2322 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2323 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2324 }
2325}
2326#endif
2327
2328
2329/**
2330 * Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2331 * as the new shadow of it.
2332 *
2333 * Unlike the other guest reg shadow helpers, this does the logging for you.
2334 * However, it is the liveness state is not asserted here, the caller must do
2335 * that.
2336 */
2337DECL_FORCE_INLINE(void)
2338iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2339 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2340{
2341 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2342 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2343 {
2344 uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2345 Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2346 if (idxHstRegOld == idxHstRegNew)
2347 return;
2348 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2349 g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2350 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2351 }
2352 else
2353 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2354 g_aGstShadowInfo[enmGstReg].pszName));
2355 iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2356}
2357
2358
2359/**
2360 * Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2361 * to @a idxRegTo.
2362 */
2363DECL_FORCE_INLINE(void)
2364iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2365 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2366{
2367 Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2368 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2369 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2370 == pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2371 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2372 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2373 == pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2374 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2375 == RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2376
2377 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2378 pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2379 if (!fGstRegShadowsFrom)
2380 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2381 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxRegTo);
2382 pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows |= RT_BIT_64(enmGstReg);
2383 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2384#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2385 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2386 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2387#else
2388 RT_NOREF(off);
2389#endif
2390}
2391
2392
2393/**
2394 * Flushes any delayed guest register writes.
2395 *
2396 * This must be called prior to calling CImpl functions and any helpers that use
2397 * the guest state (like raising exceptions) and such.
2398 *
2399 * This optimization has not yet been implemented. The first target would be
2400 * RIP updates, since these are the most common ones.
2401 *
2402 * @note This function does not flush any shadowing information for guest registers. This needs to be done by
2403 * the caller if it wishes to do so.
2404 */
2405DECL_INLINE_THROW(uint32_t)
2406iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2407{
2408#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2409 uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2410#else
2411 uint64_t const bmGstRegShadowDirty = 0;
2412#endif
2413#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2414 uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2415 & ~fGstSimdShwExcept;
2416#else
2417 uint64_t const bmGstSimdRegShadowDirty = 0;
2418#endif
2419#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2420 uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2421#else
2422 uint64_t const fWritebackPc = 0;
2423#endif
2424 if (bmGstRegShadowDirty | bmGstSimdRegShadowDirty | fWritebackPc)
2425 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2426
2427 return off;
2428}
2429
2430
2431
2432/*********************************************************************************************************************************
2433* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2434*********************************************************************************************************************************/
2435
2436#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2437
2438DECL_FORCE_INLINE(uint8_t)
2439iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2440 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2441{
2442 pReNative->Core.bmHstSimdRegs |= RT_BIT_32(idxSimdReg);
2443
2444 pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2445 pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2446 pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2447 return idxSimdReg;
2448}
2449
2450
2451/**
2452 * Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2453 * SIMD register @a enmGstSimdReg.
2454 *
2455 * ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2456 * host register before calling.
2457 */
2458DECL_FORCE_INLINE(void)
2459iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2460 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2461{
2462 Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2463 Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2464 Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2465
2466 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2467 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2468 pReNative->Core.bmGstSimdRegShadows |= RT_BIT_64(enmGstSimdReg);
2469 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxHstSimdReg);
2470#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2471 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2472 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2473#else
2474 RT_NOREF(off);
2475#endif
2476}
2477
2478
2479/**
2480 * Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2481 * to @a idxSimdRegTo.
2482 */
2483DECL_FORCE_INLINE(void)
2484iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2485 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2486{
2487 Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2488 Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2489 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2490 == pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2491 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2492 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2493 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2494 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2495 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2496 Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2497 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2498
2499 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2500 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2501 if (!fGstRegShadowsFrom)
2502 {
2503 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2504 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2505 }
2506 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxSimdRegTo);
2507 pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2508 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2509#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2510 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2511 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2512#else
2513 RT_NOREF(off);
2514#endif
2515}
2516
2517
2518/**
2519 * Clear any guest register shadow claims from @a idxHstSimdReg.
2520 *
2521 * The register does not need to be shadowing any guest registers.
2522 */
2523DECL_FORCE_INLINE(void)
2524iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2525{
2526 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2527 == pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2528 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2529 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2530 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2531 Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2532 && !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2533
2534#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2535 uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2536 if (fGstRegs)
2537 {
2538 Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2539 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2540 while (fGstRegs)
2541 {
2542 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2543 fGstRegs &= ~RT_BIT_64(iGstReg);
2544 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2545 }
2546 }
2547#else
2548 RT_NOREF(off);
2549#endif
2550
2551 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2552 pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2553 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2554 pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2555}
2556
2557#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2558
2559
2560#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2561/**
2562 * Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2563 */
2564DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2565{
2566 if (pReNative->Core.offPc)
2567 return iemNativeEmitPcWritebackSlow(pReNative, off);
2568 return off;
2569}
2570#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2571
2572
2573#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2574/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2575 * it saves us the trouble of a hidden parameter on MSC/amd64. */
2576# ifdef RT_ARCH_AMD64
2577extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2578# elif defined(RT_ARCH_ARM64)
2579extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2580# endif
2581#endif
2582
2583#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2584
2585/** @} */
2586
2587#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2588
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette