IEMN8veRecompiler.h@ 104051

Last change on this file since 104051 was 104047, checked in by vboxsync, 11 months ago

VMM/IEM: Straighten the code flushing dirty guest registers, bugref:10614 bugref:10629.

This removes the code which flushes any guest to host register shadowing information
as in most instances of the call flushing this information is not necessary anyway.
This option was introduced for the SIMD register flushing when the SIMD register allocator was implemented
but didn't age well now that there is support for delaying GPR writes as well. Instead iemNativeRegFlushPendingWrites()
will only do the register writeback and leave the rest to the caller.
For SIMD registers the shadowing is now flushed in iemNativeVarSaveVolatileRegsPreHlpCall() and
iemNativeRegMoveAndFreeAndFlushAtCall() for anything which calls an external helper (TLB miss, certain builtin functions
for loading the TLB, calling instruction helpers, etc.).

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 123.4 KB

Line
1	/* $Id: IEMN8veRecompiler.h 104047 2024-03-25 18:35:44Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/*
226	* ARM64 has 32 128-bit registers only, in order to support emulating 256-bit registers we pair
227	* two real registers statically to one virtual for now, leaving us with only 16 256-bit registers.
228	* We always pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed
229	* and the register allocator assumes that it will be always free when the lower is picked.
230	*
231	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't touch them in order to avoid
232	* having to save and restore them in the prologue/epilogue.
233	*/
234	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
235	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
236	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
237	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
238	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
239	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
240	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q1))
252	# endif
253	# endif
254
255	#else
256	# error "port me"
257	#endif
258	/** @} */
259
260	/** @name Call related registers.
261	* @{ */
262	/** @def IEMNATIVE_CALL_RET_GREG
263	* The return value register. */
264	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
265	* Number of arguments in registers. */
266	/** @def IEMNATIVE_CALL_ARG0_GREG
267	* The general purpose register carrying argument \#0. */
268	/** @def IEMNATIVE_CALL_ARG1_GREG
269	* The general purpose register carrying argument \#1. */
270	/** @def IEMNATIVE_CALL_ARG2_GREG
271	* The general purpose register carrying argument \#2. */
272	/** @def IEMNATIVE_CALL_ARG3_GREG
273	* The general purpose register carrying argument \#3. */
274	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
275	* Mask of registers the callee will not save and may trash. */
276	#ifdef RT_ARCH_AMD64
277	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
278
279	# ifdef RT_OS_WINDOWS
280	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
281	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
282	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
283	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
289	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
290	\| RT_BIT_32(X86_GREG_xCX) \
291	\| RT_BIT_32(X86_GREG_xDX) \
292	\| RT_BIT_32(X86_GREG_x8) \
293	\| RT_BIT_32(X86_GREG_x9) \
294	\| RT_BIT_32(X86_GREG_x10) \
295	\| RT_BIT_32(X86_GREG_x11) )
296	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
297	/* xmm0 - xmm5 are marked as volatile. */
298	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
299	# endif
300
301	# else
302	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
303	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
304	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
305	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
306	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
307	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
308	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
309	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
310	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
311	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
312	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
313	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
314	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
315	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
316	\| RT_BIT_32(X86_GREG_xCX) \
317	\| RT_BIT_32(X86_GREG_xDX) \
318	\| RT_BIT_32(X86_GREG_xDI) \
319	\| RT_BIT_32(X86_GREG_xSI) \
320	\| RT_BIT_32(X86_GREG_x8) \
321	\| RT_BIT_32(X86_GREG_x9) \
322	\| RT_BIT_32(X86_GREG_x10) \
323	\| RT_BIT_32(X86_GREG_x11) )
324	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
325	/* xmm0 - xmm15 are marked as volatile. */
326	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
327	# endif
328	# endif
329
330	#elif defined(RT_ARCH_ARM64)
331	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
332	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
333	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
334	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
335	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
336	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
337	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
338	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
339	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
340	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
341	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
342	\| RT_BIT_32(ARMV8_A64_REG_X1) \
343	\| RT_BIT_32(ARMV8_A64_REG_X2) \
344	\| RT_BIT_32(ARMV8_A64_REG_X3) \
345	\| RT_BIT_32(ARMV8_A64_REG_X4) \
346	\| RT_BIT_32(ARMV8_A64_REG_X5) \
347	\| RT_BIT_32(ARMV8_A64_REG_X6) \
348	\| RT_BIT_32(ARMV8_A64_REG_X7) )
349	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
350	\| RT_BIT_32(ARMV8_A64_REG_X1) \
351	\| RT_BIT_32(ARMV8_A64_REG_X2) \
352	\| RT_BIT_32(ARMV8_A64_REG_X3) \
353	\| RT_BIT_32(ARMV8_A64_REG_X4) \
354	\| RT_BIT_32(ARMV8_A64_REG_X5) \
355	\| RT_BIT_32(ARMV8_A64_REG_X6) \
356	\| RT_BIT_32(ARMV8_A64_REG_X7) \
357	\| RT_BIT_32(ARMV8_A64_REG_X8) \
358	\| RT_BIT_32(ARMV8_A64_REG_X9) \
359	\| RT_BIT_32(ARMV8_A64_REG_X10) \
360	\| RT_BIT_32(ARMV8_A64_REG_X11) \
361	\| RT_BIT_32(ARMV8_A64_REG_X12) \
362	\| RT_BIT_32(ARMV8_A64_REG_X13) \
363	\| RT_BIT_32(ARMV8_A64_REG_X14) \
364	\| RT_BIT_32(ARMV8_A64_REG_X15) \
365	\| RT_BIT_32(ARMV8_A64_REG_X16) \
366	\| RT_BIT_32(ARMV8_A64_REG_X17) )
367	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
368	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
369	* so to simplify our life a bit we just mark everything as volatile. */
370	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
371	# endif
372
373	#endif
374
375	/** This is the maximum argument count we'll ever be needing. */
376	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
377	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
378	#else
379	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
380	#endif
381	/** @} */
382
383
384	/** @def IEMNATIVE_HST_GREG_COUNT
385	* Number of host general purpose registers we tracker. */
386	/** @def IEMNATIVE_HST_GREG_MASK
387	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
388	* inverted register masks and such to get down to a correct set of regs. */
389	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
390	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
391	* Number of host SIMD registers we track. */
392	/** @def IEMNATIVE_HST_SIMD_REG_MASK
393	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
394	* inverted register masks and such to get down to a correct set of regs. */
395	#endif
396	#ifdef RT_ARCH_AMD64
397	# define IEMNATIVE_HST_GREG_COUNT 16
398	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
399
400	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
401	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
402	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
403	# endif
404
405	#elif defined(RT_ARCH_ARM64)
406	# define IEMNATIVE_HST_GREG_COUNT 32
407	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
408
409	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
410	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
411	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
412	# endif
413
414	#else
415	# error "Port me!"
416	#endif
417
418
419	/** Native code generator label types. */
420	typedef enum
421	{
422	kIemNativeLabelType_Invalid = 0,
423	/*
424	* Labels w/o data, only once instance per TB.
425	*
426	* Note! Jumps to these requires instructions that are capable of spanning
427	* the max TB length.
428	*/
429	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
430	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
431	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
432	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
433	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
434	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
435	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
436	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
437	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
438	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
439	kIemNativeLabelType_ObsoleteTb,
440	kIemNativeLabelType_NeedCsLimChecking,
441	kIemNativeLabelType_CheckBranchMiss,
442	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
443	/* Manually defined labels. */
444	kIemNativeLabelType_Return,
445	kIemNativeLabelType_ReturnBreak,
446	kIemNativeLabelType_ReturnWithFlags,
447	kIemNativeLabelType_NonZeroRetOrPassUp,
448	/** The last fixup for branches that can span almost the whole TB length. */
449	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
450
451	/*
452	* Labels with data, potentially multiple instances per TB:
453	*
454	* These are localized labels, so no fixed jump type restrictions here.
455	*/
456	kIemNativeLabelType_FirstWithMultipleInstances,
457	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
458	kIemNativeLabelType_Else,
459	kIemNativeLabelType_Endif,
460	kIemNativeLabelType_CheckIrq,
461	kIemNativeLabelType_TlbLookup,
462	kIemNativeLabelType_TlbMiss,
463	kIemNativeLabelType_TlbDone,
464	kIemNativeLabelType_End
465	} IEMNATIVELABELTYPE;
466
467	/** Native code generator label definition. */
468	typedef struct IEMNATIVELABEL
469	{
470	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
471	* the epilog. */
472	uint32_t off;
473	/** The type of label (IEMNATIVELABELTYPE). */
474	uint16_t enmType;
475	/** Additional label data, type specific. */
476	uint16_t uData;
477	} IEMNATIVELABEL;
478	/** Pointer to a label. */
479	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
480
481
482	/** Native code generator fixup types. */
483	typedef enum
484	{
485	kIemNativeFixupType_Invalid = 0,
486	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
487	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
488	kIemNativeFixupType_Rel32,
489	#elif defined(RT_ARCH_ARM64)
490	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
491	kIemNativeFixupType_RelImm26At0,
492	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
493	kIemNativeFixupType_RelImm19At5,
494	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
495	kIemNativeFixupType_RelImm14At5,
496	#endif
497	kIemNativeFixupType_End
498	} IEMNATIVEFIXUPTYPE;
499
500	/** Native code generator fixup. */
501	typedef struct IEMNATIVEFIXUP
502	{
503	/** Code offset of the fixup location. */
504	uint32_t off;
505	/** The IEMNATIVELABEL this is a fixup for. */
506	uint16_t idxLabel;
507	/** The fixup type (IEMNATIVEFIXUPTYPE). */
508	uint8_t enmType;
509	/** Addend or other data. */
510	int8_t offAddend;
511	} IEMNATIVEFIXUP;
512	/** Pointer to a native code generator fixup. */
513	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
514
515
516	/**
517	* One bit of the state.
518	*
519	* Each register state takes up two bits. We keep the two bits in two separate
520	* 64-bit words to simplify applying them to the guest shadow register mask in
521	* the register allocator.
522	*/
523	typedef union IEMLIVENESSBIT
524	{
525	uint64_t bm64;
526	RT_GCC_EXTENSION struct
527	{ /* bit no */
528	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
529	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
530	uint64_t fCr0 : 1; /*< 0x11 / 17: /
531	uint64_t fFcw : 1; /*< 0x12 / 18: /
532	uint64_t fFsw : 1; /*< 0x13 / 19: /
533	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
534	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
535	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
536	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
537	uint64_t fCr4 : 1; /*< 0x2c / 44: /
538	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
539	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
540	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
541	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
542	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
543	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
544	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
545	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
546	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
547	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
548	};
549	} IEMLIVENESSBIT;
550	AssertCompileSize(IEMLIVENESSBIT, 8);
551
552	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
553	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
554	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
555	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
556	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
557	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
558	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
559
560
561	/**
562	* A liveness state entry.
563	*
564	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
565	* Once we add a SSE register shadowing, we'll add another 64-bit element for
566	* that.
567	*/
568	typedef union IEMLIVENESSENTRY
569	{
570	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
571	uint64_t bm64[16 / 8];
572	uint16_t bm32[16 / 4];
573	uint16_t bm16[16 / 2];
574	uint8_t bm8[ 16 / 1];
575	IEMLIVENESSBIT aBits[2];
576	#else
577	uint64_t bm64[32 / 8];
578	uint16_t bm32[32 / 4];
579	uint16_t bm16[32 / 2];
580	uint8_t bm8[ 32 / 1];
581	IEMLIVENESSBIT aBits[4];
582	#endif
583	RT_GCC_EXTENSION struct
584	{
585	/** Bit \#0 of the register states. */
586	IEMLIVENESSBIT Bit0;
587	/** Bit \#1 of the register states. */
588	IEMLIVENESSBIT Bit1;
589	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
590	/** Bit \#2 of the register states. */
591	IEMLIVENESSBIT Bit2;
592	/** Bit \#3 of the register states. */
593	IEMLIVENESSBIT Bit3;
594	#endif
595	};
596	} IEMLIVENESSENTRY;
597	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
598	AssertCompileSize(IEMLIVENESSENTRY, 16);
599	#else
600	AssertCompileSize(IEMLIVENESSENTRY, 32);
601	#endif
602	/** Pointer to a liveness state entry. */
603	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
604	/** Pointer to a const liveness state entry. */
605	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
606
607	/** @name 64-bit value masks for IEMLIVENESSENTRY.
608	* @{ / / 0xzzzzyyyyxxxxwwww */
609	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
610
611	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
612	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
613	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
614
615	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
616	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
617	#endif
618
619	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
620	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
621
622	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
623	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
624	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
625	#endif
626	/** @} */
627
628
629	/** @name The liveness state for a register.
630	*
631	* The state values have been picked to with state accumulation in mind (what
632	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
633	* performance critical work done with the values.
634	*
635	* This is a compressed state that only requires 2 bits per register.
636	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
637	* 1. the incoming state from the following call,
638	* 2. the outgoing state for this call,
639	* 3. mask of the entries set in the 2nd.
640	*
641	* The mask entry (3rd one above) will be used both when updating the outgoing
642	* state and when merging in incoming state for registers not touched by the
643	* current call.
644	*
645	* @{ */
646	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
647	/** The register will be clobbered and the current value thrown away.
648	*
649	* When this is applied to the state (2) we'll simply be AND'ing it with the
650	* (old) mask (3) and adding the register to the mask. This way we'll
651	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
652	* IEMLIVENESS_STATE_INPUT states. */
653	# define IEMLIVENESS_STATE_CLOBBERED 0
654	/** The register is unused in the remainder of the TB.
655	*
656	* This is an initial state and can not be set by any of the
657	* iemNativeLivenessFunc_xxxx callbacks. */
658	# define IEMLIVENESS_STATE_UNUSED 1
659	/** The register value is required in a potential call or exception.
660	*
661	* This means that the register value must be calculated and is best written to
662	* the state, but that any shadowing registers can be flushed thereafter as it's
663	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
664	*
665	* It is typically applied across the board, but we preserve incoming
666	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
667	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
668	* 1. r0 = old & ~mask;
669	* 2. r0 = t1 & (t1 >> 1)'
670	* 3. state \|= r0 \| 0b10;
671	* 4. mask = ~0;
672	*/
673	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
674	/** The register value is used as input.
675	*
676	* This means that the register value must be calculated and it is best to keep
677	* it in a register. It does not need to be writtent out as such. This is the
678	* highest priority state.
679	*
680	* Whether the call modifies the register or not isn't relevant to earlier
681	* calls, so that's not recorded.
682	*
683	* When applying this state we just or in the value in the outgoing state and
684	* mask. */
685	# define IEMLIVENESS_STATE_INPUT 3
686	/** Mask of the state bits. */
687	# define IEMLIVENESS_STATE_MASK 3
688	/** The number of bits per state. */
689	# define IEMLIVENESS_STATE_BIT_COUNT 2
690	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
691	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
692	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
693	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
694	/** Check if a register clobbering is expected given the (previous) liveness state.
695	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
696	* include INPUT if the register is used in more than one place. */
697	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
698
699	/** Check if all status flags are going to be clobbered and doesn't need
700	* calculating in the current step.
701	* @param a_pCurEntry The current liveness entry. */
702	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
703	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
704
705	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
706	/** The register is not used any more. */
707	# define IEMLIVENESS_STATE_UNUSED 0
708	/** Flag: The register is required in a potential exception or call. */
709	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
710	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
711	/** Flag: The register is read. */
712	# define IEMLIVENESS_STATE_READ 2
713	# define IEMLIVENESS_BIT_READ 1
714	/** Flag: The register is written. */
715	# define IEMLIVENESS_STATE_WRITE 4
716	# define IEMLIVENESS_BIT_WRITE 2
717	/** Flag: Unconditional call (not needed, can be redefined for research). */
718	# define IEMLIVENESS_STATE_CALL 8
719	# define IEMLIVENESS_BIT_CALL 3
720	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
721	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
722	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
723	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
724	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
725
726	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
727	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
728	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
729	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
730
731	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
732	/** @} */
733
734	/** @name Liveness helpers for builtin functions and similar.
735	*
736	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
737	* own set of manimulator macros for those.
738	*
739	* @{ */
740	/** Initializing the state as all unused. */
741	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
742	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
743	do { \
744	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
745	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
746	} while (0)
747	#else
748	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
749	do { \
750	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
751	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
752	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
753	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
754	} while (0)
755	#endif
756
757	/** Initializing the outgoing state with a potential xcpt or call state.
758	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
759	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
760	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
761	do { \
762	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
763	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
764	} while (0)
765	#else
766	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
767	do { \
768	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
769	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
770	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
771	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
772	} while (0)
773	#endif
774
775	/** Adds a segment base register as input to the outgoing state. */
776	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
777	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
778	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
779	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
780	} while (0)
781	#else
782	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
783	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
784	} while (0)
785	#endif
786
787	/** Adds a segment attribute register as input to the outgoing state. */
788	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
789	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
790	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
791	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
792	} while (0)
793	#else
794	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
795	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
796	} while (0)
797	#endif
798
799	/** Adds a segment limit register as input to the outgoing state. */
800	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
801	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
802	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
803	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
804	} while (0)
805	#else
806	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
807	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
808	} while (0)
809	#endif
810
811	/** Adds a segment limit register as input to the outgoing state. */
812	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
813	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
814	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
815	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
816	} while (0)
817	#else
818	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
819	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
820	} while (0)
821	#endif
822	/** @} */
823
824	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
825	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
826	* calculated and up to date. This is to double check that we haven't skipped
827	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
828	* @note has to be placed in
829	*/
830	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
831	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
832	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
833	#else
834	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
835	#endif
836
837
838	/**
839	* Guest registers that can be shadowed in GPRs.
840	*
841	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
842	* must be placed last, as the liveness state tracks it as 7 subcomponents and
843	* we don't want to waste space here.
844	*
845	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
846	* friends as well as IEMAllN8veLiveness.cpp.
847	*/
848	typedef enum IEMNATIVEGSTREG : uint8_t
849	{
850	kIemNativeGstReg_GprFirst = 0,
851	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
852	kIemNativeGstReg_Pc,
853	kIemNativeGstReg_Cr0,
854	kIemNativeGstReg_FpuFcw,
855	kIemNativeGstReg_FpuFsw,
856	kIemNativeGstReg_SegBaseFirst,
857	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
858	kIemNativeGstReg_SegAttribFirst,
859	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
860	kIemNativeGstReg_SegLimitFirst,
861	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
862	kIemNativeGstReg_SegSelFirst,
863	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
864	kIemNativeGstReg_Cr4,
865	kIemNativeGstReg_Xcr0,
866	kIemNativeGstReg_MxCsr,
867	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
868	kIemNativeGstReg_End
869	} IEMNATIVEGSTREG;
870	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
871	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
872
873	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
874	* @{ */
875	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
876	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
877	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
878	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
879	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
880	/** @} */
881
882	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
883
884	/**
885	* Guest registers that can be shadowed in host SIMD registers.
886	*
887	* @todo r=aeichner Liveness tracking
888	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
889	*/
890	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
891	{
892	kIemNativeGstSimdReg_SimdRegFirst = 0,
893	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
894	kIemNativeGstSimdReg_End
895	} IEMNATIVEGSTSIMDREG;
896
897	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
898	* @{ */
899	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
900	/** @} */
901
902	/**
903	* The Load/store size for a SIMD guest register.
904	*/
905	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
906	{
907	/** Invalid size. */
908	kIemNativeGstSimdRegLdStSz_Invalid = 0,
909	/** Loads the low 128-bit of a guest SIMD register. */
910	kIemNativeGstSimdRegLdStSz_Low128,
911	/** Loads the high 128-bit of a guest SIMD register. */
912	kIemNativeGstSimdRegLdStSz_High128,
913	/** Loads the whole 256-bits of a guest SIMD register. */
914	kIemNativeGstSimdRegLdStSz_256,
915	/** End value. */
916	kIemNativeGstSimdRegLdStSz_End
917	} IEMNATIVEGSTSIMDREGLDSTSZ;
918
919	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
920
921	/**
922	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
923	*/
924	typedef enum IEMNATIVEGSTREGUSE
925	{
926	/** The usage is read-only, the register holding the guest register
927	* shadow copy will not be modified by the caller. */
928	kIemNativeGstRegUse_ReadOnly = 0,
929	/** The caller will update the guest register (think: PC += cbInstr).
930	* The guest shadow copy will follow the returned register. */
931	kIemNativeGstRegUse_ForUpdate,
932	/** The call will put an entirely new value in the guest register, so
933	* if new register is allocate it will be returned uninitialized. */
934	kIemNativeGstRegUse_ForFullWrite,
935	/** The caller will use the guest register value as input in a calculation
936	* and the host register will be modified.
937	* This means that the returned host register will not be marked as a shadow
938	* copy of the guest register. */
939	kIemNativeGstRegUse_Calculation
940	} IEMNATIVEGSTREGUSE;
941
942	/**
943	* Guest registers (classes) that can be referenced.
944	*/
945	typedef enum IEMNATIVEGSTREGREF : uint8_t
946	{
947	kIemNativeGstRegRef_Invalid = 0,
948	kIemNativeGstRegRef_Gpr,
949	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
950	kIemNativeGstRegRef_EFlags,
951	kIemNativeGstRegRef_MxCsr,
952	kIemNativeGstRegRef_FpuReg,
953	kIemNativeGstRegRef_MReg,
954	kIemNativeGstRegRef_XReg,
955	kIemNativeGstRegRef_X87,
956	kIemNativeGstRegRef_XState,
957	//kIemNativeGstRegRef_YReg, - doesn't work.
958	kIemNativeGstRegRef_End
959	} IEMNATIVEGSTREGREF;
960
961
962	/** Variable kinds. */
963	typedef enum IEMNATIVEVARKIND : uint8_t
964	{
965	/** Customary invalid zero value. */
966	kIemNativeVarKind_Invalid = 0,
967	/** This is either in a register or on the stack. */
968	kIemNativeVarKind_Stack,
969	/** Immediate value - loaded into register when needed, or can live on the
970	* stack if referenced (in theory). */
971	kIemNativeVarKind_Immediate,
972	/** Variable reference - loaded into register when needed, never stack. */
973	kIemNativeVarKind_VarRef,
974	/** Guest register reference - loaded into register when needed, never stack. */
975	kIemNativeVarKind_GstRegRef,
976	/** End of valid values. */
977	kIemNativeVarKind_End
978	} IEMNATIVEVARKIND;
979
980
981	/** Variable or argument. */
982	typedef struct IEMNATIVEVAR
983	{
984	/** The kind of variable. */
985	IEMNATIVEVARKIND enmKind;
986	/** The variable size in bytes. */
987	uint8_t cbVar;
988	/** The first stack slot (uint64_t), except for immediate and references
989	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
990	* has a stack slot it has been initialized and has a value. Unused variables
991	* has neither a stack slot nor a host register assignment. */
992	uint8_t idxStackSlot;
993	/** The host register allocated for the variable, UINT8_MAX if not. */
994	uint8_t idxReg;
995	/** The argument number if argument, UINT8_MAX if regular variable. */
996	uint8_t uArgNo;
997	/** If referenced, the index (unpacked) of the variable referencing this one,
998	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
999	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1000	uint8_t idxReferrerVar;
1001	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1002	* @todo not sure what this really is for... */
1003	IEMNATIVEGSTREG enmGstReg;
1004	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1005	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1006	* only valid when idxReg is not UINT8_MAX. */
1007	bool fSimdReg : 1;
1008	/** Set if the registered is currently used exclusively, false if the
1009	* variable is idle and the register can be grabbed. */
1010	bool fRegAcquired : 1;
1011	#else
1012	/** Set if the registered is currently used exclusively, false if the
1013	* variable is idle and the register can be grabbed. */
1014	bool fRegAcquired;
1015	#endif
1016
1017	union
1018	{
1019	/** kIemNativeVarKind_Immediate: The immediate value. */
1020	uint64_t uValue;
1021	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1022	uint8_t idxRefVar;
1023	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1024	struct
1025	{
1026	/** The class of register. */
1027	IEMNATIVEGSTREGREF enmClass;
1028	/** Index within the class. */
1029	uint8_t idx;
1030	} GstRegRef;
1031	} u;
1032	} IEMNATIVEVAR;
1033	/** Pointer to a variable or argument. */
1034	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1035	/** Pointer to a const variable or argument. */
1036	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1037
1038	/** What is being kept in a host register. */
1039	typedef enum IEMNATIVEWHAT : uint8_t
1040	{
1041	/** The traditional invalid zero value. */
1042	kIemNativeWhat_Invalid = 0,
1043	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1044	kIemNativeWhat_Var,
1045	/** Temporary register, this is typically freed when a MC completes. */
1046	kIemNativeWhat_Tmp,
1047	/** Call argument w/o a variable mapping. This is free (via
1048	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1049	kIemNativeWhat_Arg,
1050	/** Return status code.
1051	* @todo not sure if we need this... */
1052	kIemNativeWhat_rc,
1053	/** The fixed pVCpu (PVMCPUCC) register.
1054	* @todo consider offsetting this on amd64 to use negative offsets to access
1055	* more members using 8-byte disp. */
1056	kIemNativeWhat_pVCpuFixed,
1057	/** The fixed pCtx (PCPUMCTX) register.
1058	* @todo consider offsetting this on amd64 to use negative offsets to access
1059	* more members using 8-byte disp. */
1060	kIemNativeWhat_pCtxFixed,
1061	/** Fixed temporary register. */
1062	kIemNativeWhat_FixedTmp,
1063	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1064	/** Shadow RIP for the delayed RIP updating debugging. */
1065	kIemNativeWhat_PcShadow,
1066	#endif
1067	/** Register reserved by the CPU or OS architecture. */
1068	kIemNativeWhat_FixedReserved,
1069	/** End of valid values. */
1070	kIemNativeWhat_End
1071	} IEMNATIVEWHAT;
1072
1073	/**
1074	* Host general register entry.
1075	*
1076	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1077	*
1078	* @todo Track immediate values in host registers similarlly to how we track the
1079	* guest register shadow copies. For it to be real helpful, though,
1080	* we probably need to know which will be reused and put them into
1081	* non-volatile registers, otherwise it's going to be more or less
1082	* restricted to an instruction or two.
1083	*/
1084	typedef struct IEMNATIVEHSTREG
1085	{
1086	/** Set of guest registers this one shadows.
1087	*
1088	* Using a bitmap here so we can designate the same host register as a copy
1089	* for more than one guest register. This is expected to be useful in
1090	* situations where one value is copied to several registers in a sequence.
1091	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1092	* sequence we'd want to let this register follow to be a copy of and there
1093	* will always be places where we'd be picking the wrong one.
1094	*/
1095	uint64_t fGstRegShadows;
1096	/** What is being kept in this register. */
1097	IEMNATIVEWHAT enmWhat;
1098	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1099	uint8_t idxVar;
1100	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1101	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1102	* that scope. */
1103	uint8_t idxStackSlot;
1104	/** Alignment padding. */
1105	uint8_t abAlign[5];
1106	} IEMNATIVEHSTREG;
1107
1108
1109	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1110	/**
1111	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1112	* halves, on architectures where there is no 256-bit register available this entry will track
1113	* two adjacent 128-bit host registers.
1114	*
1115	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1116	*/
1117	typedef struct IEMNATIVEHSTSIMDREG
1118	{
1119	/** Set of guest registers this one shadows.
1120	*
1121	* Using a bitmap here so we can designate the same host register as a copy
1122	* for more than one guest register. This is expected to be useful in
1123	* situations where one value is copied to several registers in a sequence.
1124	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1125	* sequence we'd want to let this register follow to be a copy of and there
1126	* will always be places where we'd be picking the wrong one.
1127	*/
1128	uint64_t fGstRegShadows;
1129	/** What is being kept in this register. */
1130	IEMNATIVEWHAT enmWhat;
1131	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1132	uint8_t idxVar;
1133	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1134	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1135	/** Alignment padding. */
1136	uint8_t abAlign[5];
1137	} IEMNATIVEHSTSIMDREG;
1138	#endif
1139
1140
1141	/**
1142	* Core state for the native recompiler, that is, things that needs careful
1143	* handling when dealing with branches.
1144	*/
1145	typedef struct IEMNATIVECORESTATE
1146	{
1147	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1148	/** The current instruction offset in bytes from when the guest program counter
1149	* was updated last. Used for delaying the write to the guest context program counter
1150	* as long as possible. */
1151	uint32_t offPc;
1152	/** Number of instructions where we could skip the updating. */
1153	uint32_t cInstrPcUpdateSkipped;
1154	#endif
1155	/** Allocation bitmap for aHstRegs. */
1156	uint32_t bmHstRegs;
1157
1158	/** Bitmap marking which host register contains guest register shadow copies.
1159	* This is used during register allocation to try preserve copies. */
1160	uint32_t bmHstRegsWithGstShadow;
1161	/** Bitmap marking valid entries in aidxGstRegShadows. */
1162	uint64_t bmGstRegShadows;
1163	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1164	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1165	uint64_t bmGstRegShadowDirty;
1166	#endif
1167
1168	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1169	/** Allocation bitmap for aHstSimdRegs. */
1170	uint32_t bmHstSimdRegs;
1171
1172	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1173	* This is used during register allocation to try preserve copies. */
1174	uint32_t bmHstSimdRegsWithGstShadow;
1175	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1176	uint64_t bmGstSimdRegShadows;
1177	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1178	uint64_t bmGstSimdRegShadowDirtyLo128;
1179	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1180	uint64_t bmGstSimdRegShadowDirtyHi128;
1181	#endif
1182
1183	union
1184	{
1185	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1186	uint8_t aidxArgVars[8];
1187	/** For more efficient resetting. */
1188	uint64_t u64ArgVars;
1189	};
1190
1191	/** Allocation bitmap for the stack. */
1192	uint32_t bmStack;
1193	/** Allocation bitmap for aVars. */
1194	uint32_t bmVars;
1195
1196	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1197	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1198	* (A shadow copy of a guest register can only be held in a one host register,
1199	* there are no duplicate copies or ambiguities like that). */
1200	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1201	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1202	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1203	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1204	* (A shadow copy of a guest register can only be held in a one host register,
1205	* there are no duplicate copies or ambiguities like that). */
1206	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1207	#endif
1208
1209	/** Host register allocation tracking. */
1210	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1211	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1212	/** Host SIMD register allocation tracking. */
1213	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1214	#endif
1215
1216	/** Variables and arguments. */
1217	IEMNATIVEVAR aVars[9];
1218	} IEMNATIVECORESTATE;
1219	/** Pointer to core state. */
1220	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1221	/** Pointer to const core state. */
1222	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1223
1224	/** @def IEMNATIVE_VAR_IDX_UNPACK
1225	* @returns Index into IEMNATIVECORESTATE::aVars.
1226	* @param a_idxVar Variable index w/ magic (in strict builds).
1227	*/
1228	/** @def IEMNATIVE_VAR_IDX_PACK
1229	* @returns Variable index w/ magic (in strict builds).
1230	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1231	*/
1232	#ifdef VBOX_STRICT
1233	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1234	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1235	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1236	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1237	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1238	#else
1239	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1240	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1241	#endif
1242
1243
1244	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1245	/** Clear the dirty state of the given guest SIMD register. */
1246	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1247	do { \
1248	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1249	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1250	} while (0)
1251
1252	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1253	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1254	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1255	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1256	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1257	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1258	/** Returns whether the given guest SIMD register is dirty. */
1259	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1260	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1261
1262	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1263	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1264	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1265	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1266	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1267	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1268
1269	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1270	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1271	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1272	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1273	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1274	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1275	#endif
1276
1277
1278	/**
1279	* Conditional stack entry.
1280	*/
1281	typedef struct IEMNATIVECOND
1282	{
1283	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1284	bool fInElse;
1285	/** The label for the IEM_MC_ELSE. */
1286	uint32_t idxLabelElse;
1287	/** The label for the IEM_MC_ENDIF. */
1288	uint32_t idxLabelEndIf;
1289	/** The initial state snapshot as the if-block starts executing. */
1290	IEMNATIVECORESTATE InitialState;
1291	/** The state snapshot at the end of the if-block. */
1292	IEMNATIVECORESTATE IfFinalState;
1293	} IEMNATIVECOND;
1294	/** Pointer to a condition stack entry. */
1295	typedef IEMNATIVECOND *PIEMNATIVECOND;
1296
1297
1298	/**
1299	* Native recompiler state.
1300	*/
1301	typedef struct IEMRECOMPILERSTATE
1302	{
1303	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1304	* IEMNATIVEINSTR units. */
1305	uint32_t cInstrBufAlloc;
1306	#ifdef VBOX_STRICT
1307	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1308	uint32_t offInstrBufChecked;
1309	#else
1310	uint32_t uPadding1; /* We don't keep track of the size here... */
1311	#endif
1312	/** Fixed temporary code buffer for native recompilation. */
1313	PIEMNATIVEINSTR pInstrBuf;
1314
1315	/** Bitmaps with the label types used. */
1316	uint64_t bmLabelTypes;
1317	/** Actual number of labels in paLabels. */
1318	uint32_t cLabels;
1319	/** Max number of entries allowed in paLabels before reallocating it. */
1320	uint32_t cLabelsAlloc;
1321	/** Labels defined while recompiling (referenced by fixups). */
1322	PIEMNATIVELABEL paLabels;
1323	/** Array with indexes of unique labels (uData always 0). */
1324	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1325
1326	/** Actual number of fixups paFixups. */
1327	uint32_t cFixups;
1328	/** Max number of entries allowed in paFixups before reallocating it. */
1329	uint32_t cFixupsAlloc;
1330	/** Buffer used by the recompiler for recording fixups when generating code. */
1331	PIEMNATIVEFIXUP paFixups;
1332
1333	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1334	/** Number of debug info entries allocated for pDbgInfo. */
1335	uint32_t cDbgInfoAlloc;
1336	uint32_t uPadding;
1337	/** Debug info. */
1338	PIEMTBDBG pDbgInfo;
1339	#endif
1340
1341	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1342	/** The current call index (liveness array and threaded calls in TB). */
1343	uint32_t idxCurCall;
1344	/** Number of liveness entries allocated. */
1345	uint32_t cLivenessEntriesAlloc;
1346	/** Liveness entries for all the calls in the TB begin recompiled.
1347	* The entry for idxCurCall contains the info for what the next call will
1348	* require wrt registers. (Which means the last entry is the initial liveness
1349	* state.) */
1350	PIEMLIVENESSENTRY paLivenessEntries;
1351	#endif
1352
1353	/** The translation block being recompiled. */
1354	PCIEMTB pTbOrg;
1355	/** The VMCPU structure of the EMT. */
1356	PVMCPUCC pVCpu;
1357
1358	/** Condition sequence number (for generating unique labels). */
1359	uint16_t uCondSeqNo;
1360	/** Check IRQ seqeunce number (for generating unique labels). */
1361	uint16_t uCheckIrqSeqNo;
1362	/** TLB load sequence number (for generating unique labels). */
1363	uint16_t uTlbSeqNo;
1364	/** The current condition stack depth (aCondStack). */
1365	uint8_t cCondDepth;
1366
1367	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1368	uint8_t cArgsX;
1369	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1370	uint32_t fCImpl;
1371	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1372	uint32_t fMc;
1373	/** The expected IEMCPU::fExec value for the current call/instruction. */
1374	uint32_t fExec;
1375	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1376	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1377	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1378	*
1379	* This is an optimization because these control registers can only be changed from
1380	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1381	* consisting of multiple SIMD instructions.
1382	*/
1383	uint32_t fSimdRaiseXcptChecksEmitted;
1384	#endif
1385
1386	/** Core state requiring care with branches. */
1387	IEMNATIVECORESTATE Core;
1388
1389	/** The condition nesting stack. */
1390	IEMNATIVECOND aCondStack[2];
1391
1392	#ifndef IEM_WITH_THROW_CATCH
1393	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1394	* for recompilation error handling. */
1395	jmp_buf JmpBuf;
1396	#endif
1397	} IEMRECOMPILERSTATE;
1398	/** Pointer to a native recompiler state. */
1399	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1400
1401
1402	/** @def IEMNATIVE_TRY_SETJMP
1403	* Wrapper around setjmp / try, hiding all the ugly differences.
1404	*
1405	* @note Use with extreme care as this is a fragile macro.
1406	* @param a_pReNative The native recompile state.
1407	* @param a_rcTarget The variable that should receive the status code in case
1408	* of a longjmp/throw.
1409	*/
1410	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1411	* Start wrapper for catch / setjmp-else.
1412	*
1413	* This will set up a scope.
1414	*
1415	* @note Use with extreme care as this is a fragile macro.
1416	* @param a_pReNative The native recompile state.
1417	* @param a_rcTarget The variable that should receive the status code in case
1418	* of a longjmp/throw.
1419	*/
1420	/** @def IEMNATIVE_CATCH_LONGJMP_END
1421	* End wrapper for catch / setjmp-else.
1422	*
1423	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1424	* up the state.
1425	*
1426	* @note Use with extreme care as this is a fragile macro.
1427	* @param a_pReNative The native recompile state.
1428	*/
1429	/** @def IEMNATIVE_DO_LONGJMP
1430	*
1431	* Wrapper around longjmp / throw.
1432	*
1433	* @param a_pReNative The native recompile state.
1434	* @param a_rc The status code jump back with / throw.
1435	*/
1436	#ifdef IEM_WITH_THROW_CATCH
1437	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1438	a_rcTarget = VINF_SUCCESS; \
1439	try
1440	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1441	catch (int rcThrown) \
1442	{ \
1443	a_rcTarget = rcThrown
1444	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1445	} \
1446	((void)0)
1447	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1448	#else /* !IEM_WITH_THROW_CATCH */
1449	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1450	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1451	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1452	else \
1453	{ \
1454	((void)0)
1455	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1456	}
1457	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1458	#endif /* !IEM_WITH_THROW_CATCH */
1459
1460
1461	/**
1462	* Native recompiler worker for a threaded function.
1463	*
1464	* @returns New code buffer offset; throws VBox status code in case of a failure.
1465	* @param pReNative The native recompiler state.
1466	* @param off The current code buffer offset.
1467	* @param pCallEntry The threaded call entry.
1468	*
1469	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1470	*/
1471	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1472	/** Pointer to a native recompiler worker for a threaded function. */
1473	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1474
1475	/** Defines a native recompiler worker for a threaded function.
1476	* @see FNIEMNATIVERECOMPFUNC */
1477	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1478	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1479
1480	/** Prototypes a native recompiler function for a threaded function.
1481	* @see FNIEMNATIVERECOMPFUNC */
1482	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1483
1484
1485	/**
1486	* Native recompiler liveness analysis worker for a threaded function.
1487	*
1488	* @param pCallEntry The threaded call entry.
1489	* @param pIncoming The incoming liveness state entry.
1490	* @param pOutgoing The outgoing liveness state entry.
1491	*/
1492	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1493	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1494	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1495	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1496
1497	/** Defines a native recompiler liveness analysis worker for a threaded function.
1498	* @see FNIEMNATIVELIVENESSFUNC */
1499	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1500	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1501
1502	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1503	* @see FNIEMNATIVELIVENESSFUNC */
1504	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1505
1506
1507	/** Define a native recompiler helper function, safe to call from the TB code. */
1508	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1509	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1510	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1511	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1512	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1513	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1514	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1515	a_RetType (VBOXCALL *a_Name) a_ArgList
1516
1517
1518	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1519	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1520	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1521	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1522	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1523	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1524	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1525	uint8_t idxHstSimdReg = UINT8_MAX,
1526	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1527	# endif
1528	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1529	DECL_HIDDEN_THROW(void) iemNaitveDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1530	uint8_t idxGstReg, uint8_t idxHstReg);
1531	DECL_HIDDEN_THROW(void) iemNaitveDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1532	uint64_t fGstReg);
1533	# endif
1534	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1535	uint32_t offPc, uint32_t cInstrSkipped);
1536	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1537
1538	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1539	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1540	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1541	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1542	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1543	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1544
1545	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1546	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1547	bool fPreferVolatile = true);
1548	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1549	bool fPreferVolatile = true);
1550	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1551	IEMNATIVEGSTREG enmGstReg,
1552	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1553	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1554	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1555	IEMNATIVEGSTREG enmGstReg);
1556
1557	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1558	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1559	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1560	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1561	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1562	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1563	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1564	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1565	# endif
1566	#endif
1567	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1568	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1569	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1570	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1571	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1572	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1573	#endif
1574	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1575	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1576	uint32_t fKeepVars = 0);
1577	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1578	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1579	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1580	uint32_t fHstRegsActiveShadows);
1581	#ifdef VBOX_STRICT
1582	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1583	#endif
1584	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1585	uint64_t fGstSimdShwExcept);
1586	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1587	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1588	#endif
1589	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1590	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1591	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1592	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1593	#endif
1594
1595
1596	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1597	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1598	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1599	bool fPreferVolatile = true);
1600	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1601	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1602	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1603	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1604	bool fNoVolatileRegs = false);
1605	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1606	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1607	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1608	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1609	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1610	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1611	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1612	#endif
1613
1614	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1615	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1616	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1617	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1618	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1619	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1620	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1621	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1622	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1623	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1624	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1625	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1626	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1627	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1628	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1629	#endif
1630	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1631	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1632	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1633	uint32_t fHstRegsNotToSave);
1634	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1635	uint32_t fHstRegsNotToSave);
1636	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1637	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1638
1639	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1640	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1641	#ifdef VBOX_STRICT
1642	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1643	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1644	IEMNATIVEGSTREG enmGstReg);
1645	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1646	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1647	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1648	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1649	# endif
1650	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1651	#endif
1652	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1653	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1654	#endif
1655	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1656	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1657	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1658	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1659	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1660	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1661	PCIEMTHRDEDCALLENTRY pCallEntry);
1662	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1663	uint8_t idxAddrReg, uint8_t idxInstr);
1664	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1665	uint8_t idxAddrReg, uint8_t idxInstr);
1666	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1667	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1668
1669
1670	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1671	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1672	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1673	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1674	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1675	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1676	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1677	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1678	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1679	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1680
1681	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1682	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1683	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1684	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1685	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1686	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1687	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1688	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1689	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1690	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1691	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1692	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1693	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1694	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1695	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1696	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1697	#endif
1698	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1699	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1700	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1701	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1702	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1703	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1704	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1705	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1706	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1707	#endif
1708	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1709	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1710	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1711	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1712	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1713	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1714	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1715
1716	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1717	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1718	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1719	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1720	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1721	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1722	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1723	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1724	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1725	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1726	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1727	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1728	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1729	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1730	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1731	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1732	#endif
1733	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1734	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1735	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1736	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1737	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1738	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1739	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1740	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1741	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1742	#endif
1743	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1744	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1745	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1746	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1747	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1748	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1749	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1750
1751	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1752	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1753	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1754	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1755	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1756	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1757	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1758	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1759	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1760	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1761	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1762	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1763	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1764	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1765	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1766	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1767	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1768	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1769	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1770	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1771	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1772	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1773
1774	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1775	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1776	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1777	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1778	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1779	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1780	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1781	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1782	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1783	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1784	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1785	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1786	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1787	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1788	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1789	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1790	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1791	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1792	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1793	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1794	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1795	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1796
1797	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1798	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1799	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1800	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1801
1802
1803	/**
1804	* Info about shadowed guest register values.
1805	* @see IEMNATIVEGSTREG
1806	*/
1807	typedef struct IEMANTIVEGSTREGINFO
1808	{
1809	/** Offset in VMCPU. */
1810	uint32_t off;
1811	/** The field size. */
1812	uint8_t cb;
1813	/** Name (for logging). */
1814	const char *pszName;
1815	} IEMANTIVEGSTREGINFO;
1816	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1817	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1818	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1819	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1820	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1821
1822
1823
1824	/**
1825	* Ensures that there is sufficient space in the instruction output buffer.
1826	*
1827	* This will reallocate the buffer if needed and allowed.
1828	*
1829	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1830	* allocation size.
1831	*
1832	* @returns Pointer to the instruction output buffer on success; throws VBox
1833	* status code on failure, so no need to check it.
1834	* @param pReNative The native recompile state.
1835	* @param off Current instruction offset. Works safely for UINT32_MAX
1836	* as well.
1837	* @param cInstrReq Number of instruction about to be added. It's okay to
1838	* overestimate this a bit.
1839	*/
1840	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1841	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1842	{
1843	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1844	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1845	{
1846	#ifdef VBOX_STRICT
1847	pReNative->offInstrBufChecked = offChecked;
1848	#endif
1849	return pReNative->pInstrBuf;
1850	}
1851	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1852	}
1853
1854	/**
1855	* Checks that we didn't exceed the space requested in the last
1856	* iemNativeInstrBufEnsure() call.
1857	*/
1858	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1859	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1860	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1861
1862	/**
1863	* Checks that a variable index is valid.
1864	*/
1865	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1866	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1867	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1868	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1869	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1870	("%s=%#x\n", #a_idxVar, a_idxVar))
1871	#else
1872	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1873	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1874	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1875	#endif
1876
1877	/**
1878	* Checks that a variable index is valid and that the variable is assigned the
1879	* correct argument number.
1880	* This also adds a RT_NOREF of a_idxVar.
1881	*/
1882	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1883	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1884	RT_NOREF_PV(a_idxVar); \
1885	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1886	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1887	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1888	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1889	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1890	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1891	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1892	a_uArgNo)); \
1893	} while (0)
1894	#else
1895	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1896	RT_NOREF_PV(a_idxVar); \
1897	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1898	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1899	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1900	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1901	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1902	} while (0)
1903	#endif
1904
1905
1906	/**
1907	* Checks that a variable has the expected size.
1908	*/
1909	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1910	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1911	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1912	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1913
1914
1915	/**
1916	* Calculates the stack address of a variable as a [r]BP displacement value.
1917	*/
1918	DECL_FORCE_INLINE(int32_t)
1919	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1920	{
1921	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1922	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1923	}
1924
1925
1926	/**
1927	* Releases the variable's register.
1928	*
1929	* The register must have been previously acquired calling
1930	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1931	* iemNativeVarRegisterSetAndAcquire().
1932	*/
1933	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1934	{
1935	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1936	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1937	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1938	}
1939
1940
1941	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1942	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1943	{
1944	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
1945	iemNativeVarRegisterRelease(pReNative, idxVar);
1946	}
1947	#endif
1948
1949
1950	/**
1951	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1952	*
1953	* @returns The flush mask.
1954	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1955	* @param fGstShwFlush The starting flush mask.
1956	*/
1957	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1958	{
1959	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1960	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1961	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1962	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1963	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1964	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1965	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1966	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1967	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1968	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1969	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1970	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1971	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1972	return fGstShwFlush;
1973	}
1974
1975
1976	/** Number of hidden arguments for CIMPL calls.
1977	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1978	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1979	# define IEM_CIMPL_HIDDEN_ARGS 3
1980	#else
1981	# define IEM_CIMPL_HIDDEN_ARGS 2
1982	#endif
1983
1984
1985	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1986	/** Number of hidden arguments for SSE_AIMPL calls. */
1987	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1988	/** Number of hidden arguments for AVX_AIMPL calls. */
1989	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
1990	#endif
1991
1992
1993	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1994
1995	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1996	/**
1997	* Helper for iemNativeLivenessGetStateByGstReg.
1998	*
1999	* @returns IEMLIVENESS_STATE_XXX
2000	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2001	* ORed together.
2002	*/
2003	DECL_FORCE_INLINE(uint32_t)
2004	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2005	{
2006	/* INPUT trumps anything else. */
2007	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2008	return IEMLIVENESS_STATE_INPUT;
2009
2010	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2011	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2012	{
2013	/* If not all sub-fields are clobbered they must be considered INPUT. */
2014	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2015	return IEMLIVENESS_STATE_INPUT;
2016	return IEMLIVENESS_STATE_CLOBBERED;
2017	}
2018
2019	/* XCPT_OR_CALL trumps UNUSED. */
2020	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2021	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2022
2023	return IEMLIVENESS_STATE_UNUSED;
2024	}
2025	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2026
2027
2028	DECL_FORCE_INLINE(uint32_t)
2029	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2030	{
2031	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2032	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2033	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2034	# else
2035	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2036	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2037	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2038	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2039	# endif
2040	}
2041
2042
2043	DECL_FORCE_INLINE(uint32_t)
2044	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2045	{
2046	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2047	if (enmGstReg == kIemNativeGstReg_EFlags)
2048	{
2049	/* Merge the eflags states to one. */
2050	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2051	uRet = RT_BIT_32(uRet);
2052	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2053	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2054	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2055	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2056	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2057	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2058	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2059	# else
2060	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2061	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2062	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2063	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2064	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2065	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2066	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2067	# endif
2068	}
2069	return uRet;
2070	}
2071
2072
2073	# ifdef VBOX_STRICT
2074	/** For assertions only, user checks that idxCurCall isn't zerow. */
2075	DECL_FORCE_INLINE(uint32_t)
2076	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2077	{
2078	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2079	}
2080	# endif /* VBOX_STRICT */
2081
2082	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2083
2084
2085	/**
2086	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2087	*/
2088	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2089	{
2090	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2091	return IEM_CIMPL_HIDDEN_ARGS;
2092	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2093	return 1;
2094	return 0;
2095	}
2096
2097
2098	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2099	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2100	{
2101	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2102
2103	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2104	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2105	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2106	return (uint8_t)idxReg;
2107	}
2108
2109
2110
2111	/*********************************************************************************************************************************
2112	* Register Allocator (GPR) *
2113	*********************************************************************************************************************************/
2114
2115	/**
2116	* Marks host register @a idxHstReg as containing a shadow copy of guest
2117	* register @a enmGstReg.
2118	*
2119	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2120	* host register before calling.
2121	*/
2122	DECL_FORCE_INLINE(void)
2123	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2124	{
2125	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2126	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2127	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2128
2129	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2130	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2131	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2132	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2133	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2134	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2135	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2136	#else
2137	RT_NOREF(off);
2138	#endif
2139	}
2140
2141
2142	/**
2143	* Clear any guest register shadow claims from @a idxHstReg.
2144	*
2145	* The register does not need to be shadowing any guest registers.
2146	*/
2147	DECL_FORCE_INLINE(void)
2148	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2149	{
2150	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2151	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2152	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2153	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2154	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2155	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2156	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2157	#endif
2158
2159	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2160	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2161	if (fGstRegs)
2162	{
2163	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2164	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2165	while (fGstRegs)
2166	{
2167	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2168	fGstRegs &= ~RT_BIT_64(iGstReg);
2169	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2170	}
2171	}
2172	#else
2173	RT_NOREF(off);
2174	#endif
2175
2176	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2177	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2178	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2179	}
2180
2181
2182	/**
2183	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2184	* and global overview flags.
2185	*/
2186	DECL_FORCE_INLINE(void)
2187	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2188	{
2189	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2190	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2191	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2192	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2193	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2194	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2195	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2196	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2197	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2198	#endif
2199
2200	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2201	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2202	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2203	#else
2204	RT_NOREF(off);
2205	#endif
2206
2207	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2208	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2209	if (!fGstRegShadowsNew)
2210	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2211	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2212	}
2213
2214
2215	#if 0 /* unused */
2216	/**
2217	* Clear any guest register shadow claim for @a enmGstReg.
2218	*/
2219	DECL_FORCE_INLINE(void)
2220	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2221	{
2222	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2223	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2224	{
2225	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2226	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2227	}
2228	}
2229	#endif
2230
2231
2232	/**
2233	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2234	* as the new shadow of it.
2235	*
2236	* Unlike the other guest reg shadow helpers, this does the logging for you.
2237	* However, it is the liveness state is not asserted here, the caller must do
2238	* that.
2239	*/
2240	DECL_FORCE_INLINE(void)
2241	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2242	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2243	{
2244	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2245	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2246	{
2247	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2248	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2249	if (idxHstRegOld == idxHstRegNew)
2250	return;
2251	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2252	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2253	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2254	}
2255	else
2256	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2257	g_aGstShadowInfo[enmGstReg].pszName));
2258	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2259	}
2260
2261
2262	/**
2263	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2264	* to @a idxRegTo.
2265	*/
2266	DECL_FORCE_INLINE(void)
2267	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2268	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2269	{
2270	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2271	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2272	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2273	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2274	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2275	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2276	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2277	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2278	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2279
2280	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2281	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2282	if (!fGstRegShadowsFrom)
2283	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2284	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2285	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2286	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2287	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2288	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2289	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2290	#else
2291	RT_NOREF(off);
2292	#endif
2293	}
2294
2295
2296	/**
2297	* Flushes any delayed guest register writes.
2298	*
2299	* This must be called prior to calling CImpl functions and any helpers that use
2300	* the guest state (like raising exceptions) and such.
2301	*
2302	* This optimization has not yet been implemented. The first target would be
2303	* RIP updates, since these are the most common ones.
2304	*
2305	* @note This function does not flush any shadowing information for guest registers. This needs to be done by
2306	* the caller if it wishes to do so.
2307	*/
2308	DECL_INLINE_THROW(uint32_t)
2309	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2310	{
2311	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2312	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2313	#else
2314	uint64_t const bmGstRegShadowDirty = 0;
2315	#endif
2316	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2317	uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 \| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2318	& ~fGstSimdShwExcept;
2319	#else
2320	uint64_t const bmGstSimdRegShadowDirty = 0;
2321	#endif
2322	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2323	uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2324	#else
2325	uint64_t const fWritebackPc = 0;
2326	#endif
2327	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2328	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2329
2330	return off;
2331	}
2332
2333
2334
2335	/*********************************************************************************************************************************
2336	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2337	*********************************************************************************************************************************/
2338
2339	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2340
2341	DECL_FORCE_INLINE(uint8_t)
2342	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2343	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2344	{
2345	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2346
2347	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2348	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2349	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2350	return idxSimdReg;
2351	}
2352
2353
2354	/**
2355	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2356	* SIMD register @a enmGstSimdReg.
2357	*
2358	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2359	* host register before calling.
2360	*/
2361	DECL_FORCE_INLINE(void)
2362	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2363	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2364	{
2365	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2366	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2367	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2368
2369	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2370	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2371	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2372	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2373	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2374	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2375	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2376	#else
2377	RT_NOREF(off);
2378	#endif
2379	}
2380
2381
2382	/**
2383	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2384	* to @a idxSimdRegTo.
2385	*/
2386	DECL_FORCE_INLINE(void)
2387	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2388	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2389	{
2390	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2391	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2392	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2393	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2394	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2395	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2396	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2397	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2398	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2399	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2400	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2401
2402	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2403	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2404	if (!fGstRegShadowsFrom)
2405	{
2406	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2407	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2408	}
2409	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2410	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2411	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2412	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2413	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2414	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2415	#else
2416	RT_NOREF(off);
2417	#endif
2418	}
2419
2420
2421	/**
2422	* Clear any guest register shadow claims from @a idxHstSimdReg.
2423	*
2424	* The register does not need to be shadowing any guest registers.
2425	*/
2426	DECL_FORCE_INLINE(void)
2427	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2428	{
2429	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2430	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2431	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2432	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2433	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2434	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2435	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2436
2437	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2438	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2439	if (fGstRegs)
2440	{
2441	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2442	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2443	while (fGstRegs)
2444	{
2445	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2446	fGstRegs &= ~RT_BIT_64(iGstReg);
2447	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2448	}
2449	}
2450	#else
2451	RT_NOREF(off);
2452	#endif
2453
2454	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2455	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2456	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2457	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2458	}
2459
2460	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2461
2462
2463	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2464	/**
2465	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2466	*/
2467	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2468	{
2469	if (pReNative->Core.offPc)
2470	return iemNativeEmitPcWritebackSlow(pReNative, off);
2471	return off;
2472	}
2473	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2474
2475
2476	/** @} */
2477
2478	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2479

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 104051

Download in other formats: