IEMN8veRecompiler.h@ 104035

Last change on this file since 104035 was 104035, checked in by vboxsync, 11 months ago
VMM/IEM: Implement experimental (disabled by default) delaying writeback of dirty guest registers (for GPRs only at the moment), bugref:10629 [doxygen]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 120.9 KB

Line
1	/* $Id: IEMN8veRecompiler.h 104035 2024-03-25 10:15:13Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72	#ifdef DEBUG_aeichner
73	/** @def IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
74	* Delay the writeback or dirty registers as long as possible. */
75	# define IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
76	#endif
77
78	/** @name Stack Frame Layout
79	*
80	* @{ */
81	/** The size of the area for stack variables and spills and stuff.
82	* @note This limit is duplicated in the python script(s). We add 0x40 for
83	* alignment padding. */
84	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
85	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
86	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
87	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
88
89	#ifdef RT_ARCH_AMD64
90	/** An stack alignment adjustment (between non-volatile register pushes and
91	* the stack variable area, so the latter better aligned). */
92	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
93
94	/** Number of stack arguments slots for calls made from the frame. */
95	# ifdef RT_OS_WINDOWS
96	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
97	# else
98	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
99	# endif
100	/** Number of any shadow arguments (spill area) for calls we make. */
101	# ifdef RT_OS_WINDOWS
102	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
103	# else
104	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
105	# endif
106
107	/** Frame pointer (RBP) relative offset of the last push. */
108	# ifdef RT_OS_WINDOWS
109	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
110	# else
111	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
112	# endif
113	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
114	* address for it). */
115	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
116	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
118	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
120	# ifdef RT_OS_WINDOWS
121	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
122	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
123	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
124	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
125	# endif
126
127	# ifdef RT_OS_WINDOWS
128	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
129	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
130	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
131	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
132	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
133	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
134	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
135	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
136	# endif
137
138	#elif RT_ARCH_ARM64
139	/** No alignment padding needed for arm64. */
140	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
141	/** No stack argument slots, got 8 registers for arguments will suffice. */
142	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
143	/** There are no argument spill area. */
144	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
145
146	/** Number of saved registers at the top of our stack frame.
147	* This includes the return address and old frame pointer, so x19 thru x30. */
148	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
149	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
150	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
151
152	/** Frame pointer (BP) relative offset of the last push. */
153	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
154
155	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
156	* address for it). */
157	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
158
159	#else
160	# error "port me"
161	#endif
162	/** @} */
163
164
165	/** @name Fixed Register Allocation(s)
166	* @{ */
167	/** @def IEMNATIVE_REG_FIXED_PVMCPU
168	* The number of the register holding the pVCpu pointer. */
169	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
170	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
171	* @note This not available on AMD64, only ARM64. */
172	/** @def IEMNATIVE_REG_FIXED_TMP0
173	* Dedicated temporary register.
174	* @todo replace this by a register allocator and content tracker. */
175	/** @def IEMNATIVE_REG_FIXED_MASK
176	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
177	* architecture. */
178	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
179	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
180	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
181	* architecture. */
182	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
183	* Dedicated temporary SIMD register. */
184	#endif
185	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
186	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
187	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
188	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
189	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
190	\| RT_BIT_32(X86_GREG_xSP) \
191	\| RT_BIT_32(X86_GREG_xBP) )
192
193	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
195	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
196	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
197	# else
198	/** On Windows xmm6 through xmm15 are marked as callee saved. */
199	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
200	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
201	# endif
202	# endif
203
204	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
205	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
206	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
207	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
208	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
209	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
210	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
211	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
212	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
213	# else
214	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
215	# endif
216	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
217	\| RT_BIT_32(ARMV8_A64_REG_LR) \
218	\| RT_BIT_32(ARMV8_A64_REG_BP) \
219	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
220	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
221	\| RT_BIT_32(ARMV8_A64_REG_X18) \
222	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
223	\| IEMNATIVE_REG_FIXED_MASK_ADD)
224
225	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
226	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
227	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
228	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
229	# else
230	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
231	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
232	\| RT_BIT_32(ARMV8_A64_REG_Q30))
233	# endif
234	# endif
235
236	#else
237	# error "port me"
238	#endif
239	/** @} */
240
241	/** @name Call related registers.
242	* @{ */
243	/** @def IEMNATIVE_CALL_RET_GREG
244	* The return value register. */
245	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
246	* Number of arguments in registers. */
247	/** @def IEMNATIVE_CALL_ARG0_GREG
248	* The general purpose register carrying argument \#0. */
249	/** @def IEMNATIVE_CALL_ARG1_GREG
250	* The general purpose register carrying argument \#1. */
251	/** @def IEMNATIVE_CALL_ARG2_GREG
252	* The general purpose register carrying argument \#2. */
253	/** @def IEMNATIVE_CALL_ARG3_GREG
254	* The general purpose register carrying argument \#3. */
255	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
256	* Mask of registers the callee will not save and may trash. */
257	#ifdef RT_ARCH_AMD64
258	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
259
260	# ifdef RT_OS_WINDOWS
261	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
262	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
263	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
264	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
265	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
266	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
267	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
268	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
269	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
270	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
271	\| RT_BIT_32(X86_GREG_xCX) \
272	\| RT_BIT_32(X86_GREG_xDX) \
273	\| RT_BIT_32(X86_GREG_x8) \
274	\| RT_BIT_32(X86_GREG_x9) \
275	\| RT_BIT_32(X86_GREG_x10) \
276	\| RT_BIT_32(X86_GREG_x11) )
277	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
278	/* xmm0 - xmm5 are marked as volatile. */
279	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
280	# endif
281
282	# else
283	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
284	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
285	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
286	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
287	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
288	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
289	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
290	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
291	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
292	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
293	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
294	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
295	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
296	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
297	\| RT_BIT_32(X86_GREG_xCX) \
298	\| RT_BIT_32(X86_GREG_xDX) \
299	\| RT_BIT_32(X86_GREG_xDI) \
300	\| RT_BIT_32(X86_GREG_xSI) \
301	\| RT_BIT_32(X86_GREG_x8) \
302	\| RT_BIT_32(X86_GREG_x9) \
303	\| RT_BIT_32(X86_GREG_x10) \
304	\| RT_BIT_32(X86_GREG_x11) )
305	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
306	/* xmm0 - xmm15 are marked as volatile. */
307	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
308	# endif
309	# endif
310
311	#elif defined(RT_ARCH_ARM64)
312	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
313	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
314	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
315	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
316	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
317	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
318	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
319	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
320	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
321	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
322	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
323	\| RT_BIT_32(ARMV8_A64_REG_X1) \
324	\| RT_BIT_32(ARMV8_A64_REG_X2) \
325	\| RT_BIT_32(ARMV8_A64_REG_X3) \
326	\| RT_BIT_32(ARMV8_A64_REG_X4) \
327	\| RT_BIT_32(ARMV8_A64_REG_X5) \
328	\| RT_BIT_32(ARMV8_A64_REG_X6) \
329	\| RT_BIT_32(ARMV8_A64_REG_X7) )
330	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
331	\| RT_BIT_32(ARMV8_A64_REG_X1) \
332	\| RT_BIT_32(ARMV8_A64_REG_X2) \
333	\| RT_BIT_32(ARMV8_A64_REG_X3) \
334	\| RT_BIT_32(ARMV8_A64_REG_X4) \
335	\| RT_BIT_32(ARMV8_A64_REG_X5) \
336	\| RT_BIT_32(ARMV8_A64_REG_X6) \
337	\| RT_BIT_32(ARMV8_A64_REG_X7) \
338	\| RT_BIT_32(ARMV8_A64_REG_X8) \
339	\| RT_BIT_32(ARMV8_A64_REG_X9) \
340	\| RT_BIT_32(ARMV8_A64_REG_X10) \
341	\| RT_BIT_32(ARMV8_A64_REG_X11) \
342	\| RT_BIT_32(ARMV8_A64_REG_X12) \
343	\| RT_BIT_32(ARMV8_A64_REG_X13) \
344	\| RT_BIT_32(ARMV8_A64_REG_X14) \
345	\| RT_BIT_32(ARMV8_A64_REG_X15) \
346	\| RT_BIT_32(ARMV8_A64_REG_X16) \
347	\| RT_BIT_32(ARMV8_A64_REG_X17) )
348	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
349	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
350	* so to simplify our life a bit we just mark everything as volatile. */
351	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
352	# endif
353
354	#endif
355
356	/** This is the maximum argument count we'll ever be needing. */
357	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
358	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
359	#else
360	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
361	#endif
362	/** @} */
363
364
365	/** @def IEMNATIVE_HST_GREG_COUNT
366	* Number of host general purpose registers we tracker. */
367	/** @def IEMNATIVE_HST_GREG_MASK
368	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
369	* inverted register masks and such to get down to a correct set of regs. */
370	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
371	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
372	* Number of host SIMD registers we track. */
373	/** @def IEMNATIVE_HST_SIMD_REG_MASK
374	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
375	* inverted register masks and such to get down to a correct set of regs. */
376	#endif
377	#ifdef RT_ARCH_AMD64
378	# define IEMNATIVE_HST_GREG_COUNT 16
379	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
380
381	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
382	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
383	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
384	# endif
385
386	#elif defined(RT_ARCH_ARM64)
387	# define IEMNATIVE_HST_GREG_COUNT 32
388	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
389
390	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
391	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
392	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
393	# endif
394
395	#else
396	# error "Port me!"
397	#endif
398
399
400	/** Native code generator label types. */
401	typedef enum
402	{
403	kIemNativeLabelType_Invalid = 0,
404	/*
405	* Labels w/o data, only once instance per TB.
406	*
407	* Note! Jumps to these requires instructions that are capable of spanning
408	* the max TB length.
409	*/
410	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
411	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
412	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
413	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
414	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
415	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
416	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
417	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
418	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
419	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
420	kIemNativeLabelType_ObsoleteTb,
421	kIemNativeLabelType_NeedCsLimChecking,
422	kIemNativeLabelType_CheckBranchMiss,
423	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
424	/* Manually defined labels. */
425	kIemNativeLabelType_Return,
426	kIemNativeLabelType_ReturnBreak,
427	kIemNativeLabelType_ReturnWithFlags,
428	kIemNativeLabelType_NonZeroRetOrPassUp,
429	/** The last fixup for branches that can span almost the whole TB length. */
430	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
431
432	/*
433	* Labels with data, potentially multiple instances per TB:
434	*
435	* These are localized labels, so no fixed jump type restrictions here.
436	*/
437	kIemNativeLabelType_FirstWithMultipleInstances,
438	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
439	kIemNativeLabelType_Else,
440	kIemNativeLabelType_Endif,
441	kIemNativeLabelType_CheckIrq,
442	kIemNativeLabelType_TlbLookup,
443	kIemNativeLabelType_TlbMiss,
444	kIemNativeLabelType_TlbDone,
445	kIemNativeLabelType_End
446	} IEMNATIVELABELTYPE;
447
448	/** Native code generator label definition. */
449	typedef struct IEMNATIVELABEL
450	{
451	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
452	* the epilog. */
453	uint32_t off;
454	/** The type of label (IEMNATIVELABELTYPE). */
455	uint16_t enmType;
456	/** Additional label data, type specific. */
457	uint16_t uData;
458	} IEMNATIVELABEL;
459	/** Pointer to a label. */
460	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
461
462
463	/** Native code generator fixup types. */
464	typedef enum
465	{
466	kIemNativeFixupType_Invalid = 0,
467	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
468	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
469	kIemNativeFixupType_Rel32,
470	#elif defined(RT_ARCH_ARM64)
471	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
472	kIemNativeFixupType_RelImm26At0,
473	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
474	kIemNativeFixupType_RelImm19At5,
475	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
476	kIemNativeFixupType_RelImm14At5,
477	#endif
478	kIemNativeFixupType_End
479	} IEMNATIVEFIXUPTYPE;
480
481	/** Native code generator fixup. */
482	typedef struct IEMNATIVEFIXUP
483	{
484	/** Code offset of the fixup location. */
485	uint32_t off;
486	/** The IEMNATIVELABEL this is a fixup for. */
487	uint16_t idxLabel;
488	/** The fixup type (IEMNATIVEFIXUPTYPE). */
489	uint8_t enmType;
490	/** Addend or other data. */
491	int8_t offAddend;
492	} IEMNATIVEFIXUP;
493	/** Pointer to a native code generator fixup. */
494	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
495
496
497	/**
498	* One bit of the state.
499	*
500	* Each register state takes up two bits. We keep the two bits in two separate
501	* 64-bit words to simplify applying them to the guest shadow register mask in
502	* the register allocator.
503	*/
504	typedef union IEMLIVENESSBIT
505	{
506	uint64_t bm64;
507	RT_GCC_EXTENSION struct
508	{ /* bit no */
509	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
510	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
511	uint64_t fCr0 : 1; /*< 0x11 / 17: /
512	uint64_t fFcw : 1; /*< 0x12 / 18: /
513	uint64_t fFsw : 1; /*< 0x13 / 19: /
514	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
515	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
516	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
517	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
518	uint64_t fCr4 : 1; /*< 0x2c / 44: /
519	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
520	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
521	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
522	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
523	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
524	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
525	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
526	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
527	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
528	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
529	};
530	} IEMLIVENESSBIT;
531	AssertCompileSize(IEMLIVENESSBIT, 8);
532
533	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
534	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
535	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
536	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
537	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
538	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
539	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
540
541
542	/**
543	* A liveness state entry.
544	*
545	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
546	* Once we add a SSE register shadowing, we'll add another 64-bit element for
547	* that.
548	*/
549	typedef union IEMLIVENESSENTRY
550	{
551	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
552	uint64_t bm64[16 / 8];
553	uint16_t bm32[16 / 4];
554	uint16_t bm16[16 / 2];
555	uint8_t bm8[ 16 / 1];
556	IEMLIVENESSBIT aBits[2];
557	#else
558	uint64_t bm64[32 / 8];
559	uint16_t bm32[32 / 4];
560	uint16_t bm16[32 / 2];
561	uint8_t bm8[ 32 / 1];
562	IEMLIVENESSBIT aBits[4];
563	#endif
564	RT_GCC_EXTENSION struct
565	{
566	/** Bit \#0 of the register states. */
567	IEMLIVENESSBIT Bit0;
568	/** Bit \#1 of the register states. */
569	IEMLIVENESSBIT Bit1;
570	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
571	/** Bit \#2 of the register states. */
572	IEMLIVENESSBIT Bit2;
573	/** Bit \#3 of the register states. */
574	IEMLIVENESSBIT Bit3;
575	#endif
576	};
577	} IEMLIVENESSENTRY;
578	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
579	AssertCompileSize(IEMLIVENESSENTRY, 16);
580	#else
581	AssertCompileSize(IEMLIVENESSENTRY, 32);
582	#endif
583	/** Pointer to a liveness state entry. */
584	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
585	/** Pointer to a const liveness state entry. */
586	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
587
588	/** @name 64-bit value masks for IEMLIVENESSENTRY.
589	* @{ / / 0xzzzzyyyyxxxxwwww */
590	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
591
592	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
593	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
594	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
595
596	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
597	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
598	#endif
599
600	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
601	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
602
603	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
604	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
605	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
606	#endif
607	/** @} */
608
609
610	/** @name The liveness state for a register.
611	*
612	* The state values have been picked to with state accumulation in mind (what
613	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
614	* performance critical work done with the values.
615	*
616	* This is a compressed state that only requires 2 bits per register.
617	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
618	* 1. the incoming state from the following call,
619	* 2. the outgoing state for this call,
620	* 3. mask of the entries set in the 2nd.
621	*
622	* The mask entry (3rd one above) will be used both when updating the outgoing
623	* state and when merging in incoming state for registers not touched by the
624	* current call.
625	*
626	* @{ */
627	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
628	/** The register will be clobbered and the current value thrown away.
629	*
630	* When this is applied to the state (2) we'll simply be AND'ing it with the
631	* (old) mask (3) and adding the register to the mask. This way we'll
632	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
633	* IEMLIVENESS_STATE_INPUT states. */
634	# define IEMLIVENESS_STATE_CLOBBERED 0
635	/** The register is unused in the remainder of the TB.
636	*
637	* This is an initial state and can not be set by any of the
638	* iemNativeLivenessFunc_xxxx callbacks. */
639	# define IEMLIVENESS_STATE_UNUSED 1
640	/** The register value is required in a potential call or exception.
641	*
642	* This means that the register value must be calculated and is best written to
643	* the state, but that any shadowing registers can be flushed thereafter as it's
644	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
645	*
646	* It is typically applied across the board, but we preserve incoming
647	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
648	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
649	* 1. r0 = old & ~mask;
650	* 2. r0 = t1 & (t1 >> 1)'
651	* 3. state \|= r0 \| 0b10;
652	* 4. mask = ~0;
653	*/
654	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
655	/** The register value is used as input.
656	*
657	* This means that the register value must be calculated and it is best to keep
658	* it in a register. It does not need to be writtent out as such. This is the
659	* highest priority state.
660	*
661	* Whether the call modifies the register or not isn't relevant to earlier
662	* calls, so that's not recorded.
663	*
664	* When applying this state we just or in the value in the outgoing state and
665	* mask. */
666	# define IEMLIVENESS_STATE_INPUT 3
667	/** Mask of the state bits. */
668	# define IEMLIVENESS_STATE_MASK 3
669	/** The number of bits per state. */
670	# define IEMLIVENESS_STATE_BIT_COUNT 2
671	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
672	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
673	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
674	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
675	/** Check if a register clobbering is expected given the (previous) liveness state.
676	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
677	* include INPUT if the register is used in more than one place. */
678	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
679
680	/** Check if all status flags are going to be clobbered and doesn't need
681	* calculating in the current step.
682	* @param a_pCurEntry The current liveness entry. */
683	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
684	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
685
686	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
687	/** The register is not used any more. */
688	# define IEMLIVENESS_STATE_UNUSED 0
689	/** Flag: The register is required in a potential exception or call. */
690	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
691	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
692	/** Flag: The register is read. */
693	# define IEMLIVENESS_STATE_READ 2
694	# define IEMLIVENESS_BIT_READ 1
695	/** Flag: The register is written. */
696	# define IEMLIVENESS_STATE_WRITE 4
697	# define IEMLIVENESS_BIT_WRITE 2
698	/** Flag: Unconditional call (not needed, can be redefined for research). */
699	# define IEMLIVENESS_STATE_CALL 8
700	# define IEMLIVENESS_BIT_CALL 3
701	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
702	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
703	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
704	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
705	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
706
707	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
708	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
709	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
710	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
711
712	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
713	/** @} */
714
715	/** @name Liveness helpers for builtin functions and similar.
716	*
717	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
718	* own set of manimulator macros for those.
719	*
720	* @{ */
721	/** Initializing the state as all unused. */
722	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
723	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
724	do { \
725	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
726	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
727	} while (0)
728	#else
729	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
730	do { \
731	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
732	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
733	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
734	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
735	} while (0)
736	#endif
737
738	/** Initializing the outgoing state with a potential xcpt or call state.
739	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
740	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
741	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
742	do { \
743	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
744	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
745	} while (0)
746	#else
747	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
748	do { \
749	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
750	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
751	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
752	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
753	} while (0)
754	#endif
755
756	/** Adds a segment base register as input to the outgoing state. */
757	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
758	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
759	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
760	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
761	} while (0)
762	#else
763	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
764	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
765	} while (0)
766	#endif
767
768	/** Adds a segment attribute register as input to the outgoing state. */
769	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
770	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
771	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
772	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
773	} while (0)
774	#else
775	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
776	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
777	} while (0)
778	#endif
779
780	/** Adds a segment limit register as input to the outgoing state. */
781	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
782	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
783	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
784	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
785	} while (0)
786	#else
787	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
788	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
789	} while (0)
790	#endif
791
792	/** Adds a segment limit register as input to the outgoing state. */
793	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
794	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
795	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
796	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
797	} while (0)
798	#else
799	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
800	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
801	} while (0)
802	#endif
803	/** @} */
804
805	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
806	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
807	* calculated and up to date. This is to double check that we haven't skipped
808	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
809	* @note has to be placed in
810	*/
811	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
812	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
813	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
814	#else
815	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
816	#endif
817
818
819	/**
820	* Guest registers that can be shadowed in GPRs.
821	*
822	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
823	* must be placed last, as the liveness state tracks it as 7 subcomponents and
824	* we don't want to waste space here.
825	*
826	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
827	* friends as well as IEMAllN8veLiveness.cpp.
828	*/
829	typedef enum IEMNATIVEGSTREG : uint8_t
830	{
831	kIemNativeGstReg_GprFirst = 0,
832	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
833	kIemNativeGstReg_Pc,
834	kIemNativeGstReg_Cr0,
835	kIemNativeGstReg_FpuFcw,
836	kIemNativeGstReg_FpuFsw,
837	kIemNativeGstReg_SegBaseFirst,
838	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
839	kIemNativeGstReg_SegAttribFirst,
840	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
841	kIemNativeGstReg_SegLimitFirst,
842	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
843	kIemNativeGstReg_SegSelFirst,
844	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
845	kIemNativeGstReg_Cr4,
846	kIemNativeGstReg_Xcr0,
847	kIemNativeGstReg_MxCsr,
848	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
849	kIemNativeGstReg_End
850	} IEMNATIVEGSTREG;
851	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
852	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
853
854	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
855	* @{ */
856	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
857	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
858	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
859	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
860	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
861	/** @} */
862
863	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
864
865	/**
866	* Guest registers that can be shadowed in host SIMD registers.
867	*
868	* @todo r=aeichner Liveness tracking
869	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
870	*/
871	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
872	{
873	kIemNativeGstSimdReg_SimdRegFirst = 0,
874	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
875	kIemNativeGstSimdReg_End
876	} IEMNATIVEGSTSIMDREG;
877
878	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
879	* @{ */
880	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
881	/** @} */
882
883	/**
884	* The Load/store size for a SIMD guest register.
885	*/
886	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
887	{
888	/** Invalid size. */
889	kIemNativeGstSimdRegLdStSz_Invalid = 0,
890	/** Loads the low 128-bit of a guest SIMD register. */
891	kIemNativeGstSimdRegLdStSz_Low128,
892	/** Loads the high 128-bit of a guest SIMD register. */
893	kIemNativeGstSimdRegLdStSz_High128,
894	/** Loads the whole 256-bits of a guest SIMD register. */
895	kIemNativeGstSimdRegLdStSz_256,
896	/** End value. */
897	kIemNativeGstSimdRegLdStSz_End
898	} IEMNATIVEGSTSIMDREGLDSTSZ;
899
900	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
901
902	/**
903	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
904	*/
905	typedef enum IEMNATIVEGSTREGUSE
906	{
907	/** The usage is read-only, the register holding the guest register
908	* shadow copy will not be modified by the caller. */
909	kIemNativeGstRegUse_ReadOnly = 0,
910	/** The caller will update the guest register (think: PC += cbInstr).
911	* The guest shadow copy will follow the returned register. */
912	kIemNativeGstRegUse_ForUpdate,
913	/** The call will put an entirely new value in the guest register, so
914	* if new register is allocate it will be returned uninitialized. */
915	kIemNativeGstRegUse_ForFullWrite,
916	/** The caller will use the guest register value as input in a calculation
917	* and the host register will be modified.
918	* This means that the returned host register will not be marked as a shadow
919	* copy of the guest register. */
920	kIemNativeGstRegUse_Calculation
921	} IEMNATIVEGSTREGUSE;
922
923	/**
924	* Guest registers (classes) that can be referenced.
925	*/
926	typedef enum IEMNATIVEGSTREGREF : uint8_t
927	{
928	kIemNativeGstRegRef_Invalid = 0,
929	kIemNativeGstRegRef_Gpr,
930	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
931	kIemNativeGstRegRef_EFlags,
932	kIemNativeGstRegRef_MxCsr,
933	kIemNativeGstRegRef_FpuReg,
934	kIemNativeGstRegRef_MReg,
935	kIemNativeGstRegRef_XReg,
936	kIemNativeGstRegRef_X87,
937	kIemNativeGstRegRef_XState,
938	//kIemNativeGstRegRef_YReg, - doesn't work.
939	kIemNativeGstRegRef_End
940	} IEMNATIVEGSTREGREF;
941
942
943	/** Variable kinds. */
944	typedef enum IEMNATIVEVARKIND : uint8_t
945	{
946	/** Customary invalid zero value. */
947	kIemNativeVarKind_Invalid = 0,
948	/** This is either in a register or on the stack. */
949	kIemNativeVarKind_Stack,
950	/** Immediate value - loaded into register when needed, or can live on the
951	* stack if referenced (in theory). */
952	kIemNativeVarKind_Immediate,
953	/** Variable reference - loaded into register when needed, never stack. */
954	kIemNativeVarKind_VarRef,
955	/** Guest register reference - loaded into register when needed, never stack. */
956	kIemNativeVarKind_GstRegRef,
957	/** End of valid values. */
958	kIemNativeVarKind_End
959	} IEMNATIVEVARKIND;
960
961
962	/** Variable or argument. */
963	typedef struct IEMNATIVEVAR
964	{
965	/** The kind of variable. */
966	IEMNATIVEVARKIND enmKind;
967	/** The variable size in bytes. */
968	uint8_t cbVar;
969	/** The first stack slot (uint64_t), except for immediate and references
970	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
971	* has a stack slot it has been initialized and has a value. Unused variables
972	* has neither a stack slot nor a host register assignment. */
973	uint8_t idxStackSlot;
974	/** The host register allocated for the variable, UINT8_MAX if not. */
975	uint8_t idxReg;
976	/** The argument number if argument, UINT8_MAX if regular variable. */
977	uint8_t uArgNo;
978	/** If referenced, the index (unpacked) of the variable referencing this one,
979	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
980	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
981	uint8_t idxReferrerVar;
982	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
983	* @todo not sure what this really is for... */
984	IEMNATIVEGSTREG enmGstReg;
985	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
986	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
987	* only valid when idxReg is not UINT8_MAX. */
988	bool fSimdReg : 1;
989	/** Set if the registered is currently used exclusively, false if the
990	* variable is idle and the register can be grabbed. */
991	bool fRegAcquired : 1;
992	#else
993	/** Set if the registered is currently used exclusively, false if the
994	* variable is idle and the register can be grabbed. */
995	bool fRegAcquired;
996	#endif
997
998	union
999	{
1000	/** kIemNativeVarKind_Immediate: The immediate value. */
1001	uint64_t uValue;
1002	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1003	uint8_t idxRefVar;
1004	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1005	struct
1006	{
1007	/** The class of register. */
1008	IEMNATIVEGSTREGREF enmClass;
1009	/** Index within the class. */
1010	uint8_t idx;
1011	} GstRegRef;
1012	} u;
1013	} IEMNATIVEVAR;
1014	/** Pointer to a variable or argument. */
1015	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1016	/** Pointer to a const variable or argument. */
1017	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1018
1019	/** What is being kept in a host register. */
1020	typedef enum IEMNATIVEWHAT : uint8_t
1021	{
1022	/** The traditional invalid zero value. */
1023	kIemNativeWhat_Invalid = 0,
1024	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1025	kIemNativeWhat_Var,
1026	/** Temporary register, this is typically freed when a MC completes. */
1027	kIemNativeWhat_Tmp,
1028	/** Call argument w/o a variable mapping. This is free (via
1029	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1030	kIemNativeWhat_Arg,
1031	/** Return status code.
1032	* @todo not sure if we need this... */
1033	kIemNativeWhat_rc,
1034	/** The fixed pVCpu (PVMCPUCC) register.
1035	* @todo consider offsetting this on amd64 to use negative offsets to access
1036	* more members using 8-byte disp. */
1037	kIemNativeWhat_pVCpuFixed,
1038	/** The fixed pCtx (PCPUMCTX) register.
1039	* @todo consider offsetting this on amd64 to use negative offsets to access
1040	* more members using 8-byte disp. */
1041	kIemNativeWhat_pCtxFixed,
1042	/** Fixed temporary register. */
1043	kIemNativeWhat_FixedTmp,
1044	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1045	/** Shadow RIP for the delayed RIP updating debugging. */
1046	kIemNativeWhat_PcShadow,
1047	#endif
1048	/** Register reserved by the CPU or OS architecture. */
1049	kIemNativeWhat_FixedReserved,
1050	/** End of valid values. */
1051	kIemNativeWhat_End
1052	} IEMNATIVEWHAT;
1053
1054	/**
1055	* Host general register entry.
1056	*
1057	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1058	*
1059	* @todo Track immediate values in host registers similarlly to how we track the
1060	* guest register shadow copies. For it to be real helpful, though,
1061	* we probably need to know which will be reused and put them into
1062	* non-volatile registers, otherwise it's going to be more or less
1063	* restricted to an instruction or two.
1064	*/
1065	typedef struct IEMNATIVEHSTREG
1066	{
1067	/** Set of guest registers this one shadows.
1068	*
1069	* Using a bitmap here so we can designate the same host register as a copy
1070	* for more than one guest register. This is expected to be useful in
1071	* situations where one value is copied to several registers in a sequence.
1072	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1073	* sequence we'd want to let this register follow to be a copy of and there
1074	* will always be places where we'd be picking the wrong one.
1075	*/
1076	uint64_t fGstRegShadows;
1077	/** What is being kept in this register. */
1078	IEMNATIVEWHAT enmWhat;
1079	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1080	uint8_t idxVar;
1081	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1082	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1083	* that scope. */
1084	uint8_t idxStackSlot;
1085	/** Alignment padding. */
1086	uint8_t abAlign[5];
1087	} IEMNATIVEHSTREG;
1088
1089
1090	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1091	/**
1092	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1093	* halves, on architectures where there is no 256-bit register available this entry will track
1094	* two adjacent 128-bit host registers.
1095	*
1096	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1097	*/
1098	typedef struct IEMNATIVEHSTSIMDREG
1099	{
1100	/** Set of guest registers this one shadows.
1101	*
1102	* Using a bitmap here so we can designate the same host register as a copy
1103	* for more than one guest register. This is expected to be useful in
1104	* situations where one value is copied to several registers in a sequence.
1105	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1106	* sequence we'd want to let this register follow to be a copy of and there
1107	* will always be places where we'd be picking the wrong one.
1108	*/
1109	uint64_t fGstRegShadows;
1110	/** What is being kept in this register. */
1111	IEMNATIVEWHAT enmWhat;
1112	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1113	uint8_t idxVar;
1114	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1115	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1116	/** Alignment padding. */
1117	uint8_t abAlign[5];
1118	} IEMNATIVEHSTSIMDREG;
1119	#endif
1120
1121
1122	/**
1123	* Core state for the native recompiler, that is, things that needs careful
1124	* handling when dealing with branches.
1125	*/
1126	typedef struct IEMNATIVECORESTATE
1127	{
1128	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1129	/** The current instruction offset in bytes from when the guest program counter
1130	* was updated last. Used for delaying the write to the guest context program counter
1131	* as long as possible. */
1132	uint32_t offPc;
1133	/** Number of instructions where we could skip the updating. */
1134	uint32_t cInstrPcUpdateSkipped;
1135	#endif
1136	/** Allocation bitmap for aHstRegs. */
1137	uint32_t bmHstRegs;
1138
1139	/** Bitmap marking which host register contains guest register shadow copies.
1140	* This is used during register allocation to try preserve copies. */
1141	uint32_t bmHstRegsWithGstShadow;
1142	/** Bitmap marking valid entries in aidxGstRegShadows. */
1143	uint64_t bmGstRegShadows;
1144	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1145	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1146	uint64_t bmGstRegShadowDirty;
1147	#endif
1148
1149	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1150	/** Allocation bitmap for aHstSimdRegs. */
1151	uint32_t bmHstSimdRegs;
1152
1153	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1154	* This is used during register allocation to try preserve copies. */
1155	uint32_t bmHstSimdRegsWithGstShadow;
1156	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1157	uint64_t bmGstSimdRegShadows;
1158	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1159	uint64_t bmGstSimdRegShadowDirtyLo128;
1160	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1161	uint64_t bmGstSimdRegShadowDirtyHi128;
1162	#endif
1163
1164	union
1165	{
1166	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1167	uint8_t aidxArgVars[8];
1168	/** For more efficient resetting. */
1169	uint64_t u64ArgVars;
1170	};
1171
1172	/** Allocation bitmap for the stack. */
1173	uint32_t bmStack;
1174	/** Allocation bitmap for aVars. */
1175	uint32_t bmVars;
1176
1177	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1178	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1179	* (A shadow copy of a guest register can only be held in a one host register,
1180	* there are no duplicate copies or ambiguities like that). */
1181	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1182	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1183	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1184	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1185	* (A shadow copy of a guest register can only be held in a one host register,
1186	* there are no duplicate copies or ambiguities like that). */
1187	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1188	#endif
1189
1190	/** Host register allocation tracking. */
1191	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1192	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1193	/** Host SIMD register allocation tracking. */
1194	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1195	#endif
1196
1197	/** Variables and arguments. */
1198	IEMNATIVEVAR aVars[9];
1199	} IEMNATIVECORESTATE;
1200	/** Pointer to core state. */
1201	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1202	/** Pointer to const core state. */
1203	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1204
1205	/** @def IEMNATIVE_VAR_IDX_UNPACK
1206	* @returns Index into IEMNATIVECORESTATE::aVars.
1207	* @param a_idxVar Variable index w/ magic (in strict builds).
1208	*/
1209	/** @def IEMNATIVE_VAR_IDX_PACK
1210	* @returns Variable index w/ magic (in strict builds).
1211	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1212	*/
1213	#ifdef VBOX_STRICT
1214	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1215	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1216	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1217	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1218	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1219	#else
1220	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1221	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1222	#endif
1223
1224
1225	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1226	/** Clear the dirty state of the given guest SIMD register. */
1227	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1228	do { \
1229	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1230	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1231	} while (0)
1232
1233	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1234	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1235	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1236	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1237	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1238	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1239	/** Returns whether the given guest SIMD register is dirty. */
1240	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1241	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1242
1243	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1244	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1245	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1246	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1247	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1248	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1249
1250	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1251	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1252	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1253	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1254	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1255	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1256	#endif
1257
1258
1259	/**
1260	* Conditional stack entry.
1261	*/
1262	typedef struct IEMNATIVECOND
1263	{
1264	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1265	bool fInElse;
1266	/** The label for the IEM_MC_ELSE. */
1267	uint32_t idxLabelElse;
1268	/** The label for the IEM_MC_ENDIF. */
1269	uint32_t idxLabelEndIf;
1270	/** The initial state snapshot as the if-block starts executing. */
1271	IEMNATIVECORESTATE InitialState;
1272	/** The state snapshot at the end of the if-block. */
1273	IEMNATIVECORESTATE IfFinalState;
1274	} IEMNATIVECOND;
1275	/** Pointer to a condition stack entry. */
1276	typedef IEMNATIVECOND *PIEMNATIVECOND;
1277
1278
1279	/**
1280	* Native recompiler state.
1281	*/
1282	typedef struct IEMRECOMPILERSTATE
1283	{
1284	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1285	* IEMNATIVEINSTR units. */
1286	uint32_t cInstrBufAlloc;
1287	#ifdef VBOX_STRICT
1288	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1289	uint32_t offInstrBufChecked;
1290	#else
1291	uint32_t uPadding1; /* We don't keep track of the size here... */
1292	#endif
1293	/** Fixed temporary code buffer for native recompilation. */
1294	PIEMNATIVEINSTR pInstrBuf;
1295
1296	/** Bitmaps with the label types used. */
1297	uint64_t bmLabelTypes;
1298	/** Actual number of labels in paLabels. */
1299	uint32_t cLabels;
1300	/** Max number of entries allowed in paLabels before reallocating it. */
1301	uint32_t cLabelsAlloc;
1302	/** Labels defined while recompiling (referenced by fixups). */
1303	PIEMNATIVELABEL paLabels;
1304	/** Array with indexes of unique labels (uData always 0). */
1305	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1306
1307	/** Actual number of fixups paFixups. */
1308	uint32_t cFixups;
1309	/** Max number of entries allowed in paFixups before reallocating it. */
1310	uint32_t cFixupsAlloc;
1311	/** Buffer used by the recompiler for recording fixups when generating code. */
1312	PIEMNATIVEFIXUP paFixups;
1313
1314	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1315	/** Number of debug info entries allocated for pDbgInfo. */
1316	uint32_t cDbgInfoAlloc;
1317	uint32_t uPadding;
1318	/** Debug info. */
1319	PIEMTBDBG pDbgInfo;
1320	#endif
1321
1322	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1323	/** The current call index (liveness array and threaded calls in TB). */
1324	uint32_t idxCurCall;
1325	/** Number of liveness entries allocated. */
1326	uint32_t cLivenessEntriesAlloc;
1327	/** Liveness entries for all the calls in the TB begin recompiled.
1328	* The entry for idxCurCall contains the info for what the next call will
1329	* require wrt registers. (Which means the last entry is the initial liveness
1330	* state.) */
1331	PIEMLIVENESSENTRY paLivenessEntries;
1332	#endif
1333
1334	/** The translation block being recompiled. */
1335	PCIEMTB pTbOrg;
1336	/** The VMCPU structure of the EMT. */
1337	PVMCPUCC pVCpu;
1338
1339	/** Condition sequence number (for generating unique labels). */
1340	uint16_t uCondSeqNo;
1341	/** Check IRQ seqeunce number (for generating unique labels). */
1342	uint16_t uCheckIrqSeqNo;
1343	/** TLB load sequence number (for generating unique labels). */
1344	uint16_t uTlbSeqNo;
1345	/** The current condition stack depth (aCondStack). */
1346	uint8_t cCondDepth;
1347
1348	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1349	uint8_t cArgsX;
1350	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1351	uint32_t fCImpl;
1352	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1353	uint32_t fMc;
1354	/** The expected IEMCPU::fExec value for the current call/instruction. */
1355	uint32_t fExec;
1356	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1357	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1358	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1359	*
1360	* This is an optimization because these control registers can only be changed from
1361	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1362	* consisting of multiple SIMD instructions.
1363	*/
1364	uint32_t fSimdRaiseXcptChecksEmitted;
1365	#endif
1366
1367	/** Core state requiring care with branches. */
1368	IEMNATIVECORESTATE Core;
1369
1370	/** The condition nesting stack. */
1371	IEMNATIVECOND aCondStack[2];
1372
1373	#ifndef IEM_WITH_THROW_CATCH
1374	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1375	* for recompilation error handling. */
1376	jmp_buf JmpBuf;
1377	#endif
1378	} IEMRECOMPILERSTATE;
1379	/** Pointer to a native recompiler state. */
1380	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1381
1382
1383	/** @def IEMNATIVE_TRY_SETJMP
1384	* Wrapper around setjmp / try, hiding all the ugly differences.
1385	*
1386	* @note Use with extreme care as this is a fragile macro.
1387	* @param a_pReNative The native recompile state.
1388	* @param a_rcTarget The variable that should receive the status code in case
1389	* of a longjmp/throw.
1390	*/
1391	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1392	* Start wrapper for catch / setjmp-else.
1393	*
1394	* This will set up a scope.
1395	*
1396	* @note Use with extreme care as this is a fragile macro.
1397	* @param a_pReNative The native recompile state.
1398	* @param a_rcTarget The variable that should receive the status code in case
1399	* of a longjmp/throw.
1400	*/
1401	/** @def IEMNATIVE_CATCH_LONGJMP_END
1402	* End wrapper for catch / setjmp-else.
1403	*
1404	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1405	* up the state.
1406	*
1407	* @note Use with extreme care as this is a fragile macro.
1408	* @param a_pReNative The native recompile state.
1409	*/
1410	/** @def IEMNATIVE_DO_LONGJMP
1411	*
1412	* Wrapper around longjmp / throw.
1413	*
1414	* @param a_pReNative The native recompile state.
1415	* @param a_rc The status code jump back with / throw.
1416	*/
1417	#ifdef IEM_WITH_THROW_CATCH
1418	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1419	a_rcTarget = VINF_SUCCESS; \
1420	try
1421	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1422	catch (int rcThrown) \
1423	{ \
1424	a_rcTarget = rcThrown
1425	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1426	} \
1427	((void)0)
1428	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1429	#else /* !IEM_WITH_THROW_CATCH */
1430	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1431	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1432	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1433	else \
1434	{ \
1435	((void)0)
1436	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1437	}
1438	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1439	#endif /* !IEM_WITH_THROW_CATCH */
1440
1441
1442	/**
1443	* Native recompiler worker for a threaded function.
1444	*
1445	* @returns New code buffer offset; throws VBox status code in case of a failure.
1446	* @param pReNative The native recompiler state.
1447	* @param off The current code buffer offset.
1448	* @param pCallEntry The threaded call entry.
1449	*
1450	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1451	*/
1452	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1453	/** Pointer to a native recompiler worker for a threaded function. */
1454	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1455
1456	/** Defines a native recompiler worker for a threaded function.
1457	* @see FNIEMNATIVERECOMPFUNC */
1458	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1459	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1460
1461	/** Prototypes a native recompiler function for a threaded function.
1462	* @see FNIEMNATIVERECOMPFUNC */
1463	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1464
1465
1466	/**
1467	* Native recompiler liveness analysis worker for a threaded function.
1468	*
1469	* @param pCallEntry The threaded call entry.
1470	* @param pIncoming The incoming liveness state entry.
1471	* @param pOutgoing The outgoing liveness state entry.
1472	*/
1473	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1474	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1475	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1476	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1477
1478	/** Defines a native recompiler liveness analysis worker for a threaded function.
1479	* @see FNIEMNATIVELIVENESSFUNC */
1480	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1481	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1482
1483	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1484	* @see FNIEMNATIVELIVENESSFUNC */
1485	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1486
1487
1488	/** Define a native recompiler helper function, safe to call from the TB code. */
1489	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1490	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1491	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1492	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1493	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1494	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1495	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1496	a_RetType (VBOXCALL *a_Name) a_ArgList
1497
1498
1499	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1500	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1501	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1502	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1503	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1504	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1505	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1506	uint8_t idxHstSimdReg = UINT8_MAX,
1507	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1508	# endif
1509	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1510	uint32_t offPc, uint32_t cInstrSkipped);
1511	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1512
1513	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1514	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1515	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1516	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1517	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1518	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1519
1520	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1521	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1522	bool fPreferVolatile = true);
1523	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1524	bool fPreferVolatile = true);
1525	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1526	IEMNATIVEGSTREG enmGstReg,
1527	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1528	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1529	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1530	IEMNATIVEGSTREG enmGstReg);
1531
1532	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1533	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1534	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1535	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1536	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1537	#endif
1538	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1539	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1540	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1541	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1542	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1543	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1544	#endif
1545	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1546	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1547	uint32_t fKeepVars = 0);
1548	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1549	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1550	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1551	uint32_t fHstRegsActiveShadows);
1552	#ifdef VBOX_STRICT
1553	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1554	#endif
1555	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1556	uint64_t fGstShwExcept, bool fFlushShadows);
1557	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1558	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1559	#endif
1560	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1561	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1562	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1563	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1564	#endif
1565
1566
1567	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1568	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1569	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1570	bool fPreferVolatile = true);
1571	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1572	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1573	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1574	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1575	bool fNoVolatileRegs = false);
1576	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1577	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1578	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1579	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1580	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1581	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1582	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1583	#endif
1584
1585	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1586	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1587	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1588	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1589	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1590	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1591	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1592	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1593	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1594	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1595	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1596	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1597	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1598	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1599	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1600	#endif
1601	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1602	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1603	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1604	uint32_t fHstRegsNotToSave);
1605	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1606	uint32_t fHstRegsNotToSave);
1607	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1608	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1609
1610	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1611	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1612	#ifdef VBOX_STRICT
1613	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1614	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1615	IEMNATIVEGSTREG enmGstReg);
1616	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1617	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1618	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1619	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1620	# endif
1621	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1622	#endif
1623	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1624	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1625	#endif
1626	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1627	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1628	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1629	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1630	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1631	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1632	PCIEMTHRDEDCALLENTRY pCallEntry);
1633	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1634	uint8_t idxAddrReg, uint8_t idxInstr);
1635	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1636	uint8_t idxAddrReg, uint8_t idxInstr);
1637	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1638	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1639
1640
1641	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1642	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1643	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1644	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1645	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1646	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1647	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1648	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1649	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1650	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1651
1652	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1653	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1654	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1655	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1656	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1657	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1658	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1659	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1660	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1661	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1662	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1663	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1664	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1665	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1666	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1667	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1668	#endif
1669	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1670	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1671	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1672	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1673	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1674	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1675	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1676	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1677	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1678	#endif
1679	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1680	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1681	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1682	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1683	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1684	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1685	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1686
1687	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1688	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1689	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1690	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1691	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1692	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1693	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1694	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1695	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1696	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1697	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1698	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1699	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1700	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1701	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1702	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1703	#endif
1704	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1705	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1706	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1707	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1708	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1709	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1710	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1711	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1712	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1713	#endif
1714	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1715	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1716	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1717	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1718	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1719	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1720	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1721
1722	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1723	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1724	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1725	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1726	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1727	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1728	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1729	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1730	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1731	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1732	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1733	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1734	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1735	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1736	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1737	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1738	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1739	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1740	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1741	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1742	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1743	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1744
1745	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1746	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1747	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1748	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1749	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1750	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1751	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1752	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1753	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1754	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1755	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1756	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1757	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1758	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1759	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1760	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1761	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1762	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1763	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1764	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1765	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1766	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1767
1768	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1769	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1770	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1771	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1772
1773
1774	/**
1775	* Info about shadowed guest register values.
1776	* @see IEMNATIVEGSTREG
1777	*/
1778	typedef struct IEMANTIVEGSTREGINFO
1779	{
1780	/** Offset in VMCPU. */
1781	uint32_t off;
1782	/** The field size. */
1783	uint8_t cb;
1784	/** Name (for logging). */
1785	const char *pszName;
1786	} IEMANTIVEGSTREGINFO;
1787	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1788	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1789	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1790	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1791	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1792
1793
1794
1795	/**
1796	* Ensures that there is sufficient space in the instruction output buffer.
1797	*
1798	* This will reallocate the buffer if needed and allowed.
1799	*
1800	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1801	* allocation size.
1802	*
1803	* @returns Pointer to the instruction output buffer on success; throws VBox
1804	* status code on failure, so no need to check it.
1805	* @param pReNative The native recompile state.
1806	* @param off Current instruction offset. Works safely for UINT32_MAX
1807	* as well.
1808	* @param cInstrReq Number of instruction about to be added. It's okay to
1809	* overestimate this a bit.
1810	*/
1811	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1812	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1813	{
1814	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1815	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1816	{
1817	#ifdef VBOX_STRICT
1818	pReNative->offInstrBufChecked = offChecked;
1819	#endif
1820	return pReNative->pInstrBuf;
1821	}
1822	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1823	}
1824
1825	/**
1826	* Checks that we didn't exceed the space requested in the last
1827	* iemNativeInstrBufEnsure() call.
1828	*/
1829	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1830	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1831	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1832
1833	/**
1834	* Checks that a variable index is valid.
1835	*/
1836	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1837	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1838	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1839	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1840	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1841	("%s=%#x\n", #a_idxVar, a_idxVar))
1842	#else
1843	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1844	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1845	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1846	#endif
1847
1848	/**
1849	* Checks that a variable index is valid and that the variable is assigned the
1850	* correct argument number.
1851	* This also adds a RT_NOREF of a_idxVar.
1852	*/
1853	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1854	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1855	RT_NOREF_PV(a_idxVar); \
1856	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1857	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1858	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1859	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1860	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1861	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1862	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1863	a_uArgNo)); \
1864	} while (0)
1865	#else
1866	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1867	RT_NOREF_PV(a_idxVar); \
1868	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1869	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1870	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1871	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1872	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1873	} while (0)
1874	#endif
1875
1876
1877	/**
1878	* Checks that a variable has the expected size.
1879	*/
1880	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1881	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1882	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1883	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1884
1885
1886	/**
1887	* Calculates the stack address of a variable as a [r]BP displacement value.
1888	*/
1889	DECL_FORCE_INLINE(int32_t)
1890	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1891	{
1892	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1893	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1894	}
1895
1896
1897	/**
1898	* Releases the variable's register.
1899	*
1900	* The register must have been previously acquired calling
1901	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1902	* iemNativeVarRegisterSetAndAcquire().
1903	*/
1904	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1905	{
1906	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1907	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1908	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1909	}
1910
1911
1912	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1913	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1914	{
1915	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
1916	iemNativeVarRegisterRelease(pReNative, idxVar);
1917	}
1918	#endif
1919
1920
1921	/**
1922	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1923	*
1924	* @returns The flush mask.
1925	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1926	* @param fGstShwFlush The starting flush mask.
1927	*/
1928	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1929	{
1930	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1931	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1932	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1933	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1934	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1935	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1936	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1937	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1938	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1939	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1940	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1941	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1942	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1943	return fGstShwFlush;
1944	}
1945
1946
1947	/** Number of hidden arguments for CIMPL calls.
1948	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1949	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1950	# define IEM_CIMPL_HIDDEN_ARGS 3
1951	#else
1952	# define IEM_CIMPL_HIDDEN_ARGS 2
1953	#endif
1954
1955
1956	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1957	/** Number of hidden arguments for SSE_AIMPL calls. */
1958	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1959	/** Number of hidden arguments for AVX_AIMPL calls. */
1960	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
1961	#endif
1962
1963
1964	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1965
1966	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1967	/**
1968	* Helper for iemNativeLivenessGetStateByGstReg.
1969	*
1970	* @returns IEMLIVENESS_STATE_XXX
1971	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1972	* ORed together.
1973	*/
1974	DECL_FORCE_INLINE(uint32_t)
1975	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1976	{
1977	/* INPUT trumps anything else. */
1978	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1979	return IEMLIVENESS_STATE_INPUT;
1980
1981	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1982	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1983	{
1984	/* If not all sub-fields are clobbered they must be considered INPUT. */
1985	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1986	return IEMLIVENESS_STATE_INPUT;
1987	return IEMLIVENESS_STATE_CLOBBERED;
1988	}
1989
1990	/* XCPT_OR_CALL trumps UNUSED. */
1991	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1992	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1993
1994	return IEMLIVENESS_STATE_UNUSED;
1995	}
1996	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1997
1998
1999	DECL_FORCE_INLINE(uint32_t)
2000	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2001	{
2002	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2003	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2004	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2005	# else
2006	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2007	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2008	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2009	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2010	# endif
2011	}
2012
2013
2014	DECL_FORCE_INLINE(uint32_t)
2015	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2016	{
2017	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2018	if (enmGstReg == kIemNativeGstReg_EFlags)
2019	{
2020	/* Merge the eflags states to one. */
2021	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2022	uRet = RT_BIT_32(uRet);
2023	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2024	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2025	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2026	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2027	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2028	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2029	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2030	# else
2031	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2032	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2033	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2034	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2035	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2036	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2037	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2038	# endif
2039	}
2040	return uRet;
2041	}
2042
2043
2044	# ifdef VBOX_STRICT
2045	/** For assertions only, user checks that idxCurCall isn't zerow. */
2046	DECL_FORCE_INLINE(uint32_t)
2047	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2048	{
2049	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2050	}
2051	# endif /* VBOX_STRICT */
2052
2053	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2054
2055
2056	/**
2057	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2058	*/
2059	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2060	{
2061	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2062	return IEM_CIMPL_HIDDEN_ARGS;
2063	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2064	return 1;
2065	return 0;
2066	}
2067
2068
2069	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2070	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2071	{
2072	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2073
2074	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2075	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2076	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2077	return (uint8_t)idxReg;
2078	}
2079
2080
2081
2082	/*********************************************************************************************************************************
2083	* Register Allocator (GPR) *
2084	*********************************************************************************************************************************/
2085
2086	/**
2087	* Marks host register @a idxHstReg as containing a shadow copy of guest
2088	* register @a enmGstReg.
2089	*
2090	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2091	* host register before calling.
2092	*/
2093	DECL_FORCE_INLINE(void)
2094	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2095	{
2096	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2097	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2098	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2099
2100	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2101	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2102	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2103	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2104	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2105	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2106	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2107	#else
2108	RT_NOREF(off);
2109	#endif
2110	}
2111
2112
2113	/**
2114	* Clear any guest register shadow claims from @a idxHstReg.
2115	*
2116	* The register does not need to be shadowing any guest registers.
2117	*/
2118	DECL_FORCE_INLINE(void)
2119	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2120	{
2121	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2122	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2123	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2124	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2125	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2126	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2127	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2128	#endif
2129
2130	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2131	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2132	if (fGstRegs)
2133	{
2134	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2135	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2136	while (fGstRegs)
2137	{
2138	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2139	fGstRegs &= ~RT_BIT_64(iGstReg);
2140	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2141	}
2142	}
2143	#else
2144	RT_NOREF(off);
2145	#endif
2146
2147	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2148	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2149	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2150	}
2151
2152
2153	/**
2154	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2155	* and global overview flags.
2156	*/
2157	DECL_FORCE_INLINE(void)
2158	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2159	{
2160	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2161	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2162	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2163	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2164	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2165	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2166	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2167	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2168	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2169	#endif
2170
2171	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2172	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2173	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2174	#else
2175	RT_NOREF(off);
2176	#endif
2177
2178	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2179	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2180	if (!fGstRegShadowsNew)
2181	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2182	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2183	}
2184
2185
2186	#if 0 /* unused */
2187	/**
2188	* Clear any guest register shadow claim for @a enmGstReg.
2189	*/
2190	DECL_FORCE_INLINE(void)
2191	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2192	{
2193	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2194	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2195	{
2196	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2197	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2198	}
2199	}
2200	#endif
2201
2202
2203	/**
2204	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2205	* as the new shadow of it.
2206	*
2207	* Unlike the other guest reg shadow helpers, this does the logging for you.
2208	* However, it is the liveness state is not asserted here, the caller must do
2209	* that.
2210	*/
2211	DECL_FORCE_INLINE(void)
2212	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2213	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2214	{
2215	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2216	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2217	{
2218	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2219	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2220	if (idxHstRegOld == idxHstRegNew)
2221	return;
2222	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2223	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2224	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2225	}
2226	else
2227	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2228	g_aGstShadowInfo[enmGstReg].pszName));
2229	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2230	}
2231
2232
2233	/**
2234	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2235	* to @a idxRegTo.
2236	*/
2237	DECL_FORCE_INLINE(void)
2238	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2239	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2240	{
2241	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2242	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2243	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2244	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2245	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2246	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2247	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2248	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2249	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2250
2251	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2252	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2253	if (!fGstRegShadowsFrom)
2254	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2255	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2256	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2257	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2258	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2259	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2260	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2261	#else
2262	RT_NOREF(off);
2263	#endif
2264	}
2265
2266
2267	/**
2268	* Flushes any delayed guest register writes.
2269	*
2270	* This must be called prior to calling CImpl functions and any helpers that use
2271	* the guest state (like raising exceptions) and such.
2272	*
2273	* This optimization has not yet been implemented. The first target would be
2274	* RIP updates, since these are the most common ones.
2275	*/
2276	DECL_INLINE_THROW(uint32_t)
2277	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2278	{
2279	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2280	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2281	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2282	#else
2283	RT_NOREF(pReNative, fGstShwExcept);
2284	#endif
2285
2286	#if defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR) \|\| defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK)
2287	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2288	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2289	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2290	#else
2291	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2292	return off;
2293	#endif
2294	}
2295
2296
2297
2298	/*********************************************************************************************************************************
2299	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2300	*********************************************************************************************************************************/
2301
2302	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2303
2304	DECL_FORCE_INLINE(uint8_t)
2305	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2306	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2307	{
2308	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2309
2310	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2311	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2312	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2313	return idxSimdReg;
2314	}
2315
2316
2317	/**
2318	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2319	* SIMD register @a enmGstSimdReg.
2320	*
2321	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2322	* host register before calling.
2323	*/
2324	DECL_FORCE_INLINE(void)
2325	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2326	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2327	{
2328	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2329	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2330	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2331
2332	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2333	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2334	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2335	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2336	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2337	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2338	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2339	#else
2340	RT_NOREF(off);
2341	#endif
2342	}
2343
2344
2345	/**
2346	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2347	* to @a idxSimdRegTo.
2348	*/
2349	DECL_FORCE_INLINE(void)
2350	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2351	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2352	{
2353	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2354	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2355	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2356	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2357	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2358	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2359	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2360	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2361	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2362	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2363	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2364
2365	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2366	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2367	if (!fGstRegShadowsFrom)
2368	{
2369	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2370	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2371	}
2372	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2373	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2374	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2375	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2376	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2377	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2378	#else
2379	RT_NOREF(off);
2380	#endif
2381	}
2382
2383
2384	/**
2385	* Clear any guest register shadow claims from @a idxHstSimdReg.
2386	*
2387	* The register does not need to be shadowing any guest registers.
2388	*/
2389	DECL_FORCE_INLINE(void)
2390	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2391	{
2392	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2393	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2394	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2395	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2396	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2397	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2398	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2399
2400	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2401	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2402	if (fGstRegs)
2403	{
2404	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2405	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2406	while (fGstRegs)
2407	{
2408	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2409	fGstRegs &= ~RT_BIT_64(iGstReg);
2410	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2411	}
2412	}
2413	#else
2414	RT_NOREF(off);
2415	#endif
2416
2417	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2418	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2419	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2420	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2421	}
2422
2423	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2424
2425
2426	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2427	/**
2428	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2429	*/
2430	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2431	{
2432	if (pReNative->Core.offPc)
2433	return iemNativeEmitPcWritebackSlow(pReNative, off);
2434	return off;
2435	}
2436	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2437
2438
2439	/** @} */
2440
2441	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2442

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 104035

Download in other formats: