IEMN8veRecompiler.h@ 103993

Last change on this file since 103993 was 103993, checked in by vboxsync, 12 months ago
VMM/IEM: Implement native emitters for IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(), IEM_MC_REF_XREG_R32_CONST(), IEM_MC_REF_XREG_R64_CONST(), IEM_MC_REF_XREG_U32_CONST(), IEM_MC_REF_XREG_U64_CONST() and IEM_MC_STORE_SSE_RESULT(), bugref:10614
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 119.5 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103993 2024-03-21 17:59:07Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
227	\| RT_BIT_32(ARMV8_A64_REG_Q30))
228	# endif
229	# endif
230
231	#else
232	# error "port me"
233	#endif
234	/** @} */
235
236	/** @name Call related registers.
237	* @{ */
238	/** @def IEMNATIVE_CALL_RET_GREG
239	* The return value register. */
240	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
241	* Number of arguments in registers. */
242	/** @def IEMNATIVE_CALL_ARG0_GREG
243	* The general purpose register carrying argument \#0. */
244	/** @def IEMNATIVE_CALL_ARG1_GREG
245	* The general purpose register carrying argument \#1. */
246	/** @def IEMNATIVE_CALL_ARG2_GREG
247	* The general purpose register carrying argument \#2. */
248	/** @def IEMNATIVE_CALL_ARG3_GREG
249	* The general purpose register carrying argument \#3. */
250	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
251	* Mask of registers the callee will not save and may trash. */
252	#ifdef RT_ARCH_AMD64
253	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
254
255	# ifdef RT_OS_WINDOWS
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
257	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
258	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
259	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
260	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
261	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
262	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
263	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
264	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
265	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
266	\| RT_BIT_32(X86_GREG_xCX) \
267	\| RT_BIT_32(X86_GREG_xDX) \
268	\| RT_BIT_32(X86_GREG_x8) \
269	\| RT_BIT_32(X86_GREG_x9) \
270	\| RT_BIT_32(X86_GREG_x10) \
271	\| RT_BIT_32(X86_GREG_x11) )
272	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
273	/* xmm0 - xmm5 are marked as volatile. */
274	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
275	# endif
276
277	# else
278	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
279	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
280	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
281	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
282	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
283	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
289	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
290	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
291	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
292	\| RT_BIT_32(X86_GREG_xCX) \
293	\| RT_BIT_32(X86_GREG_xDX) \
294	\| RT_BIT_32(X86_GREG_xDI) \
295	\| RT_BIT_32(X86_GREG_xSI) \
296	\| RT_BIT_32(X86_GREG_x8) \
297	\| RT_BIT_32(X86_GREG_x9) \
298	\| RT_BIT_32(X86_GREG_x10) \
299	\| RT_BIT_32(X86_GREG_x11) )
300	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
301	/* xmm0 - xmm15 are marked as volatile. */
302	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
303	# endif
304	# endif
305
306	#elif defined(RT_ARCH_ARM64)
307	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
308	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
309	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
310	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
311	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
312	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
313	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
314	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
315	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
316	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
317	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
318	\| RT_BIT_32(ARMV8_A64_REG_X1) \
319	\| RT_BIT_32(ARMV8_A64_REG_X2) \
320	\| RT_BIT_32(ARMV8_A64_REG_X3) \
321	\| RT_BIT_32(ARMV8_A64_REG_X4) \
322	\| RT_BIT_32(ARMV8_A64_REG_X5) \
323	\| RT_BIT_32(ARMV8_A64_REG_X6) \
324	\| RT_BIT_32(ARMV8_A64_REG_X7) )
325	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
326	\| RT_BIT_32(ARMV8_A64_REG_X1) \
327	\| RT_BIT_32(ARMV8_A64_REG_X2) \
328	\| RT_BIT_32(ARMV8_A64_REG_X3) \
329	\| RT_BIT_32(ARMV8_A64_REG_X4) \
330	\| RT_BIT_32(ARMV8_A64_REG_X5) \
331	\| RT_BIT_32(ARMV8_A64_REG_X6) \
332	\| RT_BIT_32(ARMV8_A64_REG_X7) \
333	\| RT_BIT_32(ARMV8_A64_REG_X8) \
334	\| RT_BIT_32(ARMV8_A64_REG_X9) \
335	\| RT_BIT_32(ARMV8_A64_REG_X10) \
336	\| RT_BIT_32(ARMV8_A64_REG_X11) \
337	\| RT_BIT_32(ARMV8_A64_REG_X12) \
338	\| RT_BIT_32(ARMV8_A64_REG_X13) \
339	\| RT_BIT_32(ARMV8_A64_REG_X14) \
340	\| RT_BIT_32(ARMV8_A64_REG_X15) \
341	\| RT_BIT_32(ARMV8_A64_REG_X16) \
342	\| RT_BIT_32(ARMV8_A64_REG_X17) )
343	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
344	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
345	* so to simplify our life a bit we just mark everything as volatile. */
346	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
347	# endif
348
349	#endif
350
351	/** This is the maximum argument count we'll ever be needing. */
352	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
353	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
354	#else
355	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
356	#endif
357	/** @} */
358
359
360	/** @def IEMNATIVE_HST_GREG_COUNT
361	* Number of host general purpose registers we tracker. */
362	/** @def IEMNATIVE_HST_GREG_MASK
363	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
364	* inverted register masks and such to get down to a correct set of regs. */
365	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
366	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
367	* Number of host SIMD registers we track. */
368	/** @def IEMNATIVE_HST_SIMD_REG_MASK
369	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
370	* inverted register masks and such to get down to a correct set of regs. */
371	#endif
372	#ifdef RT_ARCH_AMD64
373	# define IEMNATIVE_HST_GREG_COUNT 16
374	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
375
376	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
377	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
378	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
379	# endif
380
381	#elif defined(RT_ARCH_ARM64)
382	# define IEMNATIVE_HST_GREG_COUNT 32
383	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
384
385	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
386	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
387	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
388	# endif
389
390	#else
391	# error "Port me!"
392	#endif
393
394
395	/** Native code generator label types. */
396	typedef enum
397	{
398	kIemNativeLabelType_Invalid = 0,
399	/*
400	* Labels w/o data, only once instance per TB.
401	*
402	* Note! Jumps to these requires instructions that are capable of spanning
403	* the max TB length.
404	*/
405	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
406	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
407	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
408	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
409	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
410	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
411	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
412	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
413	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
414	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
415	kIemNativeLabelType_ObsoleteTb,
416	kIemNativeLabelType_NeedCsLimChecking,
417	kIemNativeLabelType_CheckBranchMiss,
418	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
419	/* Manually defined labels. */
420	kIemNativeLabelType_Return,
421	kIemNativeLabelType_ReturnBreak,
422	kIemNativeLabelType_ReturnWithFlags,
423	kIemNativeLabelType_NonZeroRetOrPassUp,
424	/** The last fixup for branches that can span almost the whole TB length. */
425	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
426
427	/*
428	* Labels with data, potentially multiple instances per TB:
429	*
430	* These are localized labels, so no fixed jump type restrictions here.
431	*/
432	kIemNativeLabelType_FirstWithMultipleInstances,
433	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
434	kIemNativeLabelType_Else,
435	kIemNativeLabelType_Endif,
436	kIemNativeLabelType_CheckIrq,
437	kIemNativeLabelType_TlbLookup,
438	kIemNativeLabelType_TlbMiss,
439	kIemNativeLabelType_TlbDone,
440	kIemNativeLabelType_End
441	} IEMNATIVELABELTYPE;
442
443	/** Native code generator label definition. */
444	typedef struct IEMNATIVELABEL
445	{
446	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
447	* the epilog. */
448	uint32_t off;
449	/** The type of label (IEMNATIVELABELTYPE). */
450	uint16_t enmType;
451	/** Additional label data, type specific. */
452	uint16_t uData;
453	} IEMNATIVELABEL;
454	/** Pointer to a label. */
455	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
456
457
458	/** Native code generator fixup types. */
459	typedef enum
460	{
461	kIemNativeFixupType_Invalid = 0,
462	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
463	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
464	kIemNativeFixupType_Rel32,
465	#elif defined(RT_ARCH_ARM64)
466	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
467	kIemNativeFixupType_RelImm26At0,
468	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
469	kIemNativeFixupType_RelImm19At5,
470	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
471	kIemNativeFixupType_RelImm14At5,
472	#endif
473	kIemNativeFixupType_End
474	} IEMNATIVEFIXUPTYPE;
475
476	/** Native code generator fixup. */
477	typedef struct IEMNATIVEFIXUP
478	{
479	/** Code offset of the fixup location. */
480	uint32_t off;
481	/** The IEMNATIVELABEL this is a fixup for. */
482	uint16_t idxLabel;
483	/** The fixup type (IEMNATIVEFIXUPTYPE). */
484	uint8_t enmType;
485	/** Addend or other data. */
486	int8_t offAddend;
487	} IEMNATIVEFIXUP;
488	/** Pointer to a native code generator fixup. */
489	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
490
491
492	/**
493	* One bit of the state.
494	*
495	* Each register state takes up two bits. We keep the two bits in two separate
496	* 64-bit words to simplify applying them to the guest shadow register mask in
497	* the register allocator.
498	*/
499	typedef union IEMLIVENESSBIT
500	{
501	uint64_t bm64;
502	RT_GCC_EXTENSION struct
503	{ /* bit no */
504	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
505	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
506	uint64_t fCr0 : 1; /*< 0x11 / 17: /
507	uint64_t fFcw : 1; /*< 0x12 / 18: /
508	uint64_t fFsw : 1; /*< 0x13 / 19: /
509	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
510	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
511	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
512	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
513	uint64_t fCr4 : 1; /*< 0x2c / 44: /
514	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
515	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
516	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
517	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
518	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
519	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
520	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
521	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
522	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
523	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
524	};
525	} IEMLIVENESSBIT;
526	AssertCompileSize(IEMLIVENESSBIT, 8);
527
528	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
529	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
530	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
531	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
532	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
533	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
534	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
535
536
537	/**
538	* A liveness state entry.
539	*
540	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
541	* Once we add a SSE register shadowing, we'll add another 64-bit element for
542	* that.
543	*/
544	typedef union IEMLIVENESSENTRY
545	{
546	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
547	uint64_t bm64[16 / 8];
548	uint16_t bm32[16 / 4];
549	uint16_t bm16[16 / 2];
550	uint8_t bm8[ 16 / 1];
551	IEMLIVENESSBIT aBits[2];
552	#else
553	uint64_t bm64[32 / 8];
554	uint16_t bm32[32 / 4];
555	uint16_t bm16[32 / 2];
556	uint8_t bm8[ 32 / 1];
557	IEMLIVENESSBIT aBits[4];
558	#endif
559	RT_GCC_EXTENSION struct
560	{
561	/** Bit \#0 of the register states. */
562	IEMLIVENESSBIT Bit0;
563	/** Bit \#1 of the register states. */
564	IEMLIVENESSBIT Bit1;
565	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
566	/** Bit \#2 of the register states. */
567	IEMLIVENESSBIT Bit2;
568	/** Bit \#3 of the register states. */
569	IEMLIVENESSBIT Bit3;
570	#endif
571	};
572	} IEMLIVENESSENTRY;
573	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
574	AssertCompileSize(IEMLIVENESSENTRY, 16);
575	#else
576	AssertCompileSize(IEMLIVENESSENTRY, 32);
577	#endif
578	/** Pointer to a liveness state entry. */
579	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
580	/** Pointer to a const liveness state entry. */
581	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
582
583	/** @name 64-bit value masks for IEMLIVENESSENTRY.
584	* @{ / / 0xzzzzyyyyxxxxwwww */
585	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
586
587	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
588	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
589	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
590
591	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
592	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
593	#endif
594
595	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
596	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
597
598	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
599	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
600	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
601	#endif
602	/** @} */
603
604
605	/** @name The liveness state for a register.
606	*
607	* The state values have been picked to with state accumulation in mind (what
608	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
609	* performance critical work done with the values.
610	*
611	* This is a compressed state that only requires 2 bits per register.
612	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
613	* 1. the incoming state from the following call,
614	* 2. the outgoing state for this call,
615	* 3. mask of the entries set in the 2nd.
616	*
617	* The mask entry (3rd one above) will be used both when updating the outgoing
618	* state and when merging in incoming state for registers not touched by the
619	* current call.
620	*
621	* @{ */
622	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
623	/** The register will be clobbered and the current value thrown away.
624	*
625	* When this is applied to the state (2) we'll simply be AND'ing it with the
626	* (old) mask (3) and adding the register to the mask. This way we'll
627	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
628	* IEMLIVENESS_STATE_INPUT states. */
629	# define IEMLIVENESS_STATE_CLOBBERED 0
630	/** The register is unused in the remainder of the TB.
631	*
632	* This is an initial state and can not be set by any of the
633	* iemNativeLivenessFunc_xxxx callbacks. */
634	# define IEMLIVENESS_STATE_UNUSED 1
635	/** The register value is required in a potential call or exception.
636	*
637	* This means that the register value must be calculated and is best written to
638	* the state, but that any shadowing registers can be flushed thereafter as it's
639	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
640	*
641	* It is typically applied across the board, but we preserve incoming
642	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
643	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
644	* 1. r0 = old & ~mask;
645	* 2. r0 = t1 & (t1 >> 1)'
646	* 3. state \|= r0 \| 0b10;
647	* 4. mask = ~0;
648	*/
649	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
650	/** The register value is used as input.
651	*
652	* This means that the register value must be calculated and it is best to keep
653	* it in a register. It does not need to be writtent out as such. This is the
654	* highest priority state.
655	*
656	* Whether the call modifies the register or not isn't relevant to earlier
657	* calls, so that's not recorded.
658	*
659	* When applying this state we just or in the value in the outgoing state and
660	* mask. */
661	# define IEMLIVENESS_STATE_INPUT 3
662	/** Mask of the state bits. */
663	# define IEMLIVENESS_STATE_MASK 3
664	/** The number of bits per state. */
665	# define IEMLIVENESS_STATE_BIT_COUNT 2
666	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
667	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
668	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
669	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
670	/** Check if a register clobbering is expected given the (previous) liveness state.
671	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
672	* include INPUT if the register is used in more than one place. */
673	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
674
675	/** Check if all status flags are going to be clobbered and doesn't need
676	* calculating in the current step.
677	* @param a_pCurEntry The current liveness entry. */
678	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
679	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
680
681	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
682	/** The register is not used any more. */
683	# define IEMLIVENESS_STATE_UNUSED 0
684	/** Flag: The register is required in a potential exception or call. */
685	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
686	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
687	/** Flag: The register is read. */
688	# define IEMLIVENESS_STATE_READ 2
689	# define IEMLIVENESS_BIT_READ 1
690	/** Flag: The register is written. */
691	# define IEMLIVENESS_STATE_WRITE 4
692	# define IEMLIVENESS_BIT_WRITE 2
693	/** Flag: Unconditional call (not needed, can be redefined for research). */
694	# define IEMLIVENESS_STATE_CALL 8
695	# define IEMLIVENESS_BIT_CALL 3
696	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
697	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
698	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
699	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
700	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
701
702	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
703	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
704	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
705	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
706
707	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
708	/** @} */
709
710	/** @name Liveness helpers for builtin functions and similar.
711	*
712	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
713	* own set of manimulator macros for those.
714	*
715	* @{ */
716	/** Initializing the state as all unused. */
717	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
718	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
719	do { \
720	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
721	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
722	} while (0)
723	#else
724	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
725	do { \
726	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
727	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
728	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
729	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
730	} while (0)
731	#endif
732
733	/** Initializing the outgoing state with a potential xcpt or call state.
734	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
735	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
736	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
737	do { \
738	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
739	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
740	} while (0)
741	#else
742	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
743	do { \
744	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
745	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
746	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
747	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
748	} while (0)
749	#endif
750
751	/** Adds a segment base register as input to the outgoing state. */
752	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
753	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
754	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
755	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
756	} while (0)
757	#else
758	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
759	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
760	} while (0)
761	#endif
762
763	/** Adds a segment attribute register as input to the outgoing state. */
764	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
765	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
766	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
767	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
768	} while (0)
769	#else
770	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
771	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
772	} while (0)
773	#endif
774
775	/** Adds a segment limit register as input to the outgoing state. */
776	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
777	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
778	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
779	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
780	} while (0)
781	#else
782	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
783	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
784	} while (0)
785	#endif
786
787	/** Adds a segment limit register as input to the outgoing state. */
788	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
789	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
790	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
791	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
792	} while (0)
793	#else
794	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
795	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
796	} while (0)
797	#endif
798	/** @} */
799
800	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
801	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
802	* calculated and up to date. This is to double check that we haven't skipped
803	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
804	* @note has to be placed in
805	*/
806	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
807	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
808	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
809	#else
810	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
811	#endif
812
813
814	/**
815	* Guest registers that can be shadowed in GPRs.
816	*
817	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
818	* must be placed last, as the liveness state tracks it as 7 subcomponents and
819	* we don't want to waste space here.
820	*
821	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
822	* friends as well as IEMAllN8veLiveness.cpp.
823	*/
824	typedef enum IEMNATIVEGSTREG : uint8_t
825	{
826	kIemNativeGstReg_GprFirst = 0,
827	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
828	kIemNativeGstReg_Pc,
829	kIemNativeGstReg_Cr0,
830	kIemNativeGstReg_FpuFcw,
831	kIemNativeGstReg_FpuFsw,
832	kIemNativeGstReg_SegBaseFirst,
833	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
834	kIemNativeGstReg_SegAttribFirst,
835	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
836	kIemNativeGstReg_SegLimitFirst,
837	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
838	kIemNativeGstReg_SegSelFirst,
839	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
840	kIemNativeGstReg_Cr4,
841	kIemNativeGstReg_Xcr0,
842	kIemNativeGstReg_MxCsr,
843	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
844	kIemNativeGstReg_End
845	} IEMNATIVEGSTREG;
846	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
847	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
848
849	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
850	* @{ */
851	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
852	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
853	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
854	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
855	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
856	/** @} */
857
858	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
859
860	/**
861	* Guest registers that can be shadowed in host SIMD registers.
862	*
863	* @todo r=aeichner Liveness tracking
864	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
865	*/
866	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
867	{
868	kIemNativeGstSimdReg_SimdRegFirst = 0,
869	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
870	kIemNativeGstSimdReg_End
871	} IEMNATIVEGSTSIMDREG;
872
873	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
874	* @{ */
875	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
876	/** @} */
877
878	/**
879	* The Load/store size for a SIMD guest register.
880	*/
881	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
882	{
883	/** Invalid size. */
884	kIemNativeGstSimdRegLdStSz_Invalid = 0,
885	/** Loads the low 128-bit of a guest SIMD register. */
886	kIemNativeGstSimdRegLdStSz_Low128,
887	/** Loads the high 128-bit of a guest SIMD register. */
888	kIemNativeGstSimdRegLdStSz_High128,
889	/** Loads the whole 256-bits of a guest SIMD register. */
890	kIemNativeGstSimdRegLdStSz_256,
891	/** End value. */
892	kIemNativeGstSimdRegLdStSz_End
893	} IEMNATIVEGSTSIMDREGLDSTSZ;
894
895	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
896
897	/**
898	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
899	*/
900	typedef enum IEMNATIVEGSTREGUSE
901	{
902	/** The usage is read-only, the register holding the guest register
903	* shadow copy will not be modified by the caller. */
904	kIemNativeGstRegUse_ReadOnly = 0,
905	/** The caller will update the guest register (think: PC += cbInstr).
906	* The guest shadow copy will follow the returned register. */
907	kIemNativeGstRegUse_ForUpdate,
908	/** The call will put an entirely new value in the guest register, so
909	* if new register is allocate it will be returned uninitialized. */
910	kIemNativeGstRegUse_ForFullWrite,
911	/** The caller will use the guest register value as input in a calculation
912	* and the host register will be modified.
913	* This means that the returned host register will not be marked as a shadow
914	* copy of the guest register. */
915	kIemNativeGstRegUse_Calculation
916	} IEMNATIVEGSTREGUSE;
917
918	/**
919	* Guest registers (classes) that can be referenced.
920	*/
921	typedef enum IEMNATIVEGSTREGREF : uint8_t
922	{
923	kIemNativeGstRegRef_Invalid = 0,
924	kIemNativeGstRegRef_Gpr,
925	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
926	kIemNativeGstRegRef_EFlags,
927	kIemNativeGstRegRef_MxCsr,
928	kIemNativeGstRegRef_FpuReg,
929	kIemNativeGstRegRef_MReg,
930	kIemNativeGstRegRef_XReg,
931	kIemNativeGstRegRef_X87,
932	kIemNativeGstRegRef_XState,
933	//kIemNativeGstRegRef_YReg, - doesn't work.
934	kIemNativeGstRegRef_End
935	} IEMNATIVEGSTREGREF;
936
937
938	/** Variable kinds. */
939	typedef enum IEMNATIVEVARKIND : uint8_t
940	{
941	/** Customary invalid zero value. */
942	kIemNativeVarKind_Invalid = 0,
943	/** This is either in a register or on the stack. */
944	kIemNativeVarKind_Stack,
945	/** Immediate value - loaded into register when needed, or can live on the
946	* stack if referenced (in theory). */
947	kIemNativeVarKind_Immediate,
948	/** Variable reference - loaded into register when needed, never stack. */
949	kIemNativeVarKind_VarRef,
950	/** Guest register reference - loaded into register when needed, never stack. */
951	kIemNativeVarKind_GstRegRef,
952	/** End of valid values. */
953	kIemNativeVarKind_End
954	} IEMNATIVEVARKIND;
955
956
957	/** Variable or argument. */
958	typedef struct IEMNATIVEVAR
959	{
960	/** The kind of variable. */
961	IEMNATIVEVARKIND enmKind;
962	/** The variable size in bytes. */
963	uint8_t cbVar;
964	/** The first stack slot (uint64_t), except for immediate and references
965	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
966	* has a stack slot it has been initialized and has a value. Unused variables
967	* has neither a stack slot nor a host register assignment. */
968	uint8_t idxStackSlot;
969	/** The host register allocated for the variable, UINT8_MAX if not. */
970	uint8_t idxReg;
971	/** The argument number if argument, UINT8_MAX if regular variable. */
972	uint8_t uArgNo;
973	/** If referenced, the index (unpacked) of the variable referencing this one,
974	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
975	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
976	uint8_t idxReferrerVar;
977	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
978	* @todo not sure what this really is for... */
979	IEMNATIVEGSTREG enmGstReg;
980	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
981	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
982	* only valid when idxReg is not UINT8_MAX. */
983	bool fSimdReg : 1;
984	/** Set if the registered is currently used exclusively, false if the
985	* variable is idle and the register can be grabbed. */
986	bool fRegAcquired : 1;
987	#else
988	/** Set if the registered is currently used exclusively, false if the
989	* variable is idle and the register can be grabbed. */
990	bool fRegAcquired;
991	#endif
992
993	union
994	{
995	/** kIemNativeVarKind_Immediate: The immediate value. */
996	uint64_t uValue;
997	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
998	uint8_t idxRefVar;
999	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1000	struct
1001	{
1002	/** The class of register. */
1003	IEMNATIVEGSTREGREF enmClass;
1004	/** Index within the class. */
1005	uint8_t idx;
1006	} GstRegRef;
1007	} u;
1008	} IEMNATIVEVAR;
1009	/** Pointer to a variable or argument. */
1010	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1011	/** Pointer to a const variable or argument. */
1012	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1013
1014	/** What is being kept in a host register. */
1015	typedef enum IEMNATIVEWHAT : uint8_t
1016	{
1017	/** The traditional invalid zero value. */
1018	kIemNativeWhat_Invalid = 0,
1019	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1020	kIemNativeWhat_Var,
1021	/** Temporary register, this is typically freed when a MC completes. */
1022	kIemNativeWhat_Tmp,
1023	/** Call argument w/o a variable mapping. This is free (via
1024	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1025	kIemNativeWhat_Arg,
1026	/** Return status code.
1027	* @todo not sure if we need this... */
1028	kIemNativeWhat_rc,
1029	/** The fixed pVCpu (PVMCPUCC) register.
1030	* @todo consider offsetting this on amd64 to use negative offsets to access
1031	* more members using 8-byte disp. */
1032	kIemNativeWhat_pVCpuFixed,
1033	/** The fixed pCtx (PCPUMCTX) register.
1034	* @todo consider offsetting this on amd64 to use negative offsets to access
1035	* more members using 8-byte disp. */
1036	kIemNativeWhat_pCtxFixed,
1037	/** Fixed temporary register. */
1038	kIemNativeWhat_FixedTmp,
1039	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1040	/** Shadow RIP for the delayed RIP updating debugging. */
1041	kIemNativeWhat_PcShadow,
1042	#endif
1043	/** Register reserved by the CPU or OS architecture. */
1044	kIemNativeWhat_FixedReserved,
1045	/** End of valid values. */
1046	kIemNativeWhat_End
1047	} IEMNATIVEWHAT;
1048
1049	/**
1050	* Host general register entry.
1051	*
1052	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1053	*
1054	* @todo Track immediate values in host registers similarlly to how we track the
1055	* guest register shadow copies. For it to be real helpful, though,
1056	* we probably need to know which will be reused and put them into
1057	* non-volatile registers, otherwise it's going to be more or less
1058	* restricted to an instruction or two.
1059	*/
1060	typedef struct IEMNATIVEHSTREG
1061	{
1062	/** Set of guest registers this one shadows.
1063	*
1064	* Using a bitmap here so we can designate the same host register as a copy
1065	* for more than one guest register. This is expected to be useful in
1066	* situations where one value is copied to several registers in a sequence.
1067	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1068	* sequence we'd want to let this register follow to be a copy of and there
1069	* will always be places where we'd be picking the wrong one.
1070	*/
1071	uint64_t fGstRegShadows;
1072	/** What is being kept in this register. */
1073	IEMNATIVEWHAT enmWhat;
1074	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1075	uint8_t idxVar;
1076	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1077	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1078	* that scope. */
1079	uint8_t idxStackSlot;
1080	/** Alignment padding. */
1081	uint8_t abAlign[5];
1082	} IEMNATIVEHSTREG;
1083
1084
1085	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1086	/**
1087	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1088	* halves, on architectures where there is no 256-bit register available this entry will track
1089	* two adjacent 128-bit host registers.
1090	*
1091	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1092	*/
1093	typedef struct IEMNATIVEHSTSIMDREG
1094	{
1095	/** Set of guest registers this one shadows.
1096	*
1097	* Using a bitmap here so we can designate the same host register as a copy
1098	* for more than one guest register. This is expected to be useful in
1099	* situations where one value is copied to several registers in a sequence.
1100	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1101	* sequence we'd want to let this register follow to be a copy of and there
1102	* will always be places where we'd be picking the wrong one.
1103	*/
1104	uint64_t fGstRegShadows;
1105	/** What is being kept in this register. */
1106	IEMNATIVEWHAT enmWhat;
1107	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1108	uint8_t idxVar;
1109	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1110	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1111	/** Alignment padding. */
1112	uint8_t abAlign[5];
1113	} IEMNATIVEHSTSIMDREG;
1114	#endif
1115
1116
1117	/**
1118	* Core state for the native recompiler, that is, things that needs careful
1119	* handling when dealing with branches.
1120	*/
1121	typedef struct IEMNATIVECORESTATE
1122	{
1123	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1124	/** The current instruction offset in bytes from when the guest program counter
1125	* was updated last. Used for delaying the write to the guest context program counter
1126	* as long as possible. */
1127	uint32_t offPc;
1128	/** Number of instructions where we could skip the updating. */
1129	uint32_t cInstrPcUpdateSkipped;
1130	#endif
1131	/** Allocation bitmap for aHstRegs. */
1132	uint32_t bmHstRegs;
1133
1134	/** Bitmap marking which host register contains guest register shadow copies.
1135	* This is used during register allocation to try preserve copies. */
1136	uint32_t bmHstRegsWithGstShadow;
1137	/** Bitmap marking valid entries in aidxGstRegShadows. */
1138	uint64_t bmGstRegShadows;
1139
1140	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1141	/** Allocation bitmap for aHstSimdRegs. */
1142	uint32_t bmHstSimdRegs;
1143
1144	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1145	* This is used during register allocation to try preserve copies. */
1146	uint32_t bmHstSimdRegsWithGstShadow;
1147	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1148	uint64_t bmGstSimdRegShadows;
1149	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1150	uint64_t bmGstSimdRegShadowDirtyLo128;
1151	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1152	uint64_t bmGstSimdRegShadowDirtyHi128;
1153	#endif
1154
1155	union
1156	{
1157	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1158	uint8_t aidxArgVars[8];
1159	/** For more efficient resetting. */
1160	uint64_t u64ArgVars;
1161	};
1162
1163	/** Allocation bitmap for the stack. */
1164	uint32_t bmStack;
1165	/** Allocation bitmap for aVars. */
1166	uint32_t bmVars;
1167
1168	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1169	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1170	* (A shadow copy of a guest register can only be held in a one host register,
1171	* there are no duplicate copies or ambiguities like that). */
1172	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1174	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1175	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1176	* (A shadow copy of a guest register can only be held in a one host register,
1177	* there are no duplicate copies or ambiguities like that). */
1178	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1179	#endif
1180
1181	/** Host register allocation tracking. */
1182	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1183	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1184	/** Host SIMD register allocation tracking. */
1185	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1186	#endif
1187
1188	/** Variables and arguments. */
1189	IEMNATIVEVAR aVars[9];
1190	} IEMNATIVECORESTATE;
1191	/** Pointer to core state. */
1192	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1193	/** Pointer to const core state. */
1194	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1195
1196	/** @def IEMNATIVE_VAR_IDX_UNPACK
1197	* @returns Index into IEMNATIVECORESTATE::aVars.
1198	* @param a_idxVar Variable index w/ magic (in strict builds).
1199	*/
1200	/** @def IEMNATIVE_VAR_IDX_PACK
1201	* @returns Variable index w/ magic (in strict builds).
1202	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1203	*/
1204	#ifdef VBOX_STRICT
1205	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1206	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1207	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1208	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1209	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1210	#else
1211	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1212	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1213	#endif
1214
1215
1216	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1217	/** Clear the dirty state of the given guest SIMD register. */
1218	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1219	do { \
1220	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1221	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1222	} while (0)
1223
1224	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1225	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1226	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1227	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1228	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1229	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1230	/** Returns whether the given guest SIMD register is dirty. */
1231	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1232	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1233
1234	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1235	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1236	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1237	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1238	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1239	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1240
1241	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1242	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1243	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1244	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1245	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1246	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1247	#endif
1248
1249
1250	/**
1251	* Conditional stack entry.
1252	*/
1253	typedef struct IEMNATIVECOND
1254	{
1255	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1256	bool fInElse;
1257	/** The label for the IEM_MC_ELSE. */
1258	uint32_t idxLabelElse;
1259	/** The label for the IEM_MC_ENDIF. */
1260	uint32_t idxLabelEndIf;
1261	/** The initial state snapshot as the if-block starts executing. */
1262	IEMNATIVECORESTATE InitialState;
1263	/** The state snapshot at the end of the if-block. */
1264	IEMNATIVECORESTATE IfFinalState;
1265	} IEMNATIVECOND;
1266	/** Pointer to a condition stack entry. */
1267	typedef IEMNATIVECOND *PIEMNATIVECOND;
1268
1269
1270	/**
1271	* Native recompiler state.
1272	*/
1273	typedef struct IEMRECOMPILERSTATE
1274	{
1275	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1276	* IEMNATIVEINSTR units. */
1277	uint32_t cInstrBufAlloc;
1278	#ifdef VBOX_STRICT
1279	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1280	uint32_t offInstrBufChecked;
1281	#else
1282	uint32_t uPadding1; /* We don't keep track of the size here... */
1283	#endif
1284	/** Fixed temporary code buffer for native recompilation. */
1285	PIEMNATIVEINSTR pInstrBuf;
1286
1287	/** Bitmaps with the label types used. */
1288	uint64_t bmLabelTypes;
1289	/** Actual number of labels in paLabels. */
1290	uint32_t cLabels;
1291	/** Max number of entries allowed in paLabels before reallocating it. */
1292	uint32_t cLabelsAlloc;
1293	/** Labels defined while recompiling (referenced by fixups). */
1294	PIEMNATIVELABEL paLabels;
1295	/** Array with indexes of unique labels (uData always 0). */
1296	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1297
1298	/** Actual number of fixups paFixups. */
1299	uint32_t cFixups;
1300	/** Max number of entries allowed in paFixups before reallocating it. */
1301	uint32_t cFixupsAlloc;
1302	/** Buffer used by the recompiler for recording fixups when generating code. */
1303	PIEMNATIVEFIXUP paFixups;
1304
1305	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1306	/** Number of debug info entries allocated for pDbgInfo. */
1307	uint32_t cDbgInfoAlloc;
1308	uint32_t uPadding;
1309	/** Debug info. */
1310	PIEMTBDBG pDbgInfo;
1311	#endif
1312
1313	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1314	/** The current call index (liveness array and threaded calls in TB). */
1315	uint32_t idxCurCall;
1316	/** Number of liveness entries allocated. */
1317	uint32_t cLivenessEntriesAlloc;
1318	/** Liveness entries for all the calls in the TB begin recompiled.
1319	* The entry for idxCurCall contains the info for what the next call will
1320	* require wrt registers. (Which means the last entry is the initial liveness
1321	* state.) */
1322	PIEMLIVENESSENTRY paLivenessEntries;
1323	#endif
1324
1325	/** The translation block being recompiled. */
1326	PCIEMTB pTbOrg;
1327	/** The VMCPU structure of the EMT. */
1328	PVMCPUCC pVCpu;
1329
1330	/** Condition sequence number (for generating unique labels). */
1331	uint16_t uCondSeqNo;
1332	/** Check IRQ seqeunce number (for generating unique labels). */
1333	uint16_t uCheckIrqSeqNo;
1334	/** TLB load sequence number (for generating unique labels). */
1335	uint16_t uTlbSeqNo;
1336	/** The current condition stack depth (aCondStack). */
1337	uint8_t cCondDepth;
1338
1339	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1340	uint8_t cArgs;
1341	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1342	uint32_t fCImpl;
1343	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1344	uint32_t fMc;
1345	/** The expected IEMCPU::fExec value for the current call/instruction. */
1346	uint32_t fExec;
1347	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1348	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1349	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1350	*
1351	* This is an optimization because these control registers can only be changed from
1352	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1353	* consisting of multiple SIMD instructions.
1354	*/
1355	uint32_t fSimdRaiseXcptChecksEmitted;
1356	#endif
1357
1358	/** Core state requiring care with branches. */
1359	IEMNATIVECORESTATE Core;
1360
1361	/** The condition nesting stack. */
1362	IEMNATIVECOND aCondStack[2];
1363
1364	#ifndef IEM_WITH_THROW_CATCH
1365	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1366	* for recompilation error handling. */
1367	jmp_buf JmpBuf;
1368	#endif
1369	} IEMRECOMPILERSTATE;
1370	/** Pointer to a native recompiler state. */
1371	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1372
1373
1374	/** @def IEMNATIVE_TRY_SETJMP
1375	* Wrapper around setjmp / try, hiding all the ugly differences.
1376	*
1377	* @note Use with extreme care as this is a fragile macro.
1378	* @param a_pReNative The native recompile state.
1379	* @param a_rcTarget The variable that should receive the status code in case
1380	* of a longjmp/throw.
1381	*/
1382	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1383	* Start wrapper for catch / setjmp-else.
1384	*
1385	* This will set up a scope.
1386	*
1387	* @note Use with extreme care as this is a fragile macro.
1388	* @param a_pReNative The native recompile state.
1389	* @param a_rcTarget The variable that should receive the status code in case
1390	* of a longjmp/throw.
1391	*/
1392	/** @def IEMNATIVE_CATCH_LONGJMP_END
1393	* End wrapper for catch / setjmp-else.
1394	*
1395	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1396	* up the state.
1397	*
1398	* @note Use with extreme care as this is a fragile macro.
1399	* @param a_pReNative The native recompile state.
1400	*/
1401	/** @def IEMNATIVE_DO_LONGJMP
1402	*
1403	* Wrapper around longjmp / throw.
1404	*
1405	* @param a_pReNative The native recompile state.
1406	* @param a_rc The status code jump back with / throw.
1407	*/
1408	#ifdef IEM_WITH_THROW_CATCH
1409	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1410	a_rcTarget = VINF_SUCCESS; \
1411	try
1412	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1413	catch (int rcThrown) \
1414	{ \
1415	a_rcTarget = rcThrown
1416	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1417	} \
1418	((void)0)
1419	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1420	#else /* !IEM_WITH_THROW_CATCH */
1421	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1422	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1423	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1424	else \
1425	{ \
1426	((void)0)
1427	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1428	}
1429	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1430	#endif /* !IEM_WITH_THROW_CATCH */
1431
1432
1433	/**
1434	* Native recompiler worker for a threaded function.
1435	*
1436	* @returns New code buffer offset; throws VBox status code in case of a failure.
1437	* @param pReNative The native recompiler state.
1438	* @param off The current code buffer offset.
1439	* @param pCallEntry The threaded call entry.
1440	*
1441	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1442	*/
1443	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1444	/** Pointer to a native recompiler worker for a threaded function. */
1445	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1446
1447	/** Defines a native recompiler worker for a threaded function.
1448	* @see FNIEMNATIVERECOMPFUNC */
1449	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1450	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1451
1452	/** Prototypes a native recompiler function for a threaded function.
1453	* @see FNIEMNATIVERECOMPFUNC */
1454	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1455
1456
1457	/**
1458	* Native recompiler liveness analysis worker for a threaded function.
1459	*
1460	* @param pCallEntry The threaded call entry.
1461	* @param pIncoming The incoming liveness state entry.
1462	* @param pOutgoing The outgoing liveness state entry.
1463	*/
1464	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1465	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1466	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1467	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1468
1469	/** Defines a native recompiler liveness analysis worker for a threaded function.
1470	* @see FNIEMNATIVELIVENESSFUNC */
1471	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1472	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1473
1474	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1475	* @see FNIEMNATIVELIVENESSFUNC */
1476	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1477
1478
1479	/** Define a native recompiler helper function, safe to call from the TB code. */
1480	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1481	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1482	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1483	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1484	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1485	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1486	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1487	a_RetType (VBOXCALL *a_Name) a_ArgList
1488
1489
1490	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1491	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1492	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1493	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1494	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1495	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1496	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1497	uint8_t idxHstSimdReg = UINT8_MAX,
1498	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1499	# endif
1500	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1501	uint32_t offPc, uint32_t cInstrSkipped);
1502	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1503
1504	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1505	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1506	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1507	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1508	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1509	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1510
1511	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1512	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1513	bool fPreferVolatile = true);
1514	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1515	bool fPreferVolatile = true);
1516	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1517	IEMNATIVEGSTREG enmGstReg,
1518	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1519	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1520	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1521	IEMNATIVEGSTREG enmGstReg);
1522
1523	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1524	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1525	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1526	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1527	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1528	#endif
1529	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1530	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1531	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1532	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1533	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1534	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1535	#endif
1536	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1537	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1538	uint32_t fKeepVars = 0);
1539	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1540	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1541	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1542	uint32_t fHstRegsActiveShadows);
1543	#ifdef VBOX_STRICT
1544	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1545	#endif
1546	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1547	uint64_t fGstShwExcept, bool fFlushShadows);
1548	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1549	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1550	#endif
1551
1552
1553	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1554	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1555	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1556	bool fPreferVolatile = true);
1557	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1558	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1559	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1560	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1561	bool fNoVolatileRegs = false);
1562	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1563	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1564	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1565	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1566	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1567	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1568	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1569	#endif
1570
1571	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1572	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1573	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1574	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1575	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1576	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1577	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1578	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1579	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1580	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1581	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1582	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1583	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1584	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1585	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1586	#endif
1587	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1588	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1589	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1590	uint32_t fHstRegsNotToSave);
1591	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1592	uint32_t fHstRegsNotToSave);
1593	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1594	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1595
1596	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1597	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1598	#ifdef VBOX_STRICT
1599	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1600	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1601	IEMNATIVEGSTREG enmGstReg);
1602	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1603	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1604	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1605	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1606	# endif
1607	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1608	#endif
1609	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1610	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1611	#endif
1612	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1613	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1614	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1615	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1616	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1617	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1618	PCIEMTHRDEDCALLENTRY pCallEntry);
1619	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1620	uint8_t idxAddrReg, uint8_t idxInstr);
1621	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1622	uint8_t idxAddrReg, uint8_t idxInstr);
1623	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1624	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1625
1626
1627	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1628	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1629	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1630	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1631	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1632	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1633	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1634	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1635	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1636	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1637
1638	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1639	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1640	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1641	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1642	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1643	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1644	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1645	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1646	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1647	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1648	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1649	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1650	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1651	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1652	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1653	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1654	#endif
1655	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1656	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1657	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1658	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1659	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1660	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1661	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1662	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1663	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1664	#endif
1665	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1666	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1667	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1668	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1669	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1670	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1671	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1672
1673	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1674	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1675	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1676	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1677	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1678	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1679	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1680	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1681	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1682	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1683	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1684	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1685	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1686	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1687	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1688	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1689	#endif
1690	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1691	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1692	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1693	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1694	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1695	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1696	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1697	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1698	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1699	#endif
1700	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1701	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1702	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1703	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1704	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1705	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1706	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1707
1708	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1709	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1710	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1711	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1712	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1713	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1714	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1715	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1716	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1717	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1718	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1719	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1720	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1721	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1722	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1723	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1724	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1725	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1726	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1727	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1728	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1729	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1730
1731	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1732	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1733	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1734	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1735	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1736	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1737	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1738	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1739	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1740	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1741	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1742	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1743	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1744	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1745	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1746	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1747	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1748	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1749	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1750	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1751	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1752	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1753
1754	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1755	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1756	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1757	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1758
1759
1760	/**
1761	* Info about shadowed guest register values.
1762	* @see IEMNATIVEGSTREG
1763	*/
1764	typedef struct IEMANTIVEGSTREGINFO
1765	{
1766	/** Offset in VMCPU. */
1767	uint32_t off;
1768	/** The field size. */
1769	uint8_t cb;
1770	/** Name (for logging). */
1771	const char *pszName;
1772	} IEMANTIVEGSTREGINFO;
1773	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1774	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1775	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1776	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1777	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1778
1779
1780
1781	/**
1782	* Ensures that there is sufficient space in the instruction output buffer.
1783	*
1784	* This will reallocate the buffer if needed and allowed.
1785	*
1786	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1787	* allocation size.
1788	*
1789	* @returns Pointer to the instruction output buffer on success; throws VBox
1790	* status code on failure, so no need to check it.
1791	* @param pReNative The native recompile state.
1792	* @param off Current instruction offset. Works safely for UINT32_MAX
1793	* as well.
1794	* @param cInstrReq Number of instruction about to be added. It's okay to
1795	* overestimate this a bit.
1796	*/
1797	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1798	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1799	{
1800	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1801	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1802	{
1803	#ifdef VBOX_STRICT
1804	pReNative->offInstrBufChecked = offChecked;
1805	#endif
1806	return pReNative->pInstrBuf;
1807	}
1808	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1809	}
1810
1811	/**
1812	* Checks that we didn't exceed the space requested in the last
1813	* iemNativeInstrBufEnsure() call.
1814	*/
1815	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1816	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1817	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1818
1819	/**
1820	* Checks that a variable index is valid.
1821	*/
1822	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1823	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1824	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1825	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1826	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1827	("%s=%#x\n", #a_idxVar, a_idxVar))
1828	#else
1829	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1830	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1831	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1832	#endif
1833
1834	/**
1835	* Checks that a variable index is valid and that the variable is assigned the
1836	* correct argument number.
1837	* This also adds a RT_NOREF of a_idxVar.
1838	*/
1839	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1840	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1841	RT_NOREF_PV(a_idxVar); \
1842	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1843	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1844	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1845	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1846	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1847	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1848	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1849	a_uArgNo)); \
1850	} while (0)
1851	#else
1852	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1853	RT_NOREF_PV(a_idxVar); \
1854	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1855	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1856	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1857	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1858	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1859	} while (0)
1860	#endif
1861
1862
1863	/**
1864	* Checks that a variable has the expected size.
1865	*/
1866	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1867	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1868	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1869	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1870
1871
1872	/**
1873	* Calculates the stack address of a variable as a [r]BP displacement value.
1874	*/
1875	DECL_FORCE_INLINE(int32_t)
1876	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1877	{
1878	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1879	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1880	}
1881
1882
1883	/**
1884	* Releases the variable's register.
1885	*
1886	* The register must have been previously acquired calling
1887	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1888	* iemNativeVarRegisterSetAndAcquire().
1889	*/
1890	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1891	{
1892	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1893	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1894	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1895	}
1896
1897
1898	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1899	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1900	{
1901	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
1902	iemNativeVarRegisterRelease(pReNative, idxVar);
1903	}
1904	#endif
1905
1906
1907	/**
1908	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1909	*
1910	* @returns The flush mask.
1911	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1912	* @param fGstShwFlush The starting flush mask.
1913	*/
1914	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1915	{
1916	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1917	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1918	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1919	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1920	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1921	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1922	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1923	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1924	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1925	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1926	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1927	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1928	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1929	return fGstShwFlush;
1930	}
1931
1932
1933	/** Number of hidden arguments for CIMPL calls.
1934	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1935	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1936	# define IEM_CIMPL_HIDDEN_ARGS 3
1937	#else
1938	# define IEM_CIMPL_HIDDEN_ARGS 2
1939	#endif
1940
1941
1942	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1943	/** Number of hidden arguments for SSE_AIMPL calls. */
1944	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1945	#endif
1946
1947
1948	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1949
1950	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1951	/**
1952	* Helper for iemNativeLivenessGetStateByGstReg.
1953	*
1954	* @returns IEMLIVENESS_STATE_XXX
1955	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1956	* ORed together.
1957	*/
1958	DECL_FORCE_INLINE(uint32_t)
1959	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1960	{
1961	/* INPUT trumps anything else. */
1962	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1963	return IEMLIVENESS_STATE_INPUT;
1964
1965	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1966	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1967	{
1968	/* If not all sub-fields are clobbered they must be considered INPUT. */
1969	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1970	return IEMLIVENESS_STATE_INPUT;
1971	return IEMLIVENESS_STATE_CLOBBERED;
1972	}
1973
1974	/* XCPT_OR_CALL trumps UNUSED. */
1975	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1976	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1977
1978	return IEMLIVENESS_STATE_UNUSED;
1979	}
1980	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1981
1982
1983	DECL_FORCE_INLINE(uint32_t)
1984	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1985	{
1986	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1987	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1988	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1989	# else
1990	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1991	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1992	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1993	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1994	# endif
1995	}
1996
1997
1998	DECL_FORCE_INLINE(uint32_t)
1999	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2000	{
2001	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2002	if (enmGstReg == kIemNativeGstReg_EFlags)
2003	{
2004	/* Merge the eflags states to one. */
2005	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2006	uRet = RT_BIT_32(uRet);
2007	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2008	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2009	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2010	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2011	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2012	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2013	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2014	# else
2015	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2016	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2017	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2018	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2019	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2020	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2021	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2022	# endif
2023	}
2024	return uRet;
2025	}
2026
2027
2028	# ifdef VBOX_STRICT
2029	/** For assertions only, user checks that idxCurCall isn't zerow. */
2030	DECL_FORCE_INLINE(uint32_t)
2031	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2032	{
2033	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2034	}
2035	# endif /* VBOX_STRICT */
2036
2037	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2038
2039
2040	/**
2041	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2042	*/
2043	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2044	{
2045	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2046	return IEM_CIMPL_HIDDEN_ARGS;
2047	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
2048	return 1;
2049	return 0;
2050	}
2051
2052
2053	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2054	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2055	{
2056	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2057
2058	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2059	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2060	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2061	return (uint8_t)idxReg;
2062	}
2063
2064
2065
2066	/*********************************************************************************************************************************
2067	* Register Allocator (GPR) *
2068	*********************************************************************************************************************************/
2069
2070	/**
2071	* Marks host register @a idxHstReg as containing a shadow copy of guest
2072	* register @a enmGstReg.
2073	*
2074	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2075	* host register before calling.
2076	*/
2077	DECL_FORCE_INLINE(void)
2078	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2079	{
2080	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2081	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2082	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2083
2084	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2085	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2086	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2087	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2088	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2089	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2090	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2091	#else
2092	RT_NOREF(off);
2093	#endif
2094	}
2095
2096
2097	/**
2098	* Clear any guest register shadow claims from @a idxHstReg.
2099	*
2100	* The register does not need to be shadowing any guest registers.
2101	*/
2102	DECL_FORCE_INLINE(void)
2103	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2104	{
2105	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2106	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2107	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2108	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2109	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2110
2111	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2112	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2113	if (fGstRegs)
2114	{
2115	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2116	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2117	while (fGstRegs)
2118	{
2119	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2120	fGstRegs &= ~RT_BIT_64(iGstReg);
2121	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2122	}
2123	}
2124	#else
2125	RT_NOREF(off);
2126	#endif
2127
2128	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2129	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2130	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2131	}
2132
2133
2134	/**
2135	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2136	* and global overview flags.
2137	*/
2138	DECL_FORCE_INLINE(void)
2139	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2140	{
2141	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2142	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2143	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2144	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2145	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2146	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2147	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2148
2149	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2150	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2151	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2152	#else
2153	RT_NOREF(off);
2154	#endif
2155
2156	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2157	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2158	if (!fGstRegShadowsNew)
2159	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2160	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2161	}
2162
2163
2164	#if 0 /* unused */
2165	/**
2166	* Clear any guest register shadow claim for @a enmGstReg.
2167	*/
2168	DECL_FORCE_INLINE(void)
2169	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2170	{
2171	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2172	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2173	{
2174	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2175	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2176	}
2177	}
2178	#endif
2179
2180
2181	/**
2182	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2183	* as the new shadow of it.
2184	*
2185	* Unlike the other guest reg shadow helpers, this does the logging for you.
2186	* However, it is the liveness state is not asserted here, the caller must do
2187	* that.
2188	*/
2189	DECL_FORCE_INLINE(void)
2190	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2191	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2192	{
2193	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2194	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2195	{
2196	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2197	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2198	if (idxHstRegOld == idxHstRegNew)
2199	return;
2200	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2201	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2202	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2203	}
2204	else
2205	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2206	g_aGstShadowInfo[enmGstReg].pszName));
2207	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2208	}
2209
2210
2211	/**
2212	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2213	* to @a idxRegTo.
2214	*/
2215	DECL_FORCE_INLINE(void)
2216	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2217	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2218	{
2219	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2220	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2221	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2222	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2223	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2224	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2225	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2226	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2227	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2228
2229	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2230	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2231	if (!fGstRegShadowsFrom)
2232	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2233	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2234	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2235	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2236	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2237	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2238	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2239	#else
2240	RT_NOREF(off);
2241	#endif
2242	}
2243
2244
2245	/**
2246	* Flushes any delayed guest register writes.
2247	*
2248	* This must be called prior to calling CImpl functions and any helpers that use
2249	* the guest state (like raising exceptions) and such.
2250	*
2251	* This optimization has not yet been implemented. The first target would be
2252	* RIP updates, since these are the most common ones.
2253	*/
2254	DECL_INLINE_THROW(uint32_t)
2255	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2256	{
2257	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2258	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2259	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2260	#else
2261	RT_NOREF(pReNative, fGstShwExcept);
2262	#endif
2263
2264	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2265	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2266	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2267	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2268	#else
2269	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2270	return off;
2271	#endif
2272	}
2273
2274
2275
2276	/*********************************************************************************************************************************
2277	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2278	*********************************************************************************************************************************/
2279
2280	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2281
2282	DECL_FORCE_INLINE(uint8_t)
2283	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2284	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2285	{
2286	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2287
2288	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2289	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2290	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2291	return idxSimdReg;
2292	}
2293
2294
2295	/**
2296	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2297	* SIMD register @a enmGstSimdReg.
2298	*
2299	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2300	* host register before calling.
2301	*/
2302	DECL_FORCE_INLINE(void)
2303	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2304	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2305	{
2306	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2307	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2308	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2309
2310	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2311	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2312	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2313	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2314	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2315	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2316	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2317	#else
2318	RT_NOREF(off);
2319	#endif
2320	}
2321
2322
2323	/**
2324	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2325	* to @a idxSimdRegTo.
2326	*/
2327	DECL_FORCE_INLINE(void)
2328	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2329	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2330	{
2331	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2332	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2333	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2334	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2335	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2336	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2337	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2338	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2339	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2340	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2341	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2342
2343	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2344	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2345	if (!fGstRegShadowsFrom)
2346	{
2347	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2348	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2349	}
2350	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2351	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2352	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2353	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2354	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2355	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2356	#else
2357	RT_NOREF(off);
2358	#endif
2359	}
2360
2361
2362	/**
2363	* Clear any guest register shadow claims from @a idxHstSimdReg.
2364	*
2365	* The register does not need to be shadowing any guest registers.
2366	*/
2367	DECL_FORCE_INLINE(void)
2368	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2369	{
2370	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2371	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2372	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2373	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2374	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2375	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2376	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2377
2378	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2379	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2380	if (fGstRegs)
2381	{
2382	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2383	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2384	while (fGstRegs)
2385	{
2386	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2387	fGstRegs &= ~RT_BIT_64(iGstReg);
2388	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2389	}
2390	}
2391	#else
2392	RT_NOREF(off);
2393	#endif
2394
2395	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2396	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2397	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2398	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2399	}
2400
2401	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2402
2403
2404	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2405	/**
2406	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2407	*/
2408	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2409	{
2410	if (pReNative->Core.offPc)
2411	return iemNativeEmitPcWritebackSlow(pReNative, off);
2412	return off;
2413	}
2414	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2415
2416
2417	/** @} */
2418
2419	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2420

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103993

Download in other formats: