IEMN8veRecompiler.h@ 103936

Last change on this file since 103936 was 103934, checked in by vboxsync, 13 months ago
VMM/IEM: Implement native emitter for IEM_MC_FETCH_MEM_U128()/IEM_MC_FETCH_MEM_FLAT_U128(), bugref:10614
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 117.9 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103934 2024-03-20 08:21:56Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
227	\| RT_BIT_32(ARMV8_A64_REG_Q30))
228	# endif
229	# endif
230
231	#else
232	# error "port me"
233	#endif
234	/** @} */
235
236	/** @name Call related registers.
237	* @{ */
238	/** @def IEMNATIVE_CALL_RET_GREG
239	* The return value register. */
240	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
241	* Number of arguments in registers. */
242	/** @def IEMNATIVE_CALL_ARG0_GREG
243	* The general purpose register carrying argument \#0. */
244	/** @def IEMNATIVE_CALL_ARG1_GREG
245	* The general purpose register carrying argument \#1. */
246	/** @def IEMNATIVE_CALL_ARG2_GREG
247	* The general purpose register carrying argument \#2. */
248	/** @def IEMNATIVE_CALL_ARG3_GREG
249	* The general purpose register carrying argument \#3. */
250	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
251	* Mask of registers the callee will not save and may trash. */
252	#ifdef RT_ARCH_AMD64
253	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
254
255	# ifdef RT_OS_WINDOWS
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
257	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
258	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
259	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
260	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
261	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
262	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
263	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
264	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
265	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
266	\| RT_BIT_32(X86_GREG_xCX) \
267	\| RT_BIT_32(X86_GREG_xDX) \
268	\| RT_BIT_32(X86_GREG_x8) \
269	\| RT_BIT_32(X86_GREG_x9) \
270	\| RT_BIT_32(X86_GREG_x10) \
271	\| RT_BIT_32(X86_GREG_x11) )
272	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
273	/* xmm0 - xmm5 are marked as volatile. */
274	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
275	# endif
276
277	# else
278	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
279	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
280	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
281	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
282	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
283	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
289	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
290	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
291	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
292	\| RT_BIT_32(X86_GREG_xCX) \
293	\| RT_BIT_32(X86_GREG_xDX) \
294	\| RT_BIT_32(X86_GREG_xDI) \
295	\| RT_BIT_32(X86_GREG_xSI) \
296	\| RT_BIT_32(X86_GREG_x8) \
297	\| RT_BIT_32(X86_GREG_x9) \
298	\| RT_BIT_32(X86_GREG_x10) \
299	\| RT_BIT_32(X86_GREG_x11) )
300	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
301	/* xmm0 - xmm15 are marked as volatile. */
302	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
303	# endif
304	# endif
305
306	#elif defined(RT_ARCH_ARM64)
307	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
308	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
309	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
310	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
311	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
312	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
313	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
314	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
315	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
316	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
317	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
318	\| RT_BIT_32(ARMV8_A64_REG_X1) \
319	\| RT_BIT_32(ARMV8_A64_REG_X2) \
320	\| RT_BIT_32(ARMV8_A64_REG_X3) \
321	\| RT_BIT_32(ARMV8_A64_REG_X4) \
322	\| RT_BIT_32(ARMV8_A64_REG_X5) \
323	\| RT_BIT_32(ARMV8_A64_REG_X6) \
324	\| RT_BIT_32(ARMV8_A64_REG_X7) )
325	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
326	\| RT_BIT_32(ARMV8_A64_REG_X1) \
327	\| RT_BIT_32(ARMV8_A64_REG_X2) \
328	\| RT_BIT_32(ARMV8_A64_REG_X3) \
329	\| RT_BIT_32(ARMV8_A64_REG_X4) \
330	\| RT_BIT_32(ARMV8_A64_REG_X5) \
331	\| RT_BIT_32(ARMV8_A64_REG_X6) \
332	\| RT_BIT_32(ARMV8_A64_REG_X7) \
333	\| RT_BIT_32(ARMV8_A64_REG_X8) \
334	\| RT_BIT_32(ARMV8_A64_REG_X9) \
335	\| RT_BIT_32(ARMV8_A64_REG_X10) \
336	\| RT_BIT_32(ARMV8_A64_REG_X11) \
337	\| RT_BIT_32(ARMV8_A64_REG_X12) \
338	\| RT_BIT_32(ARMV8_A64_REG_X13) \
339	\| RT_BIT_32(ARMV8_A64_REG_X14) \
340	\| RT_BIT_32(ARMV8_A64_REG_X15) \
341	\| RT_BIT_32(ARMV8_A64_REG_X16) \
342	\| RT_BIT_32(ARMV8_A64_REG_X17) )
343	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
344	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
345	* so to simplify our life a bit we just mark everything as volatile. */
346	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
347	# endif
348
349	#endif
350
351	/** This is the maximum argument count we'll ever be needing. */
352	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
353	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
354	#else
355	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
356	#endif
357	/** @} */
358
359
360	/** @def IEMNATIVE_HST_GREG_COUNT
361	* Number of host general purpose registers we tracker. */
362	/** @def IEMNATIVE_HST_GREG_MASK
363	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
364	* inverted register masks and such to get down to a correct set of regs. */
365	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
366	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
367	* Number of host SIMD registers we track. */
368	/** @def IEMNATIVE_HST_SIMD_REG_MASK
369	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
370	* inverted register masks and such to get down to a correct set of regs. */
371	#endif
372	#ifdef RT_ARCH_AMD64
373	# define IEMNATIVE_HST_GREG_COUNT 16
374	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
375
376	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
377	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
378	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
379	# endif
380
381	#elif defined(RT_ARCH_ARM64)
382	# define IEMNATIVE_HST_GREG_COUNT 32
383	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
384
385	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
386	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
387	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
388	# endif
389
390	#else
391	# error "Port me!"
392	#endif
393
394
395	/** Native code generator label types. */
396	typedef enum
397	{
398	kIemNativeLabelType_Invalid = 0,
399	/*
400	* Labels w/o data, only once instance per TB.
401	*
402	* Note! Jumps to these requires instructions that are capable of spanning
403	* the max TB length.
404	*/
405	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
406	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
407	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
408	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
409	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
410	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
411	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
412	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
413	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
414	kIemNativeLabelType_ObsoleteTb,
415	kIemNativeLabelType_NeedCsLimChecking,
416	kIemNativeLabelType_CheckBranchMiss,
417	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
418	/* Manually defined labels. */
419	kIemNativeLabelType_Return,
420	kIemNativeLabelType_ReturnBreak,
421	kIemNativeLabelType_ReturnWithFlags,
422	kIemNativeLabelType_NonZeroRetOrPassUp,
423	/** The last fixup for branches that can span almost the whole TB length. */
424	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
425
426	/*
427	* Labels with data, potentially multiple instances per TB:
428	*
429	* These are localized labels, so no fixed jump type restrictions here.
430	*/
431	kIemNativeLabelType_FirstWithMultipleInstances,
432	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
433	kIemNativeLabelType_Else,
434	kIemNativeLabelType_Endif,
435	kIemNativeLabelType_CheckIrq,
436	kIemNativeLabelType_TlbLookup,
437	kIemNativeLabelType_TlbMiss,
438	kIemNativeLabelType_TlbDone,
439	kIemNativeLabelType_End
440	} IEMNATIVELABELTYPE;
441
442	/** Native code generator label definition. */
443	typedef struct IEMNATIVELABEL
444	{
445	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
446	* the epilog. */
447	uint32_t off;
448	/** The type of label (IEMNATIVELABELTYPE). */
449	uint16_t enmType;
450	/** Additional label data, type specific. */
451	uint16_t uData;
452	} IEMNATIVELABEL;
453	/** Pointer to a label. */
454	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
455
456
457	/** Native code generator fixup types. */
458	typedef enum
459	{
460	kIemNativeFixupType_Invalid = 0,
461	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
462	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
463	kIemNativeFixupType_Rel32,
464	#elif defined(RT_ARCH_ARM64)
465	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
466	kIemNativeFixupType_RelImm26At0,
467	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
468	kIemNativeFixupType_RelImm19At5,
469	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
470	kIemNativeFixupType_RelImm14At5,
471	#endif
472	kIemNativeFixupType_End
473	} IEMNATIVEFIXUPTYPE;
474
475	/** Native code generator fixup. */
476	typedef struct IEMNATIVEFIXUP
477	{
478	/** Code offset of the fixup location. */
479	uint32_t off;
480	/** The IEMNATIVELABEL this is a fixup for. */
481	uint16_t idxLabel;
482	/** The fixup type (IEMNATIVEFIXUPTYPE). */
483	uint8_t enmType;
484	/** Addend or other data. */
485	int8_t offAddend;
486	} IEMNATIVEFIXUP;
487	/** Pointer to a native code generator fixup. */
488	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
489
490
491	/**
492	* One bit of the state.
493	*
494	* Each register state takes up two bits. We keep the two bits in two separate
495	* 64-bit words to simplify applying them to the guest shadow register mask in
496	* the register allocator.
497	*/
498	typedef union IEMLIVENESSBIT
499	{
500	uint64_t bm64;
501	RT_GCC_EXTENSION struct
502	{ /* bit no */
503	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
504	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
505	uint64_t fCr0 : 1; /*< 0x11 / 17: /
506	uint64_t fFcw : 1; /*< 0x12 / 18: /
507	uint64_t fFsw : 1; /*< 0x13 / 19: /
508	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
509	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
510	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
511	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
512	uint64_t fCr4 : 1; /*< 0x2c / 44: /
513	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
514	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
515	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
516	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
517	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
518	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
519	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
520	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
521	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
522	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
523	};
524	} IEMLIVENESSBIT;
525	AssertCompileSize(IEMLIVENESSBIT, 8);
526
527	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
528	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
529	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
530	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
531	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
532	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
533	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
534
535
536	/**
537	* A liveness state entry.
538	*
539	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
540	* Once we add a SSE register shadowing, we'll add another 64-bit element for
541	* that.
542	*/
543	typedef union IEMLIVENESSENTRY
544	{
545	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
546	uint64_t bm64[16 / 8];
547	uint16_t bm32[16 / 4];
548	uint16_t bm16[16 / 2];
549	uint8_t bm8[ 16 / 1];
550	IEMLIVENESSBIT aBits[2];
551	#else
552	uint64_t bm64[32 / 8];
553	uint16_t bm32[32 / 4];
554	uint16_t bm16[32 / 2];
555	uint8_t bm8[ 32 / 1];
556	IEMLIVENESSBIT aBits[4];
557	#endif
558	RT_GCC_EXTENSION struct
559	{
560	/** Bit \#0 of the register states. */
561	IEMLIVENESSBIT Bit0;
562	/** Bit \#1 of the register states. */
563	IEMLIVENESSBIT Bit1;
564	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
565	/** Bit \#2 of the register states. */
566	IEMLIVENESSBIT Bit2;
567	/** Bit \#3 of the register states. */
568	IEMLIVENESSBIT Bit3;
569	#endif
570	};
571	} IEMLIVENESSENTRY;
572	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
573	AssertCompileSize(IEMLIVENESSENTRY, 16);
574	#else
575	AssertCompileSize(IEMLIVENESSENTRY, 32);
576	#endif
577	/** Pointer to a liveness state entry. */
578	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
579	/** Pointer to a const liveness state entry. */
580	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
581
582	/** @name 64-bit value masks for IEMLIVENESSENTRY.
583	* @{ / / 0xzzzzyyyyxxxxwwww */
584	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
585
586	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
587	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
588	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
589
590	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
591	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
592	#endif
593
594	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
595	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
596
597	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
598	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
599	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
600	#endif
601	/** @} */
602
603
604	/** @name The liveness state for a register.
605	*
606	* The state values have been picked to with state accumulation in mind (what
607	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
608	* performance critical work done with the values.
609	*
610	* This is a compressed state that only requires 2 bits per register.
611	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
612	* 1. the incoming state from the following call,
613	* 2. the outgoing state for this call,
614	* 3. mask of the entries set in the 2nd.
615	*
616	* The mask entry (3rd one above) will be used both when updating the outgoing
617	* state and when merging in incoming state for registers not touched by the
618	* current call.
619	*
620	* @{ */
621	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
622	/** The register will be clobbered and the current value thrown away.
623	*
624	* When this is applied to the state (2) we'll simply be AND'ing it with the
625	* (old) mask (3) and adding the register to the mask. This way we'll
626	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
627	* IEMLIVENESS_STATE_INPUT states. */
628	# define IEMLIVENESS_STATE_CLOBBERED 0
629	/** The register is unused in the remainder of the TB.
630	*
631	* This is an initial state and can not be set by any of the
632	* iemNativeLivenessFunc_xxxx callbacks. */
633	# define IEMLIVENESS_STATE_UNUSED 1
634	/** The register value is required in a potential call or exception.
635	*
636	* This means that the register value must be calculated and is best written to
637	* the state, but that any shadowing registers can be flushed thereafter as it's
638	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
639	*
640	* It is typically applied across the board, but we preserve incoming
641	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
642	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
643	* 1. r0 = old & ~mask;
644	* 2. r0 = t1 & (t1 >> 1)'
645	* 3. state \|= r0 \| 0b10;
646	* 4. mask = ~0;
647	*/
648	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
649	/** The register value is used as input.
650	*
651	* This means that the register value must be calculated and it is best to keep
652	* it in a register. It does not need to be writtent out as such. This is the
653	* highest priority state.
654	*
655	* Whether the call modifies the register or not isn't relevant to earlier
656	* calls, so that's not recorded.
657	*
658	* When applying this state we just or in the value in the outgoing state and
659	* mask. */
660	# define IEMLIVENESS_STATE_INPUT 3
661	/** Mask of the state bits. */
662	# define IEMLIVENESS_STATE_MASK 3
663	/** The number of bits per state. */
664	# define IEMLIVENESS_STATE_BIT_COUNT 2
665	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
666	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
667	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
668	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
669	/** Check if a register clobbering is expected given the (previous) liveness state.
670	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
671	* include INPUT if the register is used in more than one place. */
672	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
673
674	/** Check if all status flags are going to be clobbered and doesn't need
675	* calculating in the current step.
676	* @param a_pCurEntry The current liveness entry. */
677	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
678	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
679
680	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
681	/** The register is not used any more. */
682	# define IEMLIVENESS_STATE_UNUSED 0
683	/** Flag: The register is required in a potential exception or call. */
684	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
685	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
686	/** Flag: The register is read. */
687	# define IEMLIVENESS_STATE_READ 2
688	# define IEMLIVENESS_BIT_READ 1
689	/** Flag: The register is written. */
690	# define IEMLIVENESS_STATE_WRITE 4
691	# define IEMLIVENESS_BIT_WRITE 2
692	/** Flag: Unconditional call (not needed, can be redefined for research). */
693	# define IEMLIVENESS_STATE_CALL 8
694	# define IEMLIVENESS_BIT_CALL 3
695	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
696	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
697	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
698	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
699	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
700
701	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
702	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
703	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
704	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
705
706	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
707	/** @} */
708
709	/** @name Liveness helpers for builtin functions and similar.
710	*
711	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
712	* own set of manimulator macros for those.
713	*
714	* @{ */
715	/** Initializing the state as all unused. */
716	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
717	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
718	do { \
719	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
720	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
721	} while (0)
722	#else
723	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
724	do { \
725	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
726	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
727	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
728	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
729	} while (0)
730	#endif
731
732	/** Initializing the outgoing state with a potential xcpt or call state.
733	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
734	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
735	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
736	do { \
737	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
738	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
739	} while (0)
740	#else
741	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
742	do { \
743	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
744	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
745	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
746	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
747	} while (0)
748	#endif
749
750	/** Adds a segment base register as input to the outgoing state. */
751	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
752	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
753	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
754	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
755	} while (0)
756	#else
757	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
758	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
759	} while (0)
760	#endif
761
762	/** Adds a segment attribute register as input to the outgoing state. */
763	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
764	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
765	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
766	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
767	} while (0)
768	#else
769	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
770	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
771	} while (0)
772	#endif
773
774	/** Adds a segment limit register as input to the outgoing state. */
775	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
776	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
777	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
778	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
779	} while (0)
780	#else
781	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
782	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
783	} while (0)
784	#endif
785
786	/** Adds a segment limit register as input to the outgoing state. */
787	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
788	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
789	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
790	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
791	} while (0)
792	#else
793	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
794	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
795	} while (0)
796	#endif
797	/** @} */
798
799	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
800	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
801	* calculated and up to date. This is to double check that we haven't skipped
802	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
803	* @note has to be placed in
804	*/
805	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
806	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
807	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
808	#else
809	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
810	#endif
811
812
813	/**
814	* Guest registers that can be shadowed in GPRs.
815	*
816	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
817	* must be placed last, as the liveness state tracks it as 7 subcomponents and
818	* we don't want to waste space here.
819	*
820	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
821	* friends as well as IEMAllN8veLiveness.cpp.
822	*/
823	typedef enum IEMNATIVEGSTREG : uint8_t
824	{
825	kIemNativeGstReg_GprFirst = 0,
826	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
827	kIemNativeGstReg_Pc,
828	kIemNativeGstReg_Cr0,
829	kIemNativeGstReg_FpuFcw,
830	kIemNativeGstReg_FpuFsw,
831	kIemNativeGstReg_SegBaseFirst,
832	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
833	kIemNativeGstReg_SegAttribFirst,
834	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
835	kIemNativeGstReg_SegLimitFirst,
836	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
837	kIemNativeGstReg_SegSelFirst,
838	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
839	kIemNativeGstReg_Cr4,
840	kIemNativeGstReg_Xcr0,
841	kIemNativeGstReg_MxCsr,
842	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
843	kIemNativeGstReg_End
844	} IEMNATIVEGSTREG;
845	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
846	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
847
848	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
849	* @{ */
850	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
851	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
852	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
853	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
854	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
855	/** @} */
856
857	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
858
859	/**
860	* Guest registers that can be shadowed in host SIMD registers.
861	*
862	* @todo r=aeichner Liveness tracking
863	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
864	*/
865	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
866	{
867	kIemNativeGstSimdReg_SimdRegFirst = 0,
868	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
869	kIemNativeGstSimdReg_End
870	} IEMNATIVEGSTSIMDREG;
871
872	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
873	* @{ */
874	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
875	/** @} */
876
877	/**
878	* The Load/store size for a SIMD guest register.
879	*/
880	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
881	{
882	/** Invalid size. */
883	kIemNativeGstSimdRegLdStSz_Invalid = 0,
884	/** Loads the low 128-bit of a guest SIMD register. */
885	kIemNativeGstSimdRegLdStSz_Low128,
886	/** Loads the high 128-bit of a guest SIMD register. */
887	kIemNativeGstSimdRegLdStSz_High128,
888	/** Loads the whole 256-bits of a guest SIMD register. */
889	kIemNativeGstSimdRegLdStSz_256,
890	/** End value. */
891	kIemNativeGstSimdRegLdStSz_End
892	} IEMNATIVEGSTSIMDREGLDSTSZ;
893
894	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
895
896	/**
897	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
898	*/
899	typedef enum IEMNATIVEGSTREGUSE
900	{
901	/** The usage is read-only, the register holding the guest register
902	* shadow copy will not be modified by the caller. */
903	kIemNativeGstRegUse_ReadOnly = 0,
904	/** The caller will update the guest register (think: PC += cbInstr).
905	* The guest shadow copy will follow the returned register. */
906	kIemNativeGstRegUse_ForUpdate,
907	/** The call will put an entirely new value in the guest register, so
908	* if new register is allocate it will be returned uninitialized. */
909	kIemNativeGstRegUse_ForFullWrite,
910	/** The caller will use the guest register value as input in a calculation
911	* and the host register will be modified.
912	* This means that the returned host register will not be marked as a shadow
913	* copy of the guest register. */
914	kIemNativeGstRegUse_Calculation
915	} IEMNATIVEGSTREGUSE;
916
917	/**
918	* Guest registers (classes) that can be referenced.
919	*/
920	typedef enum IEMNATIVEGSTREGREF : uint8_t
921	{
922	kIemNativeGstRegRef_Invalid = 0,
923	kIemNativeGstRegRef_Gpr,
924	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
925	kIemNativeGstRegRef_EFlags,
926	kIemNativeGstRegRef_MxCsr,
927	kIemNativeGstRegRef_FpuReg,
928	kIemNativeGstRegRef_MReg,
929	kIemNativeGstRegRef_XReg,
930	kIemNativeGstRegRef_X87,
931	kIemNativeGstRegRef_XState,
932	//kIemNativeGstRegRef_YReg, - doesn't work.
933	kIemNativeGstRegRef_End
934	} IEMNATIVEGSTREGREF;
935
936
937	/** Variable kinds. */
938	typedef enum IEMNATIVEVARKIND : uint8_t
939	{
940	/** Customary invalid zero value. */
941	kIemNativeVarKind_Invalid = 0,
942	/** This is either in a register or on the stack. */
943	kIemNativeVarKind_Stack,
944	/** Immediate value - loaded into register when needed, or can live on the
945	* stack if referenced (in theory). */
946	kIemNativeVarKind_Immediate,
947	/** Variable reference - loaded into register when needed, never stack. */
948	kIemNativeVarKind_VarRef,
949	/** Guest register reference - loaded into register when needed, never stack. */
950	kIemNativeVarKind_GstRegRef,
951	/** End of valid values. */
952	kIemNativeVarKind_End
953	} IEMNATIVEVARKIND;
954
955
956	/** Variable or argument. */
957	typedef struct IEMNATIVEVAR
958	{
959	/** The kind of variable. */
960	IEMNATIVEVARKIND enmKind;
961	/** The variable size in bytes. */
962	uint8_t cbVar;
963	/** The first stack slot (uint64_t), except for immediate and references
964	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
965	* has a stack slot it has been initialized and has a value. Unused variables
966	* has neither a stack slot nor a host register assignment. */
967	uint8_t idxStackSlot;
968	/** The host register allocated for the variable, UINT8_MAX if not. */
969	uint8_t idxReg;
970	/** The argument number if argument, UINT8_MAX if regular variable. */
971	uint8_t uArgNo;
972	/** If referenced, the index (unpacked) of the variable referencing this one,
973	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
974	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
975	uint8_t idxReferrerVar;
976	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
977	* @todo not sure what this really is for... */
978	IEMNATIVEGSTREG enmGstReg;
979	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
980	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
981	* only valid when idxReg is not UINT8_MAX. */
982	bool fSimdReg : 1;
983	/** Set if the registered is currently used exclusively, false if the
984	* variable is idle and the register can be grabbed. */
985	bool fRegAcquired : 1;
986	#else
987	/** Set if the registered is currently used exclusively, false if the
988	* variable is idle and the register can be grabbed. */
989	bool fRegAcquired;
990	#endif
991
992	union
993	{
994	/** kIemNativeVarKind_Immediate: The immediate value. */
995	uint64_t uValue;
996	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
997	uint8_t idxRefVar;
998	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
999	struct
1000	{
1001	/** The class of register. */
1002	IEMNATIVEGSTREGREF enmClass;
1003	/** Index within the class. */
1004	uint8_t idx;
1005	} GstRegRef;
1006	} u;
1007	} IEMNATIVEVAR;
1008	/** Pointer to a variable or argument. */
1009	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1010	/** Pointer to a const variable or argument. */
1011	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1012
1013	/** What is being kept in a host register. */
1014	typedef enum IEMNATIVEWHAT : uint8_t
1015	{
1016	/** The traditional invalid zero value. */
1017	kIemNativeWhat_Invalid = 0,
1018	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1019	kIemNativeWhat_Var,
1020	/** Temporary register, this is typically freed when a MC completes. */
1021	kIemNativeWhat_Tmp,
1022	/** Call argument w/o a variable mapping. This is free (via
1023	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1024	kIemNativeWhat_Arg,
1025	/** Return status code.
1026	* @todo not sure if we need this... */
1027	kIemNativeWhat_rc,
1028	/** The fixed pVCpu (PVMCPUCC) register.
1029	* @todo consider offsetting this on amd64 to use negative offsets to access
1030	* more members using 8-byte disp. */
1031	kIemNativeWhat_pVCpuFixed,
1032	/** The fixed pCtx (PCPUMCTX) register.
1033	* @todo consider offsetting this on amd64 to use negative offsets to access
1034	* more members using 8-byte disp. */
1035	kIemNativeWhat_pCtxFixed,
1036	/** Fixed temporary register. */
1037	kIemNativeWhat_FixedTmp,
1038	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1039	/** Shadow RIP for the delayed RIP updating debugging. */
1040	kIemNativeWhat_PcShadow,
1041	#endif
1042	/** Register reserved by the CPU or OS architecture. */
1043	kIemNativeWhat_FixedReserved,
1044	/** End of valid values. */
1045	kIemNativeWhat_End
1046	} IEMNATIVEWHAT;
1047
1048	/**
1049	* Host general register entry.
1050	*
1051	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1052	*
1053	* @todo Track immediate values in host registers similarlly to how we track the
1054	* guest register shadow copies. For it to be real helpful, though,
1055	* we probably need to know which will be reused and put them into
1056	* non-volatile registers, otherwise it's going to be more or less
1057	* restricted to an instruction or two.
1058	*/
1059	typedef struct IEMNATIVEHSTREG
1060	{
1061	/** Set of guest registers this one shadows.
1062	*
1063	* Using a bitmap here so we can designate the same host register as a copy
1064	* for more than one guest register. This is expected to be useful in
1065	* situations where one value is copied to several registers in a sequence.
1066	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1067	* sequence we'd want to let this register follow to be a copy of and there
1068	* will always be places where we'd be picking the wrong one.
1069	*/
1070	uint64_t fGstRegShadows;
1071	/** What is being kept in this register. */
1072	IEMNATIVEWHAT enmWhat;
1073	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1074	uint8_t idxVar;
1075	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1076	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1077	* that scope. */
1078	uint8_t idxStackSlot;
1079	/** Alignment padding. */
1080	uint8_t abAlign[5];
1081	} IEMNATIVEHSTREG;
1082
1083
1084	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1085	/**
1086	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1087	* halves, on architectures where there is no 256-bit register available this entry will track
1088	* two adjacent 128-bit host registers.
1089	*
1090	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1091	*/
1092	typedef struct IEMNATIVEHSTSIMDREG
1093	{
1094	/** Set of guest registers this one shadows.
1095	*
1096	* Using a bitmap here so we can designate the same host register as a copy
1097	* for more than one guest register. This is expected to be useful in
1098	* situations where one value is copied to several registers in a sequence.
1099	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1100	* sequence we'd want to let this register follow to be a copy of and there
1101	* will always be places where we'd be picking the wrong one.
1102	*/
1103	uint64_t fGstRegShadows;
1104	/** What is being kept in this register. */
1105	IEMNATIVEWHAT enmWhat;
1106	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1107	uint8_t idxVar;
1108	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1109	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1110	/** Alignment padding. */
1111	uint8_t abAlign[5];
1112	} IEMNATIVEHSTSIMDREG;
1113	#endif
1114
1115
1116	/**
1117	* Core state for the native recompiler, that is, things that needs careful
1118	* handling when dealing with branches.
1119	*/
1120	typedef struct IEMNATIVECORESTATE
1121	{
1122	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1123	/** The current instruction offset in bytes from when the guest program counter
1124	* was updated last. Used for delaying the write to the guest context program counter
1125	* as long as possible. */
1126	uint32_t offPc;
1127	/** Number of instructions where we could skip the updating. */
1128	uint32_t cInstrPcUpdateSkipped;
1129	#endif
1130	/** Allocation bitmap for aHstRegs. */
1131	uint32_t bmHstRegs;
1132
1133	/** Bitmap marking which host register contains guest register shadow copies.
1134	* This is used during register allocation to try preserve copies. */
1135	uint32_t bmHstRegsWithGstShadow;
1136	/** Bitmap marking valid entries in aidxGstRegShadows. */
1137	uint64_t bmGstRegShadows;
1138
1139	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1140	/** Allocation bitmap for aHstSimdRegs. */
1141	uint32_t bmHstSimdRegs;
1142
1143	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1144	* This is used during register allocation to try preserve copies. */
1145	uint32_t bmHstSimdRegsWithGstShadow;
1146	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1147	uint64_t bmGstSimdRegShadows;
1148	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1149	uint64_t bmGstSimdRegShadowDirtyLo128;
1150	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1151	uint64_t bmGstSimdRegShadowDirtyHi128;
1152	#endif
1153
1154	union
1155	{
1156	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1157	uint8_t aidxArgVars[8];
1158	/** For more efficient resetting. */
1159	uint64_t u64ArgVars;
1160	};
1161
1162	/** Allocation bitmap for the stack. */
1163	uint32_t bmStack;
1164	/** Allocation bitmap for aVars. */
1165	uint32_t bmVars;
1166
1167	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1168	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1169	* (A shadow copy of a guest register can only be held in a one host register,
1170	* there are no duplicate copies or ambiguities like that). */
1171	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1172	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1173	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1174	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1175	* (A shadow copy of a guest register can only be held in a one host register,
1176	* there are no duplicate copies or ambiguities like that). */
1177	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1178	#endif
1179
1180	/** Host register allocation tracking. */
1181	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1182	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1183	/** Host SIMD register allocation tracking. */
1184	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1185	#endif
1186
1187	/** Variables and arguments. */
1188	IEMNATIVEVAR aVars[9];
1189	} IEMNATIVECORESTATE;
1190	/** Pointer to core state. */
1191	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1192	/** Pointer to const core state. */
1193	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1194
1195	/** @def IEMNATIVE_VAR_IDX_UNPACK
1196	* @returns Index into IEMNATIVECORESTATE::aVars.
1197	* @param a_idxVar Variable index w/ magic (in strict builds).
1198	*/
1199	/** @def IEMNATIVE_VAR_IDX_PACK
1200	* @returns Variable index w/ magic (in strict builds).
1201	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1202	*/
1203	#ifdef VBOX_STRICT
1204	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1205	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1206	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1207	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1208	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1209	#else
1210	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1211	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1212	#endif
1213
1214
1215	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1216	/** Clear the dirty state of the given guest SIMD register. */
1217	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1218	do { \
1219	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1220	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1221	} while (0)
1222
1223	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1224	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1225	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1226	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1227	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1228	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1229	/** Returns whether the given guest SIMD register is dirty. */
1230	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1231	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1232
1233	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1234	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1235	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1236	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1237	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1238	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1239
1240	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1241	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1242	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1243	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1244	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1245	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1246	#endif
1247
1248
1249	/**
1250	* Conditional stack entry.
1251	*/
1252	typedef struct IEMNATIVECOND
1253	{
1254	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1255	bool fInElse;
1256	/** The label for the IEM_MC_ELSE. */
1257	uint32_t idxLabelElse;
1258	/** The label for the IEM_MC_ENDIF. */
1259	uint32_t idxLabelEndIf;
1260	/** The initial state snapshot as the if-block starts executing. */
1261	IEMNATIVECORESTATE InitialState;
1262	/** The state snapshot at the end of the if-block. */
1263	IEMNATIVECORESTATE IfFinalState;
1264	} IEMNATIVECOND;
1265	/** Pointer to a condition stack entry. */
1266	typedef IEMNATIVECOND *PIEMNATIVECOND;
1267
1268
1269	/**
1270	* Native recompiler state.
1271	*/
1272	typedef struct IEMRECOMPILERSTATE
1273	{
1274	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1275	* IEMNATIVEINSTR units. */
1276	uint32_t cInstrBufAlloc;
1277	#ifdef VBOX_STRICT
1278	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1279	uint32_t offInstrBufChecked;
1280	#else
1281	uint32_t uPadding1; /* We don't keep track of the size here... */
1282	#endif
1283	/** Fixed temporary code buffer for native recompilation. */
1284	PIEMNATIVEINSTR pInstrBuf;
1285
1286	/** Bitmaps with the label types used. */
1287	uint64_t bmLabelTypes;
1288	/** Actual number of labels in paLabels. */
1289	uint32_t cLabels;
1290	/** Max number of entries allowed in paLabels before reallocating it. */
1291	uint32_t cLabelsAlloc;
1292	/** Labels defined while recompiling (referenced by fixups). */
1293	PIEMNATIVELABEL paLabels;
1294	/** Array with indexes of unique labels (uData always 0). */
1295	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1296
1297	/** Actual number of fixups paFixups. */
1298	uint32_t cFixups;
1299	/** Max number of entries allowed in paFixups before reallocating it. */
1300	uint32_t cFixupsAlloc;
1301	/** Buffer used by the recompiler for recording fixups when generating code. */
1302	PIEMNATIVEFIXUP paFixups;
1303
1304	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1305	/** Number of debug info entries allocated for pDbgInfo. */
1306	uint32_t cDbgInfoAlloc;
1307	uint32_t uPadding;
1308	/** Debug info. */
1309	PIEMTBDBG pDbgInfo;
1310	#endif
1311
1312	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1313	/** The current call index (liveness array and threaded calls in TB). */
1314	uint32_t idxCurCall;
1315	/** Number of liveness entries allocated. */
1316	uint32_t cLivenessEntriesAlloc;
1317	/** Liveness entries for all the calls in the TB begin recompiled.
1318	* The entry for idxCurCall contains the info for what the next call will
1319	* require wrt registers. (Which means the last entry is the initial liveness
1320	* state.) */
1321	PIEMLIVENESSENTRY paLivenessEntries;
1322	#endif
1323
1324	/** The translation block being recompiled. */
1325	PCIEMTB pTbOrg;
1326	/** The VMCPU structure of the EMT. */
1327	PVMCPUCC pVCpu;
1328
1329	/** Condition sequence number (for generating unique labels). */
1330	uint16_t uCondSeqNo;
1331	/** Check IRQ seqeunce number (for generating unique labels). */
1332	uint16_t uCheckIrqSeqNo;
1333	/** TLB load sequence number (for generating unique labels). */
1334	uint16_t uTlbSeqNo;
1335	/** The current condition stack depth (aCondStack). */
1336	uint8_t cCondDepth;
1337
1338	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1339	uint8_t cArgs;
1340	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1341	uint32_t fCImpl;
1342	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1343	uint32_t fMc;
1344	/** The expected IEMCPU::fExec value for the current call/instruction. */
1345	uint32_t fExec;
1346	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1347	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1348	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1349	*
1350	* This is an optimization because these control registers can only be changed from
1351	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1352	* consisting of multiple SIMD instructions.
1353	*/
1354	uint32_t fSimdRaiseXcptChecksEmitted;
1355	#endif
1356
1357	/** Core state requiring care with branches. */
1358	IEMNATIVECORESTATE Core;
1359
1360	/** The condition nesting stack. */
1361	IEMNATIVECOND aCondStack[2];
1362
1363	#ifndef IEM_WITH_THROW_CATCH
1364	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1365	* for recompilation error handling. */
1366	jmp_buf JmpBuf;
1367	#endif
1368	} IEMRECOMPILERSTATE;
1369	/** Pointer to a native recompiler state. */
1370	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1371
1372
1373	/** @def IEMNATIVE_TRY_SETJMP
1374	* Wrapper around setjmp / try, hiding all the ugly differences.
1375	*
1376	* @note Use with extreme care as this is a fragile macro.
1377	* @param a_pReNative The native recompile state.
1378	* @param a_rcTarget The variable that should receive the status code in case
1379	* of a longjmp/throw.
1380	*/
1381	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1382	* Start wrapper for catch / setjmp-else.
1383	*
1384	* This will set up a scope.
1385	*
1386	* @note Use with extreme care as this is a fragile macro.
1387	* @param a_pReNative The native recompile state.
1388	* @param a_rcTarget The variable that should receive the status code in case
1389	* of a longjmp/throw.
1390	*/
1391	/** @def IEMNATIVE_CATCH_LONGJMP_END
1392	* End wrapper for catch / setjmp-else.
1393	*
1394	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1395	* up the state.
1396	*
1397	* @note Use with extreme care as this is a fragile macro.
1398	* @param a_pReNative The native recompile state.
1399	*/
1400	/** @def IEMNATIVE_DO_LONGJMP
1401	*
1402	* Wrapper around longjmp / throw.
1403	*
1404	* @param a_pReNative The native recompile state.
1405	* @param a_rc The status code jump back with / throw.
1406	*/
1407	#ifdef IEM_WITH_THROW_CATCH
1408	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1409	a_rcTarget = VINF_SUCCESS; \
1410	try
1411	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1412	catch (int rcThrown) \
1413	{ \
1414	a_rcTarget = rcThrown
1415	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1416	} \
1417	((void)0)
1418	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1419	#else /* !IEM_WITH_THROW_CATCH */
1420	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1421	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1422	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1423	else \
1424	{ \
1425	((void)0)
1426	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1427	}
1428	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1429	#endif /* !IEM_WITH_THROW_CATCH */
1430
1431
1432	/**
1433	* Native recompiler worker for a threaded function.
1434	*
1435	* @returns New code buffer offset; throws VBox status code in case of a failure.
1436	* @param pReNative The native recompiler state.
1437	* @param off The current code buffer offset.
1438	* @param pCallEntry The threaded call entry.
1439	*
1440	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1441	*/
1442	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1443	/** Pointer to a native recompiler worker for a threaded function. */
1444	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1445
1446	/** Defines a native recompiler worker for a threaded function.
1447	* @see FNIEMNATIVERECOMPFUNC */
1448	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1449	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1450
1451	/** Prototypes a native recompiler function for a threaded function.
1452	* @see FNIEMNATIVERECOMPFUNC */
1453	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1454
1455
1456	/**
1457	* Native recompiler liveness analysis worker for a threaded function.
1458	*
1459	* @param pCallEntry The threaded call entry.
1460	* @param pIncoming The incoming liveness state entry.
1461	* @param pOutgoing The outgoing liveness state entry.
1462	*/
1463	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1464	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1465	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1466	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1467
1468	/** Defines a native recompiler liveness analysis worker for a threaded function.
1469	* @see FNIEMNATIVELIVENESSFUNC */
1470	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1471	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1472
1473	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1474	* @see FNIEMNATIVELIVENESSFUNC */
1475	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1476
1477
1478	/** Define a native recompiler helper function, safe to call from the TB code. */
1479	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1480	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1481	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1482	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1483	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1484	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1485	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1486	a_RetType (VBOXCALL *a_Name) a_ArgList
1487
1488
1489	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1490	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1491	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1492	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1493	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1494	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1495	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1496	uint8_t idxHstSimdReg = UINT8_MAX,
1497	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1498	# endif
1499	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1500	uint32_t offPc, uint32_t cInstrSkipped);
1501	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1502
1503	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1504	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1505	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1506	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1507	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1508	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1509
1510	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1511	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1512	bool fPreferVolatile = true);
1513	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1514	bool fPreferVolatile = true);
1515	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1516	IEMNATIVEGSTREG enmGstReg,
1517	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1518	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1519	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1520	IEMNATIVEGSTREG enmGstReg);
1521
1522	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1523	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1524	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1525	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1526	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1527	#endif
1528	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1529	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1530	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1531	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1532	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1533	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1534	#endif
1535	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1536	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1537	uint32_t fKeepVars = 0);
1538	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1539	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1540	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1541	uint32_t fHstRegsActiveShadows);
1542	#ifdef VBOX_STRICT
1543	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1544	#endif
1545	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1546	uint64_t fGstShwExcept, bool fFlushShadows);
1547	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1548	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1549	#endif
1550
1551
1552	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1553	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1554	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1555	bool fPreferVolatile = true);
1556	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1557	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1558	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1559	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1560	bool fNoVolatileRegs = false);
1561	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1562	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1563	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1564	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1565	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1566	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1567	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1568	#endif
1569
1570	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1571	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1572	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1573	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1574	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1575	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1576	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1577	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1578	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1579	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1580	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1581	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1582	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1583	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1584	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1585	#endif
1586	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1587	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1588	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1589	uint32_t fHstRegsNotToSave);
1590	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1591	uint32_t fHstRegsNotToSave);
1592	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1593	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1594
1595	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1596	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1597	#ifdef VBOX_STRICT
1598	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1599	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1600	IEMNATIVEGSTREG enmGstReg);
1601	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1602	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1603	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1604	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1605	# endif
1606	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1607	#endif
1608	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1609	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1610	#endif
1611	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1612	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1613	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1614	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1615	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1616	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1617	PCIEMTHRDEDCALLENTRY pCallEntry);
1618	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1619	uint8_t idxAddrReg, uint8_t idxInstr);
1620	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1621	uint8_t idxAddrReg, uint8_t idxInstr);
1622	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1623	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1624
1625
1626	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1627	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1628	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1629	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1630	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1631	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1632	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1633	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1634	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1635	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1636
1637	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1638	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1639	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1640	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1641	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1642	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1643	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1644	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1645	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1646	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1647	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1648	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1649	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1650	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1651	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1652	#endif
1653	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1654	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1655	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1656	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1657	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1658	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1659	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1660	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1661	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1662	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1663	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1664
1665	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1666	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1667	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1668	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1669	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1670	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1671	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1672	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1673	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1674	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1675	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1676	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1677	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1678	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1679	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1680	#endif
1681	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1682	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1683	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1684	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1685	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1686	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1687	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1688	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1689	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1690	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1691	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1692
1693	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1694	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1695	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1696	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1697	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1698	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1699	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1700	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1701	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1702	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1703	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1704	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1705	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1706	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1707	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1708	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1709	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1710	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1711	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1712	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1713	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1714	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1715
1716	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1717	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1718	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1719	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1720	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1721	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1722	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1723	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1724	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1725	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1726	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1727	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1728	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1729	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1730	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1731	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1732	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1733	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1734	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1735	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1736	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1737	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1738
1739	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1740	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1741	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1742	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1743
1744
1745	/**
1746	* Info about shadowed guest register values.
1747	* @see IEMNATIVEGSTREG
1748	*/
1749	typedef struct IEMANTIVEGSTREGINFO
1750	{
1751	/** Offset in VMCPU. */
1752	uint32_t off;
1753	/** The field size. */
1754	uint8_t cb;
1755	/** Name (for logging). */
1756	const char *pszName;
1757	} IEMANTIVEGSTREGINFO;
1758	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1759	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1760	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1761	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1762	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1763
1764
1765
1766	/**
1767	* Ensures that there is sufficient space in the instruction output buffer.
1768	*
1769	* This will reallocate the buffer if needed and allowed.
1770	*
1771	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1772	* allocation size.
1773	*
1774	* @returns Pointer to the instruction output buffer on success; throws VBox
1775	* status code on failure, so no need to check it.
1776	* @param pReNative The native recompile state.
1777	* @param off Current instruction offset. Works safely for UINT32_MAX
1778	* as well.
1779	* @param cInstrReq Number of instruction about to be added. It's okay to
1780	* overestimate this a bit.
1781	*/
1782	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1783	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1784	{
1785	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1786	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1787	{
1788	#ifdef VBOX_STRICT
1789	pReNative->offInstrBufChecked = offChecked;
1790	#endif
1791	return pReNative->pInstrBuf;
1792	}
1793	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1794	}
1795
1796	/**
1797	* Checks that we didn't exceed the space requested in the last
1798	* iemNativeInstrBufEnsure() call.
1799	*/
1800	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1801	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1802	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1803
1804	/**
1805	* Checks that a variable index is valid.
1806	*/
1807	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1808	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1809	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1810	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1811	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1812	("%s=%#x\n", #a_idxVar, a_idxVar))
1813	#else
1814	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1815	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1816	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1817	#endif
1818
1819	/**
1820	* Checks that a variable index is valid and that the variable is assigned the
1821	* correct argument number.
1822	* This also adds a RT_NOREF of a_idxVar.
1823	*/
1824	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1825	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1826	RT_NOREF_PV(a_idxVar); \
1827	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1828	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1829	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1830	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1831	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1832	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1833	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1834	a_uArgNo)); \
1835	} while (0)
1836	#else
1837	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1838	RT_NOREF_PV(a_idxVar); \
1839	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1840	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1841	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1842	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1843	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1844	} while (0)
1845	#endif
1846
1847
1848	/**
1849	* Checks that a variable has the expected size.
1850	*/
1851	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1852	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1853	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1854	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1855
1856
1857	/**
1858	* Calculates the stack address of a variable as a [r]BP displacement value.
1859	*/
1860	DECL_FORCE_INLINE(int32_t)
1861	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1862	{
1863	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1864	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1865	}
1866
1867
1868	/**
1869	* Releases the variable's register.
1870	*
1871	* The register must have been previously acquired calling
1872	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1873	* iemNativeVarRegisterSetAndAcquire().
1874	*/
1875	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1876	{
1877	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1878	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1879	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1880	}
1881
1882
1883	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1884	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1885	{
1886	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
1887	iemNativeVarRegisterRelease(pReNative, idxVar);
1888	}
1889	#endif
1890
1891
1892	/**
1893	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1894	*
1895	* @returns The flush mask.
1896	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1897	* @param fGstShwFlush The starting flush mask.
1898	*/
1899	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1900	{
1901	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1902	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1903	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1904	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1905	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1906	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1907	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1908	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1909	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1910	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1911	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1912	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1913	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1914	return fGstShwFlush;
1915	}
1916
1917
1918	/** Number of hidden arguments for CIMPL calls.
1919	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1920	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1921	# define IEM_CIMPL_HIDDEN_ARGS 3
1922	#else
1923	# define IEM_CIMPL_HIDDEN_ARGS 2
1924	#endif
1925
1926
1927	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1928	/** Number of hidden arguments for SSE_AIMPL calls. */
1929	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1930	#endif
1931
1932
1933	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1934
1935	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1936	/**
1937	* Helper for iemNativeLivenessGetStateByGstReg.
1938	*
1939	* @returns IEMLIVENESS_STATE_XXX
1940	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1941	* ORed together.
1942	*/
1943	DECL_FORCE_INLINE(uint32_t)
1944	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1945	{
1946	/* INPUT trumps anything else. */
1947	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1948	return IEMLIVENESS_STATE_INPUT;
1949
1950	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1951	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1952	{
1953	/* If not all sub-fields are clobbered they must be considered INPUT. */
1954	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1955	return IEMLIVENESS_STATE_INPUT;
1956	return IEMLIVENESS_STATE_CLOBBERED;
1957	}
1958
1959	/* XCPT_OR_CALL trumps UNUSED. */
1960	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1961	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1962
1963	return IEMLIVENESS_STATE_UNUSED;
1964	}
1965	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1966
1967
1968	DECL_FORCE_INLINE(uint32_t)
1969	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1970	{
1971	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1972	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1973	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1974	# else
1975	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1976	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1977	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1978	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1979	# endif
1980	}
1981
1982
1983	DECL_FORCE_INLINE(uint32_t)
1984	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
1985	{
1986	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
1987	if (enmGstReg == kIemNativeGstReg_EFlags)
1988	{
1989	/* Merge the eflags states to one. */
1990	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1991	uRet = RT_BIT_32(uRet);
1992	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
1993	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
1994	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
1995	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
1996	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
1997	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
1998	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
1999	# else
2000	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2001	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2002	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2003	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2004	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2005	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2006	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2007	# endif
2008	}
2009	return uRet;
2010	}
2011
2012
2013	# ifdef VBOX_STRICT
2014	/** For assertions only, user checks that idxCurCall isn't zerow. */
2015	DECL_FORCE_INLINE(uint32_t)
2016	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2017	{
2018	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2019	}
2020	# endif /* VBOX_STRICT */
2021
2022	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2023
2024
2025	/**
2026	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2027	*/
2028	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2029	{
2030	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2031	return IEM_CIMPL_HIDDEN_ARGS;
2032	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
2033	return 1;
2034	return 0;
2035	}
2036
2037
2038	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2039	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2040	{
2041	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2042
2043	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2044	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2045	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2046	return (uint8_t)idxReg;
2047	}
2048
2049
2050
2051	/*********************************************************************************************************************************
2052	* Register Allocator (GPR) *
2053	*********************************************************************************************************************************/
2054
2055	/**
2056	* Marks host register @a idxHstReg as containing a shadow copy of guest
2057	* register @a enmGstReg.
2058	*
2059	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2060	* host register before calling.
2061	*/
2062	DECL_FORCE_INLINE(void)
2063	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2064	{
2065	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2066	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2067	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2068
2069	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2070	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2071	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2072	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2073	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2074	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2075	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2076	#else
2077	RT_NOREF(off);
2078	#endif
2079	}
2080
2081
2082	/**
2083	* Clear any guest register shadow claims from @a idxHstReg.
2084	*
2085	* The register does not need to be shadowing any guest registers.
2086	*/
2087	DECL_FORCE_INLINE(void)
2088	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2089	{
2090	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2091	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2092	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2093	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2094	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2095
2096	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2097	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2098	if (fGstRegs)
2099	{
2100	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2101	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2102	while (fGstRegs)
2103	{
2104	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2105	fGstRegs &= ~RT_BIT_64(iGstReg);
2106	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2107	}
2108	}
2109	#else
2110	RT_NOREF(off);
2111	#endif
2112
2113	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2114	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2115	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2116	}
2117
2118
2119	/**
2120	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2121	* and global overview flags.
2122	*/
2123	DECL_FORCE_INLINE(void)
2124	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2125	{
2126	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2127	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2128	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2129	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2130	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2131	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2132	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2133
2134	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2135	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2136	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2137	#else
2138	RT_NOREF(off);
2139	#endif
2140
2141	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2142	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2143	if (!fGstRegShadowsNew)
2144	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2145	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2146	}
2147
2148
2149	#if 0 /* unused */
2150	/**
2151	* Clear any guest register shadow claim for @a enmGstReg.
2152	*/
2153	DECL_FORCE_INLINE(void)
2154	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2155	{
2156	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2157	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2158	{
2159	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2160	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2161	}
2162	}
2163	#endif
2164
2165
2166	/**
2167	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2168	* as the new shadow of it.
2169	*
2170	* Unlike the other guest reg shadow helpers, this does the logging for you.
2171	* However, it is the liveness state is not asserted here, the caller must do
2172	* that.
2173	*/
2174	DECL_FORCE_INLINE(void)
2175	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2176	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2177	{
2178	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2179	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2180	{
2181	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2182	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2183	if (idxHstRegOld == idxHstRegNew)
2184	return;
2185	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2186	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2187	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2188	}
2189	else
2190	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2191	g_aGstShadowInfo[enmGstReg].pszName));
2192	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2193	}
2194
2195
2196	/**
2197	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2198	* to @a idxRegTo.
2199	*/
2200	DECL_FORCE_INLINE(void)
2201	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2202	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2203	{
2204	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2205	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2206	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2207	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2208	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2209	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2210	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2211	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2212	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2213
2214	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2215	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2216	if (!fGstRegShadowsFrom)
2217	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2218	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2219	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2220	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2221	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2222	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2223	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2224	#else
2225	RT_NOREF(off);
2226	#endif
2227	}
2228
2229
2230	/**
2231	* Flushes any delayed guest register writes.
2232	*
2233	* This must be called prior to calling CImpl functions and any helpers that use
2234	* the guest state (like raising exceptions) and such.
2235	*
2236	* This optimization has not yet been implemented. The first target would be
2237	* RIP updates, since these are the most common ones.
2238	*/
2239	DECL_INLINE_THROW(uint32_t)
2240	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2241	{
2242	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2243	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2244	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2245	#else
2246	RT_NOREF(pReNative, fGstShwExcept);
2247	#endif
2248
2249	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2250	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2251	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2252	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2253	#else
2254	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2255	return off;
2256	#endif
2257	}
2258
2259
2260
2261	/*********************************************************************************************************************************
2262	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2263	*********************************************************************************************************************************/
2264
2265	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2266
2267	DECL_FORCE_INLINE(uint8_t)
2268	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2269	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2270	{
2271	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2272
2273	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2274	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2275	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2276	return idxSimdReg;
2277	}
2278
2279
2280	/**
2281	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2282	* SIMD register @a enmGstSimdReg.
2283	*
2284	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2285	* host register before calling.
2286	*/
2287	DECL_FORCE_INLINE(void)
2288	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2289	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2290	{
2291	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2292	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2293	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2294
2295	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2296	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2297	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2298	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2299	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2300	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2301	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2302	#else
2303	RT_NOREF(off);
2304	#endif
2305	}
2306
2307
2308	/**
2309	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2310	* to @a idxSimdRegTo.
2311	*/
2312	DECL_FORCE_INLINE(void)
2313	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2314	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2315	{
2316	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2317	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2318	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2319	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2320	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2321	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2322	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2323	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2324	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2325	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2326	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2327
2328	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2329	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2330	if (!fGstRegShadowsFrom)
2331	{
2332	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2333	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2334	}
2335	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2336	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2337	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2338	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2339	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2340	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2341	#else
2342	RT_NOREF(off);
2343	#endif
2344	}
2345
2346
2347	/**
2348	* Clear any guest register shadow claims from @a idxHstSimdReg.
2349	*
2350	* The register does not need to be shadowing any guest registers.
2351	*/
2352	DECL_FORCE_INLINE(void)
2353	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2354	{
2355	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2356	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2357	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2358	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2359	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2360	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2361	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2362
2363	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2364	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2365	if (fGstRegs)
2366	{
2367	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2368	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2369	while (fGstRegs)
2370	{
2371	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2372	fGstRegs &= ~RT_BIT_64(iGstReg);
2373	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2374	}
2375	}
2376	#else
2377	RT_NOREF(off);
2378	#endif
2379
2380	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2381	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2382	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2383	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2384	}
2385
2386	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2387
2388
2389	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2390	/**
2391	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2392	*/
2393	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2394	{
2395	if (pReNative->Core.offPc)
2396	return iemNativeEmitPcWritebackSlow(pReNative, off);
2397	return off;
2398	}
2399	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2400
2401
2402	/** @} */
2403
2404	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2405

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103936

Download in other formats: