IEMN8veRecompiler.h@ 104298

Last change on this file since 104298 was 104274, checked in by vboxsync, 8 months ago
VMM/IEM: Implement native emitter for IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() (enables wait instruction recompilation), bugref:10371 [missing files]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 124.1 KB

Line
1	/* $Id: IEMN8veRecompiler.h 104274 2024-04-10 12:55:16Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/*
226	* ARM64 has 32 128-bit registers only, in order to support emulating 256-bit registers we pair
227	* two real registers statically to one virtual for now, leaving us with only 16 256-bit registers.
228	* We always pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed
229	* and the register allocator assumes that it will be always free when the lower is picked.
230	*
231	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't touch them in order to avoid
232	* having to save and restore them in the prologue/epilogue.
233	*/
234	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
235	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
236	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
237	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
238	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
239	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
240	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q1))
252	# endif
253	# endif
254
255	#else
256	# error "port me"
257	#endif
258	/** @} */
259
260	/** @name Call related registers.
261	* @{ */
262	/** @def IEMNATIVE_CALL_RET_GREG
263	* The return value register. */
264	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
265	* Number of arguments in registers. */
266	/** @def IEMNATIVE_CALL_ARG0_GREG
267	* The general purpose register carrying argument \#0. */
268	/** @def IEMNATIVE_CALL_ARG1_GREG
269	* The general purpose register carrying argument \#1. */
270	/** @def IEMNATIVE_CALL_ARG2_GREG
271	* The general purpose register carrying argument \#2. */
272	/** @def IEMNATIVE_CALL_ARG3_GREG
273	* The general purpose register carrying argument \#3. */
274	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
275	* Mask of registers the callee will not save and may trash. */
276	#ifdef RT_ARCH_AMD64
277	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
278
279	# ifdef RT_OS_WINDOWS
280	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
281	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
282	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
283	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
289	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
290	\| RT_BIT_32(X86_GREG_xCX) \
291	\| RT_BIT_32(X86_GREG_xDX) \
292	\| RT_BIT_32(X86_GREG_x8) \
293	\| RT_BIT_32(X86_GREG_x9) \
294	\| RT_BIT_32(X86_GREG_x10) \
295	\| RT_BIT_32(X86_GREG_x11) )
296	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
297	/* xmm0 - xmm5 are marked as volatile. */
298	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
299	# endif
300
301	# else
302	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
303	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
304	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
305	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
306	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
307	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
308	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
309	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
310	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
311	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
312	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
313	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
314	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
315	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
316	\| RT_BIT_32(X86_GREG_xCX) \
317	\| RT_BIT_32(X86_GREG_xDX) \
318	\| RT_BIT_32(X86_GREG_xDI) \
319	\| RT_BIT_32(X86_GREG_xSI) \
320	\| RT_BIT_32(X86_GREG_x8) \
321	\| RT_BIT_32(X86_GREG_x9) \
322	\| RT_BIT_32(X86_GREG_x10) \
323	\| RT_BIT_32(X86_GREG_x11) )
324	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
325	/* xmm0 - xmm15 are marked as volatile. */
326	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
327	# endif
328	# endif
329
330	#elif defined(RT_ARCH_ARM64)
331	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
332	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
333	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
334	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
335	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
336	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
337	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
338	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
339	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
340	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
341	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
342	\| RT_BIT_32(ARMV8_A64_REG_X1) \
343	\| RT_BIT_32(ARMV8_A64_REG_X2) \
344	\| RT_BIT_32(ARMV8_A64_REG_X3) \
345	\| RT_BIT_32(ARMV8_A64_REG_X4) \
346	\| RT_BIT_32(ARMV8_A64_REG_X5) \
347	\| RT_BIT_32(ARMV8_A64_REG_X6) \
348	\| RT_BIT_32(ARMV8_A64_REG_X7) )
349	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
350	\| RT_BIT_32(ARMV8_A64_REG_X1) \
351	\| RT_BIT_32(ARMV8_A64_REG_X2) \
352	\| RT_BIT_32(ARMV8_A64_REG_X3) \
353	\| RT_BIT_32(ARMV8_A64_REG_X4) \
354	\| RT_BIT_32(ARMV8_A64_REG_X5) \
355	\| RT_BIT_32(ARMV8_A64_REG_X6) \
356	\| RT_BIT_32(ARMV8_A64_REG_X7) \
357	\| RT_BIT_32(ARMV8_A64_REG_X8) \
358	\| RT_BIT_32(ARMV8_A64_REG_X9) \
359	\| RT_BIT_32(ARMV8_A64_REG_X10) \
360	\| RT_BIT_32(ARMV8_A64_REG_X11) \
361	\| RT_BIT_32(ARMV8_A64_REG_X12) \
362	\| RT_BIT_32(ARMV8_A64_REG_X13) \
363	\| RT_BIT_32(ARMV8_A64_REG_X14) \
364	\| RT_BIT_32(ARMV8_A64_REG_X15) \
365	\| RT_BIT_32(ARMV8_A64_REG_X16) \
366	\| RT_BIT_32(ARMV8_A64_REG_X17) )
367	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
368	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
369	* so to simplify our life a bit we just mark everything as volatile. */
370	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
371	# endif
372
373	#endif
374
375	/** This is the maximum argument count we'll ever be needing. */
376	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
377	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
378	#else
379	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
380	#endif
381	/** @} */
382
383
384	/** @def IEMNATIVE_HST_GREG_COUNT
385	* Number of host general purpose registers we tracker. */
386	/** @def IEMNATIVE_HST_GREG_MASK
387	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
388	* inverted register masks and such to get down to a correct set of regs. */
389	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
390	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
391	* Number of host SIMD registers we track. */
392	/** @def IEMNATIVE_HST_SIMD_REG_MASK
393	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
394	* inverted register masks and such to get down to a correct set of regs. */
395	#endif
396	#ifdef RT_ARCH_AMD64
397	# define IEMNATIVE_HST_GREG_COUNT 16
398	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
399
400	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
401	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
402	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
403	# endif
404
405	#elif defined(RT_ARCH_ARM64)
406	# define IEMNATIVE_HST_GREG_COUNT 32
407	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
408
409	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
410	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
411	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
412	# endif
413
414	#else
415	# error "Port me!"
416	#endif
417
418
419	/** Native code generator label types. */
420	typedef enum
421	{
422	kIemNativeLabelType_Invalid = 0,
423	/*
424	* Labels w/o data, only once instance per TB.
425	*
426	* Note! Jumps to these requires instructions that are capable of spanning
427	* the max TB length.
428	*/
429	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
430	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
431	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
432	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
433	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
434	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
435	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
436	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
437	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
438	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
439	kIemNativeLabelType_ObsoleteTb,
440	kIemNativeLabelType_NeedCsLimChecking,
441	kIemNativeLabelType_CheckBranchMiss,
442	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
443	/* Manually defined labels. */
444	kIemNativeLabelType_Return,
445	kIemNativeLabelType_ReturnBreak,
446	kIemNativeLabelType_ReturnWithFlags,
447	kIemNativeLabelType_NonZeroRetOrPassUp,
448	/** The last fixup for branches that can span almost the whole TB length. */
449	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
450
451	/*
452	* Labels with data, potentially multiple instances per TB:
453	*
454	* These are localized labels, so no fixed jump type restrictions here.
455	*/
456	kIemNativeLabelType_FirstWithMultipleInstances,
457	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
458	kIemNativeLabelType_Else,
459	kIemNativeLabelType_Endif,
460	kIemNativeLabelType_CheckIrq,
461	kIemNativeLabelType_TlbLookup,
462	kIemNativeLabelType_TlbMiss,
463	kIemNativeLabelType_TlbDone,
464	kIemNativeLabelType_End
465	} IEMNATIVELABELTYPE;
466
467	/** Native code generator label definition. */
468	typedef struct IEMNATIVELABEL
469	{
470	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
471	* the epilog. */
472	uint32_t off;
473	/** The type of label (IEMNATIVELABELTYPE). */
474	uint16_t enmType;
475	/** Additional label data, type specific. */
476	uint16_t uData;
477	} IEMNATIVELABEL;
478	/** Pointer to a label. */
479	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
480
481
482	/** Native code generator fixup types. */
483	typedef enum
484	{
485	kIemNativeFixupType_Invalid = 0,
486	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
487	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
488	kIemNativeFixupType_Rel32,
489	#elif defined(RT_ARCH_ARM64)
490	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
491	kIemNativeFixupType_RelImm26At0,
492	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
493	kIemNativeFixupType_RelImm19At5,
494	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
495	kIemNativeFixupType_RelImm14At5,
496	#endif
497	kIemNativeFixupType_End
498	} IEMNATIVEFIXUPTYPE;
499
500	/** Native code generator fixup. */
501	typedef struct IEMNATIVEFIXUP
502	{
503	/** Code offset of the fixup location. */
504	uint32_t off;
505	/** The IEMNATIVELABEL this is a fixup for. */
506	uint16_t idxLabel;
507	/** The fixup type (IEMNATIVEFIXUPTYPE). */
508	uint8_t enmType;
509	/** Addend or other data. */
510	int8_t offAddend;
511	} IEMNATIVEFIXUP;
512	/** Pointer to a native code generator fixup. */
513	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
514
515
516	/**
517	* One bit of the state.
518	*
519	* Each register state takes up two bits. We keep the two bits in two separate
520	* 64-bit words to simplify applying them to the guest shadow register mask in
521	* the register allocator.
522	*/
523	typedef union IEMLIVENESSBIT
524	{
525	uint64_t bm64;
526	RT_GCC_EXTENSION struct
527	{ /* bit no */
528	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
529	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
530	uint64_t fCr0 : 1; /*< 0x11 / 17: /
531	uint64_t fFcw : 1; /*< 0x12 / 18: /
532	uint64_t fFsw : 1; /*< 0x13 / 19: /
533	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
534	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
535	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
536	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
537	uint64_t fCr4 : 1; /*< 0x2c / 44: /
538	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
539	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
540	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
541	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
542	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
543	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
544	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
545	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
546	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
547	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
548	};
549	} IEMLIVENESSBIT;
550	AssertCompileSize(IEMLIVENESSBIT, 8);
551
552	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
553	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
554	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
555	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
556	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
557	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
558	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
559
560
561	/**
562	* A liveness state entry.
563	*
564	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
565	* Once we add a SSE register shadowing, we'll add another 64-bit element for
566	* that.
567	*/
568	typedef union IEMLIVENESSENTRY
569	{
570	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
571	uint64_t bm64[16 / 8];
572	uint16_t bm32[16 / 4];
573	uint16_t bm16[16 / 2];
574	uint8_t bm8[ 16 / 1];
575	IEMLIVENESSBIT aBits[2];
576	#else
577	uint64_t bm64[32 / 8];
578	uint16_t bm32[32 / 4];
579	uint16_t bm16[32 / 2];
580	uint8_t bm8[ 32 / 1];
581	IEMLIVENESSBIT aBits[4];
582	#endif
583	RT_GCC_EXTENSION struct
584	{
585	/** Bit \#0 of the register states. */
586	IEMLIVENESSBIT Bit0;
587	/** Bit \#1 of the register states. */
588	IEMLIVENESSBIT Bit1;
589	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
590	/** Bit \#2 of the register states. */
591	IEMLIVENESSBIT Bit2;
592	/** Bit \#3 of the register states. */
593	IEMLIVENESSBIT Bit3;
594	#endif
595	};
596	} IEMLIVENESSENTRY;
597	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
598	AssertCompileSize(IEMLIVENESSENTRY, 16);
599	#else
600	AssertCompileSize(IEMLIVENESSENTRY, 32);
601	#endif
602	/** Pointer to a liveness state entry. */
603	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
604	/** Pointer to a const liveness state entry. */
605	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
606
607	/** @name 64-bit value masks for IEMLIVENESSENTRY.
608	* @{ / / 0xzzzzyyyyxxxxwwww */
609	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
610
611	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
612	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
613	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
614
615	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
616	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
617	#endif
618
619	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
620	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
621
622	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
623	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
624	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
625	#endif
626	/** @} */
627
628
629	/** @name The liveness state for a register.
630	*
631	* The state values have been picked to with state accumulation in mind (what
632	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
633	* performance critical work done with the values.
634	*
635	* This is a compressed state that only requires 2 bits per register.
636	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
637	* 1. the incoming state from the following call,
638	* 2. the outgoing state for this call,
639	* 3. mask of the entries set in the 2nd.
640	*
641	* The mask entry (3rd one above) will be used both when updating the outgoing
642	* state and when merging in incoming state for registers not touched by the
643	* current call.
644	*
645	* @{ */
646	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
647	/** The register will be clobbered and the current value thrown away.
648	*
649	* When this is applied to the state (2) we'll simply be AND'ing it with the
650	* (old) mask (3) and adding the register to the mask. This way we'll
651	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
652	* IEMLIVENESS_STATE_INPUT states. */
653	# define IEMLIVENESS_STATE_CLOBBERED 0
654	/** The register is unused in the remainder of the TB.
655	*
656	* This is an initial state and can not be set by any of the
657	* iemNativeLivenessFunc_xxxx callbacks. */
658	# define IEMLIVENESS_STATE_UNUSED 1
659	/** The register value is required in a potential call or exception.
660	*
661	* This means that the register value must be calculated and is best written to
662	* the state, but that any shadowing registers can be flushed thereafter as it's
663	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
664	*
665	* It is typically applied across the board, but we preserve incoming
666	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
667	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
668	* 1. r0 = old & ~mask;
669	* 2. r0 = t1 & (t1 >> 1)'
670	* 3. state \|= r0 \| 0b10;
671	* 4. mask = ~0;
672	*/
673	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
674	/** The register value is used as input.
675	*
676	* This means that the register value must be calculated and it is best to keep
677	* it in a register. It does not need to be writtent out as such. This is the
678	* highest priority state.
679	*
680	* Whether the call modifies the register or not isn't relevant to earlier
681	* calls, so that's not recorded.
682	*
683	* When applying this state we just or in the value in the outgoing state and
684	* mask. */
685	# define IEMLIVENESS_STATE_INPUT 3
686	/** Mask of the state bits. */
687	# define IEMLIVENESS_STATE_MASK 3
688	/** The number of bits per state. */
689	# define IEMLIVENESS_STATE_BIT_COUNT 2
690	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
691	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
692	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
693	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
694	/** Check if a register clobbering is expected given the (previous) liveness state.
695	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
696	* include INPUT if the register is used in more than one place. */
697	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
698
699	/** Check if all status flags are going to be clobbered and doesn't need
700	* calculating in the current step.
701	* @param a_pCurEntry The current liveness entry. */
702	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
703	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
704
705	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
706	/** The register is not used any more. */
707	# define IEMLIVENESS_STATE_UNUSED 0
708	/** Flag: The register is required in a potential exception or call. */
709	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
710	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
711	/** Flag: The register is read. */
712	# define IEMLIVENESS_STATE_READ 2
713	# define IEMLIVENESS_BIT_READ 1
714	/** Flag: The register is written. */
715	# define IEMLIVENESS_STATE_WRITE 4
716	# define IEMLIVENESS_BIT_WRITE 2
717	/** Flag: Unconditional call (not needed, can be redefined for research). */
718	# define IEMLIVENESS_STATE_CALL 8
719	# define IEMLIVENESS_BIT_CALL 3
720	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
721	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
722	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
723	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
724	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
725
726	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
727	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
728	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
729	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
730
731	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
732	/** @} */
733
734	/** @name Liveness helpers for builtin functions and similar.
735	*
736	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
737	* own set of manimulator macros for those.
738	*
739	* @{ */
740	/** Initializing the state as all unused. */
741	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
742	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
743	do { \
744	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
745	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
746	} while (0)
747	#else
748	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
749	do { \
750	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
751	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
752	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
753	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
754	} while (0)
755	#endif
756
757	/** Initializing the outgoing state with a potential xcpt or call state.
758	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
759	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
760	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
761	do { \
762	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
763	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
764	} while (0)
765	#else
766	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
767	do { \
768	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
769	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
770	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
771	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
772	} while (0)
773	#endif
774
775	/** Adds a segment base register as input to the outgoing state. */
776	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
777	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
778	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
779	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
780	} while (0)
781	#else
782	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
783	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
784	} while (0)
785	#endif
786
787	/** Adds a segment attribute register as input to the outgoing state. */
788	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
789	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
790	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
791	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
792	} while (0)
793	#else
794	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
795	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
796	} while (0)
797	#endif
798
799	/** Adds a segment limit register as input to the outgoing state. */
800	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
801	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
802	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
803	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
804	} while (0)
805	#else
806	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
807	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
808	} while (0)
809	#endif
810
811	/** Adds a segment limit register as input to the outgoing state. */
812	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
813	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
814	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
815	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
816	} while (0)
817	#else
818	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
819	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
820	} while (0)
821	#endif
822	/** @} */
823
824	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
825	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
826	* calculated and up to date. This is to double check that we haven't skipped
827	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
828	* @note has to be placed in
829	*/
830	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
831	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
832	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
833	#else
834	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
835	#endif
836
837
838	/**
839	* Guest registers that can be shadowed in GPRs.
840	*
841	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
842	* must be placed last, as the liveness state tracks it as 7 subcomponents and
843	* we don't want to waste space here.
844	*
845	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
846	* friends as well as IEMAllN8veLiveness.cpp.
847	*/
848	typedef enum IEMNATIVEGSTREG : uint8_t
849	{
850	kIemNativeGstReg_GprFirst = 0,
851	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
852	kIemNativeGstReg_Pc,
853	kIemNativeGstReg_Cr0,
854	kIemNativeGstReg_FpuFcw,
855	kIemNativeGstReg_FpuFsw,
856	kIemNativeGstReg_SegBaseFirst,
857	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
858	kIemNativeGstReg_SegAttribFirst,
859	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
860	kIemNativeGstReg_SegLimitFirst,
861	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
862	kIemNativeGstReg_SegSelFirst,
863	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
864	kIemNativeGstReg_Cr4,
865	kIemNativeGstReg_Xcr0,
866	kIemNativeGstReg_MxCsr,
867	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
868	kIemNativeGstReg_End
869	} IEMNATIVEGSTREG;
870	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
871	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
872
873	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
874	* @{ */
875	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
876	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
877	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
878	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
879	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
880	/** @} */
881
882	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
883
884	/**
885	* Guest registers that can be shadowed in host SIMD registers.
886	*
887	* @todo r=aeichner Liveness tracking
888	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
889	*/
890	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
891	{
892	kIemNativeGstSimdReg_SimdRegFirst = 0,
893	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
894	kIemNativeGstSimdReg_End
895	} IEMNATIVEGSTSIMDREG;
896
897	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
898	* @{ */
899	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
900	/** @} */
901
902	/**
903	* The Load/store size for a SIMD guest register.
904	*/
905	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
906	{
907	/** Invalid size. */
908	kIemNativeGstSimdRegLdStSz_Invalid = 0,
909	/** Loads the low 128-bit of a guest SIMD register. */
910	kIemNativeGstSimdRegLdStSz_Low128,
911	/** Loads the high 128-bit of a guest SIMD register. */
912	kIemNativeGstSimdRegLdStSz_High128,
913	/** Loads the whole 256-bits of a guest SIMD register. */
914	kIemNativeGstSimdRegLdStSz_256,
915	/** End value. */
916	kIemNativeGstSimdRegLdStSz_End
917	} IEMNATIVEGSTSIMDREGLDSTSZ;
918
919	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
920
921	/**
922	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
923	*/
924	typedef enum IEMNATIVEGSTREGUSE
925	{
926	/** The usage is read-only, the register holding the guest register
927	* shadow copy will not be modified by the caller. */
928	kIemNativeGstRegUse_ReadOnly = 0,
929	/** The caller will update the guest register (think: PC += cbInstr).
930	* The guest shadow copy will follow the returned register. */
931	kIemNativeGstRegUse_ForUpdate,
932	/** The call will put an entirely new value in the guest register, so
933	* if new register is allocate it will be returned uninitialized. */
934	kIemNativeGstRegUse_ForFullWrite,
935	/** The caller will use the guest register value as input in a calculation
936	* and the host register will be modified.
937	* This means that the returned host register will not be marked as a shadow
938	* copy of the guest register. */
939	kIemNativeGstRegUse_Calculation
940	} IEMNATIVEGSTREGUSE;
941
942	/**
943	* Guest registers (classes) that can be referenced.
944	*/
945	typedef enum IEMNATIVEGSTREGREF : uint8_t
946	{
947	kIemNativeGstRegRef_Invalid = 0,
948	kIemNativeGstRegRef_Gpr,
949	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
950	kIemNativeGstRegRef_EFlags,
951	kIemNativeGstRegRef_MxCsr,
952	kIemNativeGstRegRef_FpuReg,
953	kIemNativeGstRegRef_MReg,
954	kIemNativeGstRegRef_XReg,
955	kIemNativeGstRegRef_X87,
956	kIemNativeGstRegRef_XState,
957	//kIemNativeGstRegRef_YReg, - doesn't work.
958	kIemNativeGstRegRef_End
959	} IEMNATIVEGSTREGREF;
960
961
962	/** Variable kinds. */
963	typedef enum IEMNATIVEVARKIND : uint8_t
964	{
965	/** Customary invalid zero value. */
966	kIemNativeVarKind_Invalid = 0,
967	/** This is either in a register or on the stack. */
968	kIemNativeVarKind_Stack,
969	/** Immediate value - loaded into register when needed, or can live on the
970	* stack if referenced (in theory). */
971	kIemNativeVarKind_Immediate,
972	/** Variable reference - loaded into register when needed, never stack. */
973	kIemNativeVarKind_VarRef,
974	/** Guest register reference - loaded into register when needed, never stack. */
975	kIemNativeVarKind_GstRegRef,
976	/** End of valid values. */
977	kIemNativeVarKind_End
978	} IEMNATIVEVARKIND;
979
980
981	/** Variable or argument. */
982	typedef struct IEMNATIVEVAR
983	{
984	/** The kind of variable. */
985	IEMNATIVEVARKIND enmKind;
986	/** The variable size in bytes. */
987	uint8_t cbVar;
988	/** The first stack slot (uint64_t), except for immediate and references
989	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
990	* has a stack slot it has been initialized and has a value. Unused variables
991	* has neither a stack slot nor a host register assignment. */
992	uint8_t idxStackSlot;
993	/** The host register allocated for the variable, UINT8_MAX if not. */
994	uint8_t idxReg;
995	/** The argument number if argument, UINT8_MAX if regular variable. */
996	uint8_t uArgNo;
997	/** If referenced, the index (unpacked) of the variable referencing this one,
998	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
999	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1000	uint8_t idxReferrerVar;
1001	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1002	* @todo not sure what this really is for... */
1003	IEMNATIVEGSTREG enmGstReg;
1004	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1005	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1006	* only valid when idxReg is not UINT8_MAX. */
1007	bool fSimdReg : 1;
1008	/** Set if the registered is currently used exclusively, false if the
1009	* variable is idle and the register can be grabbed. */
1010	bool fRegAcquired : 1;
1011	#else
1012	/** Set if the registered is currently used exclusively, false if the
1013	* variable is idle and the register can be grabbed. */
1014	bool fRegAcquired;
1015	#endif
1016
1017	union
1018	{
1019	/** kIemNativeVarKind_Immediate: The immediate value. */
1020	uint64_t uValue;
1021	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1022	uint8_t idxRefVar;
1023	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1024	struct
1025	{
1026	/** The class of register. */
1027	IEMNATIVEGSTREGREF enmClass;
1028	/** Index within the class. */
1029	uint8_t idx;
1030	} GstRegRef;
1031	} u;
1032	} IEMNATIVEVAR;
1033	/** Pointer to a variable or argument. */
1034	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1035	/** Pointer to a const variable or argument. */
1036	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1037
1038	/** What is being kept in a host register. */
1039	typedef enum IEMNATIVEWHAT : uint8_t
1040	{
1041	/** The traditional invalid zero value. */
1042	kIemNativeWhat_Invalid = 0,
1043	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1044	kIemNativeWhat_Var,
1045	/** Temporary register, this is typically freed when a MC completes. */
1046	kIemNativeWhat_Tmp,
1047	/** Call argument w/o a variable mapping. This is free (via
1048	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1049	kIemNativeWhat_Arg,
1050	/** Return status code.
1051	* @todo not sure if we need this... */
1052	kIemNativeWhat_rc,
1053	/** The fixed pVCpu (PVMCPUCC) register.
1054	* @todo consider offsetting this on amd64 to use negative offsets to access
1055	* more members using 8-byte disp. */
1056	kIemNativeWhat_pVCpuFixed,
1057	/** The fixed pCtx (PCPUMCTX) register.
1058	* @todo consider offsetting this on amd64 to use negative offsets to access
1059	* more members using 8-byte disp. */
1060	kIemNativeWhat_pCtxFixed,
1061	/** Fixed temporary register. */
1062	kIemNativeWhat_FixedTmp,
1063	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1064	/** Shadow RIP for the delayed RIP updating debugging. */
1065	kIemNativeWhat_PcShadow,
1066	#endif
1067	/** Register reserved by the CPU or OS architecture. */
1068	kIemNativeWhat_FixedReserved,
1069	/** End of valid values. */
1070	kIemNativeWhat_End
1071	} IEMNATIVEWHAT;
1072
1073	/**
1074	* Host general register entry.
1075	*
1076	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1077	*
1078	* @todo Track immediate values in host registers similarlly to how we track the
1079	* guest register shadow copies. For it to be real helpful, though,
1080	* we probably need to know which will be reused and put them into
1081	* non-volatile registers, otherwise it's going to be more or less
1082	* restricted to an instruction or two.
1083	*/
1084	typedef struct IEMNATIVEHSTREG
1085	{
1086	/** Set of guest registers this one shadows.
1087	*
1088	* Using a bitmap here so we can designate the same host register as a copy
1089	* for more than one guest register. This is expected to be useful in
1090	* situations where one value is copied to several registers in a sequence.
1091	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1092	* sequence we'd want to let this register follow to be a copy of and there
1093	* will always be places where we'd be picking the wrong one.
1094	*/
1095	uint64_t fGstRegShadows;
1096	/** What is being kept in this register. */
1097	IEMNATIVEWHAT enmWhat;
1098	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1099	uint8_t idxVar;
1100	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1101	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1102	* that scope. */
1103	uint8_t idxStackSlot;
1104	/** Alignment padding. */
1105	uint8_t abAlign[5];
1106	} IEMNATIVEHSTREG;
1107
1108
1109	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1110	/**
1111	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1112	* halves, on architectures where there is no 256-bit register available this entry will track
1113	* two adjacent 128-bit host registers.
1114	*
1115	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1116	*/
1117	typedef struct IEMNATIVEHSTSIMDREG
1118	{
1119	/** Set of guest registers this one shadows.
1120	*
1121	* Using a bitmap here so we can designate the same host register as a copy
1122	* for more than one guest register. This is expected to be useful in
1123	* situations where one value is copied to several registers in a sequence.
1124	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1125	* sequence we'd want to let this register follow to be a copy of and there
1126	* will always be places where we'd be picking the wrong one.
1127	*/
1128	uint64_t fGstRegShadows;
1129	/** What is being kept in this register. */
1130	IEMNATIVEWHAT enmWhat;
1131	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1132	uint8_t idxVar;
1133	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1134	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1135	/** Alignment padding. */
1136	uint8_t abAlign[5];
1137	} IEMNATIVEHSTSIMDREG;
1138	#endif
1139
1140
1141	/**
1142	* Core state for the native recompiler, that is, things that needs careful
1143	* handling when dealing with branches.
1144	*/
1145	typedef struct IEMNATIVECORESTATE
1146	{
1147	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1148	/** The current instruction offset in bytes from when the guest program counter
1149	* was updated last. Used for delaying the write to the guest context program counter
1150	* as long as possible. */
1151	uint32_t offPc;
1152	/** Number of instructions where we could skip the updating. */
1153	uint32_t cInstrPcUpdateSkipped;
1154	#endif
1155	/** Allocation bitmap for aHstRegs. */
1156	uint32_t bmHstRegs;
1157
1158	/** Bitmap marking which host register contains guest register shadow copies.
1159	* This is used during register allocation to try preserve copies. */
1160	uint32_t bmHstRegsWithGstShadow;
1161	/** Bitmap marking valid entries in aidxGstRegShadows. */
1162	uint64_t bmGstRegShadows;
1163	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1164	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1165	uint64_t bmGstRegShadowDirty;
1166	#endif
1167
1168	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1169	/** Allocation bitmap for aHstSimdRegs. */
1170	uint32_t bmHstSimdRegs;
1171
1172	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1173	* This is used during register allocation to try preserve copies. */
1174	uint32_t bmHstSimdRegsWithGstShadow;
1175	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1176	uint64_t bmGstSimdRegShadows;
1177	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1178	uint64_t bmGstSimdRegShadowDirtyLo128;
1179	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1180	uint64_t bmGstSimdRegShadowDirtyHi128;
1181	#endif
1182
1183	union
1184	{
1185	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1186	uint8_t aidxArgVars[8];
1187	/** For more efficient resetting. */
1188	uint64_t u64ArgVars;
1189	};
1190
1191	/** Allocation bitmap for the stack. */
1192	uint32_t bmStack;
1193	/** Allocation bitmap for aVars. */
1194	uint32_t bmVars;
1195
1196	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1197	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1198	* (A shadow copy of a guest register can only be held in a one host register,
1199	* there are no duplicate copies or ambiguities like that). */
1200	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1201	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1202	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1203	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1204	* (A shadow copy of a guest register can only be held in a one host register,
1205	* there are no duplicate copies or ambiguities like that). */
1206	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1207	#endif
1208
1209	/** Host register allocation tracking. */
1210	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1211	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1212	/** Host SIMD register allocation tracking. */
1213	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1214	#endif
1215
1216	/** Variables and arguments. */
1217	IEMNATIVEVAR aVars[9];
1218	} IEMNATIVECORESTATE;
1219	/** Pointer to core state. */
1220	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1221	/** Pointer to const core state. */
1222	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1223
1224	/** @def IEMNATIVE_VAR_IDX_UNPACK
1225	* @returns Index into IEMNATIVECORESTATE::aVars.
1226	* @param a_idxVar Variable index w/ magic (in strict builds).
1227	*/
1228	/** @def IEMNATIVE_VAR_IDX_PACK
1229	* @returns Variable index w/ magic (in strict builds).
1230	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1231	*/
1232	#ifdef VBOX_STRICT
1233	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1234	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1235	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1236	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1237	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1238	#else
1239	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1240	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1241	#endif
1242
1243
1244	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1245	/** Clear the dirty state of the given guest SIMD register. */
1246	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1247	do { \
1248	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1249	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1250	} while (0)
1251
1252	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1253	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1254	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1255	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1256	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1257	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1258	/** Returns whether the given guest SIMD register is dirty. */
1259	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1260	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1261
1262	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1263	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1264	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1265	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1266	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1267	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1268
1269	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1270	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1271	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1272	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1273	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1274	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1275	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1276	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1277	#endif
1278
1279
1280	/**
1281	* Conditional stack entry.
1282	*/
1283	typedef struct IEMNATIVECOND
1284	{
1285	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1286	bool fInElse;
1287	/** The label for the IEM_MC_ELSE. */
1288	uint32_t idxLabelElse;
1289	/** The label for the IEM_MC_ENDIF. */
1290	uint32_t idxLabelEndIf;
1291	/** The initial state snapshot as the if-block starts executing. */
1292	IEMNATIVECORESTATE InitialState;
1293	/** The state snapshot at the end of the if-block. */
1294	IEMNATIVECORESTATE IfFinalState;
1295	} IEMNATIVECOND;
1296	/** Pointer to a condition stack entry. */
1297	typedef IEMNATIVECOND *PIEMNATIVECOND;
1298
1299
1300	/**
1301	* Native recompiler state.
1302	*/
1303	typedef struct IEMRECOMPILERSTATE
1304	{
1305	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1306	* IEMNATIVEINSTR units. */
1307	uint32_t cInstrBufAlloc;
1308	#ifdef VBOX_STRICT
1309	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1310	uint32_t offInstrBufChecked;
1311	#else
1312	uint32_t uPadding1; /* We don't keep track of the size here... */
1313	#endif
1314	/** Fixed temporary code buffer for native recompilation. */
1315	PIEMNATIVEINSTR pInstrBuf;
1316
1317	/** Bitmaps with the label types used. */
1318	uint64_t bmLabelTypes;
1319	/** Actual number of labels in paLabels. */
1320	uint32_t cLabels;
1321	/** Max number of entries allowed in paLabels before reallocating it. */
1322	uint32_t cLabelsAlloc;
1323	/** Labels defined while recompiling (referenced by fixups). */
1324	PIEMNATIVELABEL paLabels;
1325	/** Array with indexes of unique labels (uData always 0). */
1326	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1327
1328	/** Actual number of fixups paFixups. */
1329	uint32_t cFixups;
1330	/** Max number of entries allowed in paFixups before reallocating it. */
1331	uint32_t cFixupsAlloc;
1332	/** Buffer used by the recompiler for recording fixups when generating code. */
1333	PIEMNATIVEFIXUP paFixups;
1334
1335	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1336	/** Number of debug info entries allocated for pDbgInfo. */
1337	uint32_t cDbgInfoAlloc;
1338	uint32_t uPadding;
1339	/** Debug info. */
1340	PIEMTBDBG pDbgInfo;
1341	#endif
1342
1343	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1344	/** The current call index (liveness array and threaded calls in TB). */
1345	uint32_t idxCurCall;
1346	/** Number of liveness entries allocated. */
1347	uint32_t cLivenessEntriesAlloc;
1348	/** Liveness entries for all the calls in the TB begin recompiled.
1349	* The entry for idxCurCall contains the info for what the next call will
1350	* require wrt registers. (Which means the last entry is the initial liveness
1351	* state.) */
1352	PIEMLIVENESSENTRY paLivenessEntries;
1353	#endif
1354
1355	/** The translation block being recompiled. */
1356	PCIEMTB pTbOrg;
1357	/** The VMCPU structure of the EMT. */
1358	PVMCPUCC pVCpu;
1359
1360	/** Condition sequence number (for generating unique labels). */
1361	uint16_t uCondSeqNo;
1362	/** Check IRQ seqeunce number (for generating unique labels). */
1363	uint16_t uCheckIrqSeqNo;
1364	/** TLB load sequence number (for generating unique labels). */
1365	uint16_t uTlbSeqNo;
1366	/** The current condition stack depth (aCondStack). */
1367	uint8_t cCondDepth;
1368
1369	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1370	uint8_t cArgsX;
1371	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1372	uint32_t fCImpl;
1373	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1374	uint32_t fMc;
1375	/** The expected IEMCPU::fExec value for the current call/instruction. */
1376	uint32_t fExec;
1377	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1378	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1379	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1380	*
1381	* This is an optimization because these control registers can only be changed from
1382	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1383	* consisting of multiple SIMD instructions.
1384	*/
1385	uint32_t fSimdRaiseXcptChecksEmitted;
1386	#endif
1387
1388	/** Core state requiring care with branches. */
1389	IEMNATIVECORESTATE Core;
1390
1391	/** The condition nesting stack. */
1392	IEMNATIVECOND aCondStack[2];
1393
1394	#ifndef IEM_WITH_THROW_CATCH
1395	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1396	* for recompilation error handling. */
1397	jmp_buf JmpBuf;
1398	#endif
1399	} IEMRECOMPILERSTATE;
1400	/** Pointer to a native recompiler state. */
1401	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1402
1403
1404	/** @def IEMNATIVE_TRY_SETJMP
1405	* Wrapper around setjmp / try, hiding all the ugly differences.
1406	*
1407	* @note Use with extreme care as this is a fragile macro.
1408	* @param a_pReNative The native recompile state.
1409	* @param a_rcTarget The variable that should receive the status code in case
1410	* of a longjmp/throw.
1411	*/
1412	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1413	* Start wrapper for catch / setjmp-else.
1414	*
1415	* This will set up a scope.
1416	*
1417	* @note Use with extreme care as this is a fragile macro.
1418	* @param a_pReNative The native recompile state.
1419	* @param a_rcTarget The variable that should receive the status code in case
1420	* of a longjmp/throw.
1421	*/
1422	/** @def IEMNATIVE_CATCH_LONGJMP_END
1423	* End wrapper for catch / setjmp-else.
1424	*
1425	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1426	* up the state.
1427	*
1428	* @note Use with extreme care as this is a fragile macro.
1429	* @param a_pReNative The native recompile state.
1430	*/
1431	/** @def IEMNATIVE_DO_LONGJMP
1432	*
1433	* Wrapper around longjmp / throw.
1434	*
1435	* @param a_pReNative The native recompile state.
1436	* @param a_rc The status code jump back with / throw.
1437	*/
1438	#ifdef IEM_WITH_THROW_CATCH
1439	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1440	a_rcTarget = VINF_SUCCESS; \
1441	try
1442	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1443	catch (int rcThrown) \
1444	{ \
1445	a_rcTarget = rcThrown
1446	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1447	} \
1448	((void)0)
1449	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1450	#else /* !IEM_WITH_THROW_CATCH */
1451	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1452	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1453	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1454	else \
1455	{ \
1456	((void)0)
1457	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1458	}
1459	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1460	#endif /* !IEM_WITH_THROW_CATCH */
1461
1462
1463	/**
1464	* Native recompiler worker for a threaded function.
1465	*
1466	* @returns New code buffer offset; throws VBox status code in case of a failure.
1467	* @param pReNative The native recompiler state.
1468	* @param off The current code buffer offset.
1469	* @param pCallEntry The threaded call entry.
1470	*
1471	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1472	*/
1473	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1474	/** Pointer to a native recompiler worker for a threaded function. */
1475	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1476
1477	/** Defines a native recompiler worker for a threaded function.
1478	* @see FNIEMNATIVERECOMPFUNC */
1479	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1480	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1481
1482	/** Prototypes a native recompiler function for a threaded function.
1483	* @see FNIEMNATIVERECOMPFUNC */
1484	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1485
1486
1487	/**
1488	* Native recompiler liveness analysis worker for a threaded function.
1489	*
1490	* @param pCallEntry The threaded call entry.
1491	* @param pIncoming The incoming liveness state entry.
1492	* @param pOutgoing The outgoing liveness state entry.
1493	*/
1494	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1495	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1496	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1497	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1498
1499	/** Defines a native recompiler liveness analysis worker for a threaded function.
1500	* @see FNIEMNATIVELIVENESSFUNC */
1501	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1502	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1503
1504	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1505	* @see FNIEMNATIVELIVENESSFUNC */
1506	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1507
1508
1509	/** Define a native recompiler helper function, safe to call from the TB code. */
1510	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1511	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1512	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1513	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1514	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1515	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1516	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1517	a_RetType (VBOXCALL *a_Name) a_ArgList
1518
1519
1520	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1521	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1522	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1523	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1524	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1525	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1526	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1527	uint8_t idxHstSimdReg = UINT8_MAX,
1528	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1529	# endif
1530	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1531	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1532	uint8_t idxGstReg, uint8_t idxHstReg);
1533	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1534	uint64_t fGstReg);
1535	# endif
1536	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1537	uint32_t offPc, uint32_t cInstrSkipped);
1538	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1539
1540	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1541	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1542	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1543	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1544	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1545	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1546
1547	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1548	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1549	bool fPreferVolatile = true);
1550	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1551	bool fPreferVolatile = true);
1552	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1553	IEMNATIVEGSTREG enmGstReg,
1554	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1555	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1556	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1557	IEMNATIVEGSTREG enmGstReg);
1558
1559	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1560	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1561	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1562	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1563	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1564	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1565	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1566	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1567	# endif
1568	#endif
1569	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1570	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1571	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1572	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1573	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1574	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1575	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1576	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1577	# endif
1578	#endif
1579	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1580	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1581	uint32_t fKeepVars = 0);
1582	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1583	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1584	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1585	uint32_t fHstRegsActiveShadows);
1586	#ifdef VBOX_STRICT
1587	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1588	#endif
1589	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1590	uint64_t fGstSimdShwExcept);
1591	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1592	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1593	#endif
1594	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1595	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1596	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1597	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1598	#endif
1599
1600
1601	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1602	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1603	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1604	bool fPreferVolatile = true);
1605	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1606	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1607	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1608	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1609	bool fNoVolatileRegs = false);
1610	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1611	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1612	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1613	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1614	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1615	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1616	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1617	#endif
1618
1619	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1620	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1621	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1622	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1623	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1624	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1625	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1626	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1627	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1628	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1629	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1630	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1631	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1632	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1633	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1634	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1635	#endif
1636	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1637	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1638	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1639	uint32_t fHstRegsNotToSave);
1640	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1641	uint32_t fHstRegsNotToSave);
1642	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1643	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1644
1645	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1646	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1647	#ifdef VBOX_STRICT
1648	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1649	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1650	IEMNATIVEGSTREG enmGstReg);
1651	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1652	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1653	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1654	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1655	# endif
1656	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1657	#endif
1658	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1659	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1660	#endif
1661	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1662	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1663	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1664	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1665	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1666	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1667	PCIEMTHRDEDCALLENTRY pCallEntry);
1668	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1669	uint8_t idxAddrReg, uint8_t idxInstr);
1670	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1671	uint8_t idxAddrReg, uint8_t idxInstr);
1672	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1673	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1674
1675
1676	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1677	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1678	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1679	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1680	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1681	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1682	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1683	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1684	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1685	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1686
1687	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1688	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1689	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1690	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1691	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1692	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1693	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1694	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1695	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1696	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1697	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1698	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1699	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1700	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1701	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1702	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1703	#endif
1704	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1705	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1706	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1707	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1708	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1709	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1710	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1711	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1712	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1713	#endif
1714	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1715	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1716	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1717	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1718	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1719	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1720	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1721
1722	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1723	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1724	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1725	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1726	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1727	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1728	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1729	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1730	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1731	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1732	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1733	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1734	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1735	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1736	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1737	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1738	#endif
1739	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1740	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1741	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1742	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1743	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1744	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1745	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1746	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1747	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1748	#endif
1749	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1750	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1751	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1752	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1753	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1754	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1755	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1756
1757	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1758	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1759	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1760	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1761	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1762	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1763	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1764	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1765	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1766	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1767	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1768	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1769	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1770	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1771	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1772	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1773	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1774	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1775	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1776	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1777	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1778	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1779
1780	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1781	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1782	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1783	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1784	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1785	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1786	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1787	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1788	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1789	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1790	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1791	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1792	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1793	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1794	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1795	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1796	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1797	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1798	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1799	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1800	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1801	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1802
1803	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1804	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1805	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1806	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1807
1808
1809	/**
1810	* Info about shadowed guest register values.
1811	* @see IEMNATIVEGSTREG
1812	*/
1813	typedef struct IEMANTIVEGSTREGINFO
1814	{
1815	/** Offset in VMCPU. */
1816	uint32_t off;
1817	/** The field size. */
1818	uint8_t cb;
1819	/** Name (for logging). */
1820	const char *pszName;
1821	} IEMANTIVEGSTREGINFO;
1822	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1823	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1824	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1825	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1826	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1827
1828
1829
1830	/**
1831	* Ensures that there is sufficient space in the instruction output buffer.
1832	*
1833	* This will reallocate the buffer if needed and allowed.
1834	*
1835	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1836	* allocation size.
1837	*
1838	* @returns Pointer to the instruction output buffer on success; throws VBox
1839	* status code on failure, so no need to check it.
1840	* @param pReNative The native recompile state.
1841	* @param off Current instruction offset. Works safely for UINT32_MAX
1842	* as well.
1843	* @param cInstrReq Number of instruction about to be added. It's okay to
1844	* overestimate this a bit.
1845	*/
1846	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1847	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1848	{
1849	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1850	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1851	{
1852	#ifdef VBOX_STRICT
1853	pReNative->offInstrBufChecked = offChecked;
1854	#endif
1855	return pReNative->pInstrBuf;
1856	}
1857	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1858	}
1859
1860	/**
1861	* Checks that we didn't exceed the space requested in the last
1862	* iemNativeInstrBufEnsure() call.
1863	*/
1864	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1865	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1866	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1867
1868	/**
1869	* Checks that a variable index is valid.
1870	*/
1871	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1872	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1873	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1874	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1875	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1876	("%s=%#x\n", #a_idxVar, a_idxVar))
1877	#else
1878	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1879	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1880	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1881	#endif
1882
1883	/**
1884	* Checks that a variable index is valid and that the variable is assigned the
1885	* correct argument number.
1886	* This also adds a RT_NOREF of a_idxVar.
1887	*/
1888	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1889	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1890	RT_NOREF_PV(a_idxVar); \
1891	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1892	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1893	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1894	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1895	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1896	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1897	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1898	a_uArgNo)); \
1899	} while (0)
1900	#else
1901	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1902	RT_NOREF_PV(a_idxVar); \
1903	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1904	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1905	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1906	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1907	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1908	} while (0)
1909	#endif
1910
1911
1912	/**
1913	* Checks that a variable has the expected size.
1914	*/
1915	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1916	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1917	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1918	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1919
1920
1921	/**
1922	* Calculates the stack address of a variable as a [r]BP displacement value.
1923	*/
1924	DECL_FORCE_INLINE(int32_t)
1925	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1926	{
1927	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1928	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1929	}
1930
1931
1932	/**
1933	* Releases the variable's register.
1934	*
1935	* The register must have been previously acquired calling
1936	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1937	* iemNativeVarRegisterSetAndAcquire().
1938	*/
1939	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1940	{
1941	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1942	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1943	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1944	}
1945
1946
1947	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1948	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1949	{
1950	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
1951	iemNativeVarRegisterRelease(pReNative, idxVar);
1952	}
1953	#endif
1954
1955
1956	/**
1957	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1958	*
1959	* @returns The flush mask.
1960	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1961	* @param fGstShwFlush The starting flush mask.
1962	*/
1963	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1964	{
1965	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1966	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1967	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1968	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1969	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1970	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1971	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1972	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1973	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1974	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1975	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1976	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1977	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1978	return fGstShwFlush;
1979	}
1980
1981
1982	/** Number of hidden arguments for CIMPL calls.
1983	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1984	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1985	# define IEM_CIMPL_HIDDEN_ARGS 3
1986	#else
1987	# define IEM_CIMPL_HIDDEN_ARGS 2
1988	#endif
1989
1990
1991	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1992	/** Number of hidden arguments for SSE_AIMPL calls. */
1993	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
1994	/** Number of hidden arguments for AVX_AIMPL calls. */
1995	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
1996	#endif
1997
1998
1999	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2000
2001	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2002	/**
2003	* Helper for iemNativeLivenessGetStateByGstReg.
2004	*
2005	* @returns IEMLIVENESS_STATE_XXX
2006	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2007	* ORed together.
2008	*/
2009	DECL_FORCE_INLINE(uint32_t)
2010	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2011	{
2012	/* INPUT trumps anything else. */
2013	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2014	return IEMLIVENESS_STATE_INPUT;
2015
2016	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2017	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2018	{
2019	/* If not all sub-fields are clobbered they must be considered INPUT. */
2020	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2021	return IEMLIVENESS_STATE_INPUT;
2022	return IEMLIVENESS_STATE_CLOBBERED;
2023	}
2024
2025	/* XCPT_OR_CALL trumps UNUSED. */
2026	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2027	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2028
2029	return IEMLIVENESS_STATE_UNUSED;
2030	}
2031	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2032
2033
2034	DECL_FORCE_INLINE(uint32_t)
2035	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2036	{
2037	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2038	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2039	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2040	# else
2041	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2042	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2043	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2044	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2045	# endif
2046	}
2047
2048
2049	DECL_FORCE_INLINE(uint32_t)
2050	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2051	{
2052	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2053	if (enmGstReg == kIemNativeGstReg_EFlags)
2054	{
2055	/* Merge the eflags states to one. */
2056	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2057	uRet = RT_BIT_32(uRet);
2058	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2059	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2060	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2061	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2062	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2063	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2064	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2065	# else
2066	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2067	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2068	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2069	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2070	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2071	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2072	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2073	# endif
2074	}
2075	return uRet;
2076	}
2077
2078
2079	# ifdef VBOX_STRICT
2080	/** For assertions only, user checks that idxCurCall isn't zerow. */
2081	DECL_FORCE_INLINE(uint32_t)
2082	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2083	{
2084	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2085	}
2086	# endif /* VBOX_STRICT */
2087
2088	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2089
2090
2091	/**
2092	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2093	*/
2094	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2095	{
2096	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2097	return IEM_CIMPL_HIDDEN_ARGS;
2098	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2099	return 1;
2100	return 0;
2101	}
2102
2103
2104	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2105	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2106	{
2107	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2108
2109	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2110	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2111	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2112	return (uint8_t)idxReg;
2113	}
2114
2115
2116
2117	/*********************************************************************************************************************************
2118	* Register Allocator (GPR) *
2119	*********************************************************************************************************************************/
2120
2121	/**
2122	* Marks host register @a idxHstReg as containing a shadow copy of guest
2123	* register @a enmGstReg.
2124	*
2125	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2126	* host register before calling.
2127	*/
2128	DECL_FORCE_INLINE(void)
2129	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2130	{
2131	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2132	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2133	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2134
2135	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2136	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2137	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2138	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2139	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2140	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2141	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2142	#else
2143	RT_NOREF(off);
2144	#endif
2145	}
2146
2147
2148	/**
2149	* Clear any guest register shadow claims from @a idxHstReg.
2150	*
2151	* The register does not need to be shadowing any guest registers.
2152	*/
2153	DECL_FORCE_INLINE(void)
2154	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2155	{
2156	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2157	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2158	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2159	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2160	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2161	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2162	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2163	#endif
2164
2165	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2166	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2167	if (fGstRegs)
2168	{
2169	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2170	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2171	while (fGstRegs)
2172	{
2173	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2174	fGstRegs &= ~RT_BIT_64(iGstReg);
2175	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2176	}
2177	}
2178	#else
2179	RT_NOREF(off);
2180	#endif
2181
2182	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2183	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2184	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2185	}
2186
2187
2188	/**
2189	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2190	* and global overview flags.
2191	*/
2192	DECL_FORCE_INLINE(void)
2193	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2194	{
2195	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2196	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2197	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2198	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2199	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2200	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2201	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2202	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2203	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2204	#endif
2205
2206	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2207	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2208	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2209	#else
2210	RT_NOREF(off);
2211	#endif
2212
2213	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2214	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2215	if (!fGstRegShadowsNew)
2216	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2217	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2218	}
2219
2220
2221	#if 0 /* unused */
2222	/**
2223	* Clear any guest register shadow claim for @a enmGstReg.
2224	*/
2225	DECL_FORCE_INLINE(void)
2226	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2227	{
2228	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2229	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2230	{
2231	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2232	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2233	}
2234	}
2235	#endif
2236
2237
2238	/**
2239	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2240	* as the new shadow of it.
2241	*
2242	* Unlike the other guest reg shadow helpers, this does the logging for you.
2243	* However, it is the liveness state is not asserted here, the caller must do
2244	* that.
2245	*/
2246	DECL_FORCE_INLINE(void)
2247	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2248	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2249	{
2250	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2251	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2252	{
2253	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2254	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2255	if (idxHstRegOld == idxHstRegNew)
2256	return;
2257	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2258	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2259	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2260	}
2261	else
2262	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2263	g_aGstShadowInfo[enmGstReg].pszName));
2264	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2265	}
2266
2267
2268	/**
2269	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2270	* to @a idxRegTo.
2271	*/
2272	DECL_FORCE_INLINE(void)
2273	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2274	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2275	{
2276	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2277	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2278	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2279	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2280	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2281	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2282	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2283	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2284	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2285
2286	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2287	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2288	if (!fGstRegShadowsFrom)
2289	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2290	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2291	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2292	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2293	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2294	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2295	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2296	#else
2297	RT_NOREF(off);
2298	#endif
2299	}
2300
2301
2302	/**
2303	* Flushes any delayed guest register writes.
2304	*
2305	* This must be called prior to calling CImpl functions and any helpers that use
2306	* the guest state (like raising exceptions) and such.
2307	*
2308	* This optimization has not yet been implemented. The first target would be
2309	* RIP updates, since these are the most common ones.
2310	*
2311	* @note This function does not flush any shadowing information for guest registers. This needs to be done by
2312	* the caller if it wishes to do so.
2313	*/
2314	DECL_INLINE_THROW(uint32_t)
2315	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2316	{
2317	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2318	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2319	#else
2320	uint64_t const bmGstRegShadowDirty = 0;
2321	#endif
2322	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2323	uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 \| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2324	& ~fGstSimdShwExcept;
2325	#else
2326	uint64_t const bmGstSimdRegShadowDirty = 0;
2327	#endif
2328	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2329	uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2330	#else
2331	uint64_t const fWritebackPc = 0;
2332	#endif
2333	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2334	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2335
2336	return off;
2337	}
2338
2339
2340
2341	/*********************************************************************************************************************************
2342	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2343	*********************************************************************************************************************************/
2344
2345	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2346
2347	DECL_FORCE_INLINE(uint8_t)
2348	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2349	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2350	{
2351	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2352
2353	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2354	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2355	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2356	return idxSimdReg;
2357	}
2358
2359
2360	/**
2361	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2362	* SIMD register @a enmGstSimdReg.
2363	*
2364	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2365	* host register before calling.
2366	*/
2367	DECL_FORCE_INLINE(void)
2368	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2369	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2370	{
2371	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2372	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2373	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2374
2375	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2376	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2377	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2378	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2379	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2380	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2381	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2382	#else
2383	RT_NOREF(off);
2384	#endif
2385	}
2386
2387
2388	/**
2389	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2390	* to @a idxSimdRegTo.
2391	*/
2392	DECL_FORCE_INLINE(void)
2393	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2394	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2395	{
2396	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2397	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2398	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2399	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2400	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2401	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2402	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2403	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2404	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2405	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2406	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2407
2408	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2409	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2410	if (!fGstRegShadowsFrom)
2411	{
2412	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2413	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2414	}
2415	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2416	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2417	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2418	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2419	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2420	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2421	#else
2422	RT_NOREF(off);
2423	#endif
2424	}
2425
2426
2427	/**
2428	* Clear any guest register shadow claims from @a idxHstSimdReg.
2429	*
2430	* The register does not need to be shadowing any guest registers.
2431	*/
2432	DECL_FORCE_INLINE(void)
2433	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2434	{
2435	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2436	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2437	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2438	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2439	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2440	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2441	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2442
2443	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2444	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2445	if (fGstRegs)
2446	{
2447	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2448	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2449	while (fGstRegs)
2450	{
2451	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2452	fGstRegs &= ~RT_BIT_64(iGstReg);
2453	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2454	}
2455	}
2456	#else
2457	RT_NOREF(off);
2458	#endif
2459
2460	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2461	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2462	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2463	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2464	}
2465
2466	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2467
2468
2469	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2470	/**
2471	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2472	*/
2473	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2474	{
2475	if (pReNative->Core.offPc)
2476	return iemNativeEmitPcWritebackSlow(pReNative, off);
2477	return off;
2478	}
2479	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2480
2481
2482	/** @} */
2483
2484	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2485

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 104298

Download in other formats: