IEMN8veRecompiler.h@ 106099

Last change on this file since 106099 was 106099, checked in by vboxsync, 7 months ago
VMM/IEM: Moved the kIemNativeGstReg_Pc to the end of the enum to optimize IEMLIVENESSBIT_MASK loading on arm, reducing the liveness code by ~80KB (compressed) / 144 KB (extended layout). bugref:10372
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 136.0 KB

Line
1	/* $Id: IEMN8veRecompiler.h 106099 2024-09-19 20:18:15Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	/# define IEMLIVENESS_EXTENDED_LAYOUT/
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62	* Enables strict consistency checks around EFLAGS skipping.
63	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65	# ifdef VBOX_STRICT
66	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67	# endif
68	#elif defined(DOXYGEN_RUNNING)
69	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70	#endif
71
72	#ifdef VBOX_WITH_STATISTICS
73	/** Always count instructions for now. */
74	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75	#endif
76
77	/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78	* Enables having only a single prologue for native TBs. */
79	#if 1 \|\| defined(DOXYGEN_RUNNING)
80	# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81	#endif
82
83	/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84	* Enable this to use common epilogue and tail code for all TBs in a chunk. */
85	#if 1 \|\| defined(DOXYGEN_RUNNING)
86	# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87	#endif
88
89
90	/** @name Stack Frame Layout
91	*
92	* @{ */
93	/** The size of the area for stack variables and spills and stuff.
94	* @note This limit is duplicated in the python script(s). We add 0x40 for
95	* alignment padding. */
96	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101	#ifdef RT_ARCH_AMD64
102	/** An stack alignment adjustment (between non-volatile register pushes and
103	* the stack variable area, so the latter better aligned). */
104	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106	/** Number of stack arguments slots for calls made from the frame. */
107	# ifdef RT_OS_WINDOWS
108	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109	# else
110	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111	# endif
112	/** Number of any shadow arguments (spill area) for calls we make. */
113	# ifdef RT_OS_WINDOWS
114	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115	# else
116	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117	# endif
118
119	/** Frame pointer (RBP) relative offset of the last push. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122	# else
123	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124	# endif
125	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126	* address for it). */
127	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132	# ifdef RT_OS_WINDOWS
133	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137	# endif
138
139	# ifdef RT_OS_WINDOWS
140	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148	# endif
149
150	#elif RT_ARCH_ARM64
151	/** No alignment padding needed for arm64. */
152	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153	/** No stack argument slots, got 8 registers for arguments will suffice. */
154	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155	/** There are no argument spill area. */
156	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158	/** Number of saved registers at the top of our stack frame.
159	* This includes the return address and old frame pointer, so x19 thru x30. */
160	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164	/** Frame pointer (BP) relative offset of the last push. */
165	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168	* address for it). */
169	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171	#else
172	# error "port me"
173	#endif
174	/** @} */
175
176
177	/** @name Fixed Register Allocation(s)
178	* @{ */
179	/** @def IEMNATIVE_REG_FIXED_PVMCPU
180	* The number of the register holding the pVCpu pointer. */
181	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183	* @note This not available on AMD64, only ARM64. */
184	/** @def IEMNATIVE_REG_FIXED_TMP0
185	* Dedicated temporary register.
186	* @note This has extremely short lifetime, must be used with great care to make
187	* sure any calling code or code being called is making use of it.
188	* It will definitely not survive a call or anything of that nature.
189	* @todo replace this by a register allocator and content tracker. */
190	/** @def IEMNATIVE_REG_FIXED_MASK
191	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
192	* architecture. */
193	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
196	* architecture. */
197	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
198	* Dedicated temporary SIMD register. */
199	#endif
200	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
201	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
202	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
203	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
204	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
205	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
206	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
207	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
208	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
209	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
210	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
211	# else
212	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
213	# endif
214	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
215	\| RT_BIT_32(ARMV8_A64_REG_LR) \
216	\| RT_BIT_32(ARMV8_A64_REG_BP) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
218	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
219	\| RT_BIT_32(ARMV8_A64_REG_X18) \
220	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
221	\| IEMNATIVE_REG_FIXED_MASK_ADD)
222
223	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
224	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
225	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
227	# else
228	/** @note
229	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
230	* support emulating 256-bit registers we pair two real registers statically to
231	* one virtual for now, leaving us with only 16 256-bit registers. We always
232	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
233	* the register allocator assumes that it will be always free when the lower is
234	* picked.
235	*
236	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
237	* touch them in order to avoid having to save and restore them in the
238	* prologue/epilogue.
239	*/
240	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
252	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
253	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
255	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
256	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
257	\| RT_BIT_32(ARMV8_A64_REG_Q1))
258	# endif
259	# endif
260
261	#elif defined(RT_ARCH_AMD64)
262	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
263	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
264	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
265	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
266	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
267	\| RT_BIT_32(X86_GREG_xSP) \
268	\| RT_BIT_32(X86_GREG_xBP) )
269
270	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
271	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
272	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
273	# ifndef _MSC_VER
274	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275	# endif
276	# endif
277	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
278	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
279	# else
280	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
281	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
282	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
283	# endif
284	# endif
285
286	#else
287	# error "port me"
288	#endif
289	/** @} */
290
291	/** @name Call related registers.
292	* @{ */
293	/** @def IEMNATIVE_CALL_RET_GREG
294	* The return value register. */
295	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
296	* Number of arguments in registers. */
297	/** @def IEMNATIVE_CALL_ARG0_GREG
298	* The general purpose register carrying argument \#0. */
299	/** @def IEMNATIVE_CALL_ARG1_GREG
300	* The general purpose register carrying argument \#1. */
301	/** @def IEMNATIVE_CALL_ARG2_GREG
302	* The general purpose register carrying argument \#2. */
303	/** @def IEMNATIVE_CALL_ARG3_GREG
304	* The general purpose register carrying argument \#3. */
305	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
306	* Mask of registers the callee will not save and may trash. */
307	#ifdef RT_ARCH_AMD64
308	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
309
310	# ifdef RT_OS_WINDOWS
311	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
312	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
313	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
314	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
315	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
316	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
317	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
318	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
319	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
320	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
321	\| RT_BIT_32(X86_GREG_xCX) \
322	\| RT_BIT_32(X86_GREG_xDX) \
323	\| RT_BIT_32(X86_GREG_x8) \
324	\| RT_BIT_32(X86_GREG_x9) \
325	\| RT_BIT_32(X86_GREG_x10) \
326	\| RT_BIT_32(X86_GREG_x11) )
327	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
328	/* xmm0 - xmm5 are marked as volatile. */
329	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
330	# endif
331
332	# else /* !RT_OS_WINDOWS */
333	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
334	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
335	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
336	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
337	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
338	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
339	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
340	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
341	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
342	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
343	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
344	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
345	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
346	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
347	\| RT_BIT_32(X86_GREG_xCX) \
348	\| RT_BIT_32(X86_GREG_xDX) \
349	\| RT_BIT_32(X86_GREG_xDI) \
350	\| RT_BIT_32(X86_GREG_xSI) \
351	\| RT_BIT_32(X86_GREG_x8) \
352	\| RT_BIT_32(X86_GREG_x9) \
353	\| RT_BIT_32(X86_GREG_x10) \
354	\| RT_BIT_32(X86_GREG_x11) )
355	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
356	/* xmm0 - xmm15 are marked as volatile. */
357	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
358	# endif
359	# endif /* !RT_OS_WINDOWS */
360
361	#elif defined(RT_ARCH_ARM64)
362	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
363	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
364	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
365	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
366	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
367	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
368	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
369	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
370	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
371	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
372	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373	\| RT_BIT_32(ARMV8_A64_REG_X1) \
374	\| RT_BIT_32(ARMV8_A64_REG_X2) \
375	\| RT_BIT_32(ARMV8_A64_REG_X3) \
376	\| RT_BIT_32(ARMV8_A64_REG_X4) \
377	\| RT_BIT_32(ARMV8_A64_REG_X5) \
378	\| RT_BIT_32(ARMV8_A64_REG_X6) \
379	\| RT_BIT_32(ARMV8_A64_REG_X7) )
380	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
381	\| RT_BIT_32(ARMV8_A64_REG_X1) \
382	\| RT_BIT_32(ARMV8_A64_REG_X2) \
383	\| RT_BIT_32(ARMV8_A64_REG_X3) \
384	\| RT_BIT_32(ARMV8_A64_REG_X4) \
385	\| RT_BIT_32(ARMV8_A64_REG_X5) \
386	\| RT_BIT_32(ARMV8_A64_REG_X6) \
387	\| RT_BIT_32(ARMV8_A64_REG_X7) \
388	\| RT_BIT_32(ARMV8_A64_REG_X8) \
389	\| RT_BIT_32(ARMV8_A64_REG_X9) \
390	\| RT_BIT_32(ARMV8_A64_REG_X10) \
391	\| RT_BIT_32(ARMV8_A64_REG_X11) \
392	\| RT_BIT_32(ARMV8_A64_REG_X12) \
393	\| RT_BIT_32(ARMV8_A64_REG_X13) \
394	\| RT_BIT_32(ARMV8_A64_REG_X14) \
395	\| RT_BIT_32(ARMV8_A64_REG_X15) \
396	\| RT_BIT_32(ARMV8_A64_REG_X16) \
397	\| RT_BIT_32(ARMV8_A64_REG_X17) )
398	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
399	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
400	* so to simplify our life a bit we just mark everything as volatile. */
401	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
402	# endif
403
404	#endif
405
406	/** This is the maximum argument count we'll ever be needing. */
407	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
408	#ifdef RT_OS_WINDOWS
409	# ifdef VBOXSTRICTRC_STRICT_ENABLED
410	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
411	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
412	# endif
413	#endif
414
415	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
416	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
417	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
418	#ifdef IEMNATIVE_REG_FIXED_TMP0
419	# ifdef IEMNATIVE_REG_FIXED_TMP1
420	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
421	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
422	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
423	# else
424	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
425	# endif
426	#else
427	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
428	#endif
429	/** @} */
430
431
432	/** @def IEMNATIVE_HST_GREG_COUNT
433	* Number of host general purpose registers we tracker. */
434	/** @def IEMNATIVE_HST_GREG_MASK
435	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
436	* inverted register masks and such to get down to a correct set of regs. */
437	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
438	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
439	* Number of host SIMD registers we track. */
440	/** @def IEMNATIVE_HST_SIMD_REG_MASK
441	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
442	* inverted register masks and such to get down to a correct set of regs. */
443	#endif
444	#ifdef RT_ARCH_AMD64
445	# define IEMNATIVE_HST_GREG_COUNT 16
446	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
447
448	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
449	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
450	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
451	# endif
452
453	#elif defined(RT_ARCH_ARM64)
454	# define IEMNATIVE_HST_GREG_COUNT 32
455	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
456
457	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
458	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
459	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
460	# endif
461
462	#else
463	# error "Port me!"
464	#endif
465
466
467	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
468
469
470	/** Native code generator label types. */
471	typedef enum
472	{
473	kIemNativeLabelType_Invalid = 0,
474	/*
475	* Labels w/o data, only once instance per TB - aka exit reasons.
476	*
477	* Note! Jumps to these requires instructions that are capable of spanning
478	* the max TB length.
479	*/
480	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
481	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
482	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
483	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
484	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
485	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
486	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
487	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
488	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
489	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
490	kIemNativeLabelType_ObsoleteTb,
491	kIemNativeLabelType_NeedCsLimChecking,
492	kIemNativeLabelType_CheckBranchMiss,
493	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
494	/* Manually defined labels. */
495	kIemNativeLabelType_ReturnBreak,
496	kIemNativeLabelType_ReturnBreakFF,
497	kIemNativeLabelType_ReturnBreakViaLookup,
498	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
499	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
500	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
501	kIemNativeLabelType_ReturnWithFlags,
502	kIemNativeLabelType_NonZeroRetOrPassUp,
503	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
504	kIemNativeLabelType_ReturnSuccess, /*< Sets eax/w0 to zero and returns. /
505	#else
506	kIemNativeLabelType_Return,
507	#endif
508	/** The last fixup for branches that can span almost the whole TB length.
509	* @note Whether kIemNativeLabelType_Return needs to be one of these is
510	* a bit questionable, since nobody jumps to it except other tail code. */
511	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
512	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
513	#else
514	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
515	#endif
516	/** The last fixup for branches that exits the TB. */
517	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
518	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_ReturnSuccess,
519	#else
520	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
521	#endif
522
523	/** Loop-jump target. */
524	kIemNativeLabelType_LoopJumpTarget,
525
526	/*
527	* Labels with data, potentially multiple instances per TB:
528	*
529	* These are localized labels, so no fixed jump type restrictions here.
530	*/
531	kIemNativeLabelType_FirstWithMultipleInstances,
532	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
533	kIemNativeLabelType_Else,
534	kIemNativeLabelType_Endif,
535	kIemNativeLabelType_CheckIrq,
536	kIemNativeLabelType_TlbLookup,
537	kIemNativeLabelType_TlbMiss,
538	kIemNativeLabelType_TlbDone,
539	kIemNativeLabelType_End
540	} IEMNATIVELABELTYPE;
541
542	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
543	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
544
545
546	/** Native code generator label definition. */
547	typedef struct IEMNATIVELABEL
548	{
549	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
550	* the epilog. */
551	uint32_t off;
552	/** The type of label (IEMNATIVELABELTYPE). */
553	uint16_t enmType;
554	/** Additional label data, type specific. */
555	uint16_t uData;
556	} IEMNATIVELABEL;
557	/** Pointer to a label. */
558	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
559
560
561	/** Native code generator fixup types. */
562	typedef enum
563	{
564	kIemNativeFixupType_Invalid = 0,
565	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
566	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
567	kIemNativeFixupType_Rel32,
568	#elif defined(RT_ARCH_ARM64)
569	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
570	kIemNativeFixupType_RelImm26At0,
571	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
572	kIemNativeFixupType_RelImm19At5,
573	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
574	kIemNativeFixupType_RelImm14At5,
575	#endif
576	kIemNativeFixupType_End
577	} IEMNATIVEFIXUPTYPE;
578
579	/** Native code generator fixup. */
580	typedef struct IEMNATIVEFIXUP
581	{
582	/** Code offset of the fixup location. */
583	uint32_t off;
584	/** The IEMNATIVELABEL this is a fixup for. */
585	uint16_t idxLabel;
586	/** The fixup type (IEMNATIVEFIXUPTYPE). */
587	uint8_t enmType;
588	/** Addend or other data. */
589	int8_t offAddend;
590	} IEMNATIVEFIXUP;
591	/** Pointer to a native code generator fixup. */
592	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
593
594	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
595
596	/** Native code generator fixup to per chunk TB tail code. */
597	typedef struct IEMNATIVEEXITFIXUP
598	{
599	/** Code offset of the fixup location. */
600	uint32_t off;
601	/** The exit reason. */
602	IEMNATIVELABELTYPE enmExitReason;
603	} IEMNATIVEEXITFIXUP;
604	/** Pointer to a native code generator TB exit fixup. */
605	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
606
607	/**
608	* Per executable memory chunk context with addresses for common code.
609	*/
610	typedef struct IEMNATIVEPERCHUNKCTX
611	{
612	/** Pointers to the exit labels */
613	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
614	} IEMNATIVEPERCHUNKCTX;
615	/** Pointer to per-chunk recompiler context. */
616	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
617	/** Pointer to const per-chunk recompiler context. */
618	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
619
620	#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
621
622
623	/**
624	* One bit of the state.
625	*
626	* Each register state takes up two bits. We keep the two bits in two separate
627	* 64-bit words to simplify applying them to the guest shadow register mask in
628	* the register allocator.
629	*/
630	typedef union IEMLIVENESSBIT
631	{
632	uint64_t bm64;
633	RT_GCC_EXTENSION struct
634	{ /* bit no */
635	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
636	uint64_t fCr0 : 1; /*< 0x10 / 16: /
637	uint64_t fCr4 : 1; /*< 0x11 / 17: /
638	uint64_t fFcw : 1; /*< 0x12 / 18: /
639	uint64_t fFsw : 1; /*< 0x13 / 19: /
640	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
641	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
642	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
643	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
644	uint64_t fXcr0 : 1; /*< 0x2c / 44: /
645	uint64_t fMxCsr : 1; /*< 0x2d / 45: /
646	uint64_t fEflOther : 1; /*< 0x2e / 46: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
647	uint64_t fEflCf : 1; /*< 0x2f / 47: Carry flag (X86_EFL_CF / 0). /
648	uint64_t fEflPf : 1; /*< 0x30 / 48: Parity flag (X86_EFL_PF / 2). /
649	uint64_t fEflAf : 1; /*< 0x31 / 59: Auxilary carry flag (X86_EFL_AF / 4). /
650	uint64_t fEflZf : 1; /*< 0x32 / 50: Zero flag (X86_EFL_ZF / 6). /
651	uint64_t fEflSf : 1; /*< 0x33 / 51: Signed flag (X86_EFL_SF / 7). /
652	uint64_t fEflOf : 1; /*< 0x34 / 52: Overflow flag (X86_EFL_OF / 12). /
653	uint64_t fUnusedPc : 1; /*< 0x35 / 53: (PC in ) /
654	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
655	};
656	} IEMLIVENESSBIT;
657	AssertCompileSize(IEMLIVENESSBIT, 8);
658
659	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
660	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
661	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
662	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
663	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
664	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
665	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
666
667
668	/**
669	* A liveness state entry.
670	*
671	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
672	* Once we add a SSE register shadowing, we'll add another 64-bit element for
673	* that.
674	*/
675	typedef union IEMLIVENESSENTRY
676	{
677	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
678	uint64_t bm64[16 / 8];
679	uint16_t bm32[16 / 4];
680	uint16_t bm16[16 / 2];
681	uint8_t bm8[ 16 / 1];
682	IEMLIVENESSBIT aBits[2];
683	#else
684	uint64_t bm64[32 / 8];
685	uint16_t bm32[32 / 4];
686	uint16_t bm16[32 / 2];
687	uint8_t bm8[ 32 / 1];
688	IEMLIVENESSBIT aBits[4];
689	#endif
690	RT_GCC_EXTENSION struct
691	{
692	/** Bit \#0 of the register states. */
693	IEMLIVENESSBIT Bit0;
694	/** Bit \#1 of the register states. */
695	IEMLIVENESSBIT Bit1;
696	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
697	/** Bit \#2 of the register states. */
698	IEMLIVENESSBIT Bit2;
699	/** Bit \#3 of the register states. */
700	IEMLIVENESSBIT Bit3;
701	#endif
702	};
703	} IEMLIVENESSENTRY;
704	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
705	AssertCompileSize(IEMLIVENESSENTRY, 16);
706	#else
707	AssertCompileSize(IEMLIVENESSENTRY, 32);
708	#endif
709	/** Pointer to a liveness state entry. */
710	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
711	/** Pointer to a const liveness state entry. */
712	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
713
714	/** @name 64-bit value masks for IEMLIVENESSENTRY.
715	* @{ / / 0xzzzzyyyyxxxxwwww */
716	#define IEMLIVENESSBIT_MASK UINT64_C(0x001fffffffffffff)
717
718	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
719	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
720	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
721
722	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
723	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
724	#endif
725
726	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x001fc00000000000)
727	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x001f800000000000)
728
729	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
730	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
731	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
732	#endif
733	/** @} */
734
735
736	/** @name The liveness state for a register.
737	*
738	* The state values have been picked to with state accumulation in mind (what
739	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
740	* performance critical work done with the values.
741	*
742	* This is a compressed state that only requires 2 bits per register.
743	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
744	* 1. the incoming state from the following call,
745	* 2. the outgoing state for this call,
746	* 3. mask of the entries set in the 2nd.
747	*
748	* The mask entry (3rd one above) will be used both when updating the outgoing
749	* state and when merging in incoming state for registers not touched by the
750	* current call.
751	*
752	*
753	* Extended Layout:
754	*
755	* The extended layout variation differs from the above as it records the
756	* different register accesses as individual bits, and it is currently used for
757	* the delayed EFLAGS calculation experiments. The latter means that
758	* calls/tb-exits and potential calls/exceptions/tb-exits are recorded
759	* separately so the latter can be checked for in combination with clobbering.
760	*
761	* @{ */
762	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
763	/** The register will be clobbered and the current value thrown away.
764	*
765	* When this is applied to the state (2) we'll simply be AND'ing it with the
766	* (old) mask (3) and adding the register to the mask. This way we'll
767	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
768	* IEMLIVENESS_STATE_INPUT states. */
769	# define IEMLIVENESS_STATE_CLOBBERED 0
770	/** The register is unused in the remainder of the TB.
771	*
772	* This is an initial state and can not be set by any of the
773	* iemNativeLivenessFunc_xxxx callbacks. */
774	# define IEMLIVENESS_STATE_UNUSED 1
775	/** The register value is required in a potential call or exception.
776	*
777	* This means that the register value must be calculated and is best written to
778	* the state, but that any shadowing registers can be flushed thereafter as it's
779	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
780	*
781	* It is typically applied across the board, but we preserve incoming
782	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
783	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
784	* 1. r0 = old & ~mask;
785	* 2. r0 = t1 & (t1 >> 1);
786	* 3. state \|= r0 \| 0b10;
787	* 4. mask = ~0;
788	*/
789	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
790	/** The register value is used as input.
791	*
792	* This means that the register value must be calculated and it is best to keep
793	* it in a register. It does not need to be writtent out as such. This is the
794	* highest priority state.
795	*
796	* Whether the call modifies the register or not isn't relevant to earlier
797	* calls, so that's not recorded.
798	*
799	* When applying this state we just or in the value in the outgoing state and
800	* mask. */
801	# define IEMLIVENESS_STATE_INPUT 3
802	/** Mask of the state bits. */
803	# define IEMLIVENESS_STATE_MASK 3
804	/** The number of bits per state. */
805	# define IEMLIVENESS_STATE_BIT_COUNT 2
806
807	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state.
808	* @note only used in assertions. */
809	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
810	/** Check if we're expecting read accesses to a register with the given (previous) liveness state.
811	* @note only used in assertions. */
812	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
813	/** Check if a register clobbering is expected given the (previous) liveness state.
814	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
815	* include INPUT if the register is used in more than one place.
816	* @note only used in assertions. */
817	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
818
819	/** Check if all status flags are going to be clobbered and doesn't need
820	* calculating in the current step.
821	* @param a_pCurEntry The current liveness entry.
822	* @note Used by actual code. */
823	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
824	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
825
826	/** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
827	* states, as these are no longer needed.
828	* @param a_pCurEntry The current liveness entry.
829	* @note Used by actual code. */
830	AssertCompile(IEMLIVENESS_STATE_UNUSED == 1 && IEMLIVENESS_STATE_XCPT_OR_CALL == 2);
831	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
832	( (a_pCurEntry)->Bit0.bm64 ^ (a_pCurEntry)->Bit1.bm64 )
833
834
835	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
836	/** The register is not used any more. */
837	# define IEMLIVENESS_STATE_UNUSED 0
838	/** Flag: The register is required in a potential call or/and exception. */
839	# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
840	# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
841	/** Flag: The register is read. */
842	# define IEMLIVENESS_STATE_READ 2
843	# define IEMLIVENESS_BIT_READ 1
844	/** Flag: The register is written. */
845	# define IEMLIVENESS_STATE_WRITE 4
846	# define IEMLIVENESS_BIT_WRITE 2
847	/** Flag: Unconditional call. */
848	# define IEMLIVENESS_STATE_CALL 8
849	# define IEMLIVENESS_BIT_CALL 3
850
851	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
852	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
853	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
854	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
855
856	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
857	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
858	&& !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
859	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
860	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
861	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
862
863	/** Construct a mask of the registers not in the read or write state.
864	* @note We could skips writes, if they aren't from us, as this is just a hack
865	* to prevent trashing registers that have just been written or will be
866	* written when we retire the current instruction.
867	* @param a_pCurEntry The current liveness entry.
868	* @note Used by actual code. */
869	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
870	( ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
871	& ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
872	& IEMLIVENESSBIT_MASK )
873
874
875	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
876	/** @} */
877
878	/** @name Liveness helpers for builtin functions and similar.
879	*
880	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
881	* own set of manipulator macros for those.
882	*
883	* @{ */
884	/** Initializing the state as all unused. */
885	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
886	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
887	do { \
888	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
889	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
890	} while (0)
891	#else
892	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
893	do { \
894	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
895	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
896	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
897	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
898	} while (0)
899	#endif
900
901	/** Initializing the outgoing state with a potential xcpt or call state.
902	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT.
903	*
904	* @note Must invoke IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL when done!
905	*/
906	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
907	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
908	do { \
909	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
910	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
911	} while (0)
912	#else
913	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
914	do { \
915	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
916	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
917	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
918	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
919	} while (0)
920	#endif
921
922	/** Completes IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL after applying any
923	* other state modifications.
924	*/
925	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
926	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) ((void)0)
927	#else
928	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
929	do { \
930	uint64_t const fInhMask = ~( (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL].bm64 \
931	\| (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE].bm64); \
932	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 & fInhMask; \
933	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64 & fInhMask; \
934	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & fInhMask; \
935	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_CALL].bm64 & fInhMask; \
936	} while (0)
937	#endif
938
939	/** Initializing the outgoing state with an unconditional call state.
940	* This should only really be used alone. */
941	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
942	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
943	do { \
944	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
945	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
946	} while (0)
947	#else
948	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
949	do { \
950	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
951	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
952	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
953	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
954	RT_NOREF(a_pIncoming); \
955	} while (0)
956	#endif
957
958	#if 0 /* unused */
959	/** Initializing the outgoing state with an unconditional call state as well as
960	* an potential call/exception preceeding it.
961	* This should only really be used alone. */
962	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
963	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
964	do { \
965	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
966	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
967	} while (0)
968	#else
969	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
970	do { \
971	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
972	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
973	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
974	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
975	} while (0)
976	#endif
977	#endif
978
979	/** Adds a segment base register as input to the outgoing state. */
980	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
981	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
982	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
983	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
984	} while (0)
985	#else
986	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
987	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
988	} while (0)
989	#endif
990
991	/** Adds a segment attribute register as input to the outgoing state. */
992	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
993	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
994	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
995	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
996	} while (0)
997	#else
998	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
999	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1000	} while (0)
1001	#endif
1002
1003	/** Adds a segment limit register as input to the outgoing state. */
1004	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1005	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1006	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1007	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1008	} while (0)
1009	#else
1010	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1011	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
1012	} while (0)
1013	#endif
1014
1015	/** Adds a segment limit register as input to the outgoing state. */
1016	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1017	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1018	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
1019	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
1020	} while (0)
1021	#else
1022	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1023	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
1024	} while (0)
1025	#endif
1026	/** @} */
1027
1028	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
1029	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
1030	* calculated and up to date. This is to double check that we haven't skipped
1031	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
1032	* @note has to be placed in
1033	*/
1034	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1035	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
1036	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
1037	#else
1038	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
1039	#endif
1040
1041
1042	/**
1043	* Guest registers that can be shadowed in GPRs.
1044	*
1045	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
1046	* must be placed last, as the liveness state tracks it as 7 subcomponents and
1047	* we don't want to waste space here.
1048	*
1049	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
1050	* friends as well as IEMAllN8veLiveness.cpp.
1051	*/
1052	typedef enum IEMNATIVEGSTREG : uint8_t
1053	{
1054	kIemNativeGstReg_GprFirst = 0,
1055	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
1056	kIemNativeGstReg_Cr0,
1057	kIemNativeGstReg_Cr4,
1058	kIemNativeGstReg_FpuFcw,
1059	kIemNativeGstReg_FpuFsw,
1060	kIemNativeGstReg_SegBaseFirst,
1061	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1062	kIemNativeGstReg_SegAttribFirst,
1063	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1064	kIemNativeGstReg_SegLimitFirst,
1065	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1066	kIemNativeGstReg_SegSelFirst,
1067	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1068	kIemNativeGstReg_Xcr0,
1069	kIemNativeGstReg_MxCsr,
1070	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags. /
1071	/* 6 entry gap for liveness EFlags subdivisions. */
1072	kIemNativeGstReg_Pc = kIemNativeGstReg_EFlags + 7,
1073	kIemNativeGstReg_End
1074	} IEMNATIVEGSTREG;
1075	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1076	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1077	AssertCompile(RT_BIT_64(kIemNativeGstReg_Pc) - UINT64_C(1) == IEMLIVENESSBIT_MASK);
1078
1079	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1080	* @{ */
1081	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1082	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1083	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1084	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1085	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1086	/** @} */
1087
1088	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1089
1090	/**
1091	* Guest registers that can be shadowed in host SIMD registers.
1092	*
1093	* @todo r=aeichner Liveness tracking
1094	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1095	*/
1096	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1097	{
1098	kIemNativeGstSimdReg_SimdRegFirst = 0,
1099	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1100	kIemNativeGstSimdReg_End
1101	} IEMNATIVEGSTSIMDREG;
1102
1103	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1104	* @{ */
1105	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1106	/** @} */
1107
1108	/**
1109	* The Load/store size for a SIMD guest register.
1110	*/
1111	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1112	{
1113	/** Invalid size. */
1114	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1115	/** Loads the low 128-bit of a guest SIMD register. */
1116	kIemNativeGstSimdRegLdStSz_Low128,
1117	/** Loads the high 128-bit of a guest SIMD register. */
1118	kIemNativeGstSimdRegLdStSz_High128,
1119	/** Loads the whole 256-bits of a guest SIMD register. */
1120	kIemNativeGstSimdRegLdStSz_256,
1121	/** End value. */
1122	kIemNativeGstSimdRegLdStSz_End
1123	} IEMNATIVEGSTSIMDREGLDSTSZ;
1124
1125	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1126
1127	/**
1128	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1129	*/
1130	typedef enum IEMNATIVEGSTREGUSE
1131	{
1132	/** The usage is read-only, the register holding the guest register
1133	* shadow copy will not be modified by the caller. */
1134	kIemNativeGstRegUse_ReadOnly = 0,
1135	/** The caller will update the guest register (think: PC += cbInstr).
1136	* The guest shadow copy will follow the returned register. */
1137	kIemNativeGstRegUse_ForUpdate,
1138	/** The call will put an entirely new value in the guest register, so
1139	* if new register is allocate it will be returned uninitialized. */
1140	kIemNativeGstRegUse_ForFullWrite,
1141	/** The caller will use the guest register value as input in a calculation
1142	* and the host register will be modified.
1143	* This means that the returned host register will not be marked as a shadow
1144	* copy of the guest register. */
1145	kIemNativeGstRegUse_Calculation
1146	} IEMNATIVEGSTREGUSE;
1147
1148	/**
1149	* Guest registers (classes) that can be referenced.
1150	*/
1151	typedef enum IEMNATIVEGSTREGREF : uint8_t
1152	{
1153	kIemNativeGstRegRef_Invalid = 0,
1154	kIemNativeGstRegRef_Gpr,
1155	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1156	kIemNativeGstRegRef_EFlags,
1157	kIemNativeGstRegRef_MxCsr,
1158	kIemNativeGstRegRef_FpuReg,
1159	kIemNativeGstRegRef_MReg,
1160	kIemNativeGstRegRef_XReg,
1161	kIemNativeGstRegRef_X87,
1162	kIemNativeGstRegRef_XState,
1163	//kIemNativeGstRegRef_YReg, - doesn't work.
1164	kIemNativeGstRegRef_End
1165	} IEMNATIVEGSTREGREF;
1166
1167
1168	/** Variable kinds. */
1169	typedef enum IEMNATIVEVARKIND : uint8_t
1170	{
1171	/** Customary invalid zero value. */
1172	kIemNativeVarKind_Invalid = 0,
1173	/** This is either in a register or on the stack. */
1174	kIemNativeVarKind_Stack,
1175	/** Immediate value - loaded into register when needed, or can live on the
1176	* stack if referenced (in theory). */
1177	kIemNativeVarKind_Immediate,
1178	/** Variable reference - loaded into register when needed, never stack. */
1179	kIemNativeVarKind_VarRef,
1180	/** Guest register reference - loaded into register when needed, never stack. */
1181	kIemNativeVarKind_GstRegRef,
1182	/** End of valid values. */
1183	kIemNativeVarKind_End
1184	} IEMNATIVEVARKIND;
1185
1186
1187	/** Variable or argument. */
1188	typedef struct IEMNATIVEVAR
1189	{
1190	/** The kind of variable. */
1191	IEMNATIVEVARKIND enmKind;
1192	/** The variable size in bytes. */
1193	uint8_t cbVar;
1194	/** The first stack slot (uint64_t), except for immediate and references
1195	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1196	* has a stack slot it has been initialized and has a value. Unused variables
1197	* has neither a stack slot nor a host register assignment. */
1198	uint8_t idxStackSlot;
1199	/** The host register allocated for the variable, UINT8_MAX if not. */
1200	uint8_t idxReg;
1201	/** The argument number if argument, UINT8_MAX if regular variable. */
1202	uint8_t uArgNo;
1203	/** If referenced, the index (unpacked) of the variable referencing this one,
1204	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1205	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1206	uint8_t idxReferrerVar;
1207	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1208	* @todo not sure what this really is for... */
1209	IEMNATIVEGSTREG enmGstReg;
1210	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1211	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1212	* only valid when idxReg is not UINT8_MAX. */
1213	bool fSimdReg : 1;
1214	/** Set if the registered is currently used exclusively, false if the
1215	* variable is idle and the register can be grabbed. */
1216	bool fRegAcquired : 1;
1217	#else
1218	/** Set if the registered is currently used exclusively, false if the
1219	* variable is idle and the register can be grabbed. */
1220	bool fRegAcquired;
1221	#endif
1222
1223	union
1224	{
1225	/** kIemNativeVarKind_Immediate: The immediate value. */
1226	uint64_t uValue;
1227	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1228	uint8_t idxRefVar;
1229	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1230	struct
1231	{
1232	/** The class of register. */
1233	IEMNATIVEGSTREGREF enmClass;
1234	/** Index within the class. */
1235	uint8_t idx;
1236	} GstRegRef;
1237	} u;
1238	} IEMNATIVEVAR;
1239	/** Pointer to a variable or argument. */
1240	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1241	/** Pointer to a const variable or argument. */
1242	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1243
1244	/** What is being kept in a host register. */
1245	typedef enum IEMNATIVEWHAT : uint8_t
1246	{
1247	/** The traditional invalid zero value. */
1248	kIemNativeWhat_Invalid = 0,
1249	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1250	kIemNativeWhat_Var,
1251	/** Temporary register, this is typically freed when a MC completes. */
1252	kIemNativeWhat_Tmp,
1253	/** Call argument w/o a variable mapping. This is free (via
1254	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1255	kIemNativeWhat_Arg,
1256	/** Return status code.
1257	* @todo not sure if we need this... */
1258	kIemNativeWhat_rc,
1259	/** The fixed pVCpu (PVMCPUCC) register.
1260	* @todo consider offsetting this on amd64 to use negative offsets to access
1261	* more members using 8-byte disp. */
1262	kIemNativeWhat_pVCpuFixed,
1263	/** The fixed pCtx (PCPUMCTX) register.
1264	* @todo consider offsetting this on amd64 to use negative offsets to access
1265	* more members using 8-byte disp. */
1266	kIemNativeWhat_pCtxFixed,
1267	/** Fixed temporary register. */
1268	kIemNativeWhat_FixedTmp,
1269	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1270	/** Shadow RIP for the delayed RIP updating debugging. */
1271	kIemNativeWhat_PcShadow,
1272	#endif
1273	/** Register reserved by the CPU or OS architecture. */
1274	kIemNativeWhat_FixedReserved,
1275	/** End of valid values. */
1276	kIemNativeWhat_End
1277	} IEMNATIVEWHAT;
1278
1279	/**
1280	* Host general register entry.
1281	*
1282	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1283	*
1284	* @todo Track immediate values in host registers similarlly to how we track the
1285	* guest register shadow copies. For it to be real helpful, though,
1286	* we probably need to know which will be reused and put them into
1287	* non-volatile registers, otherwise it's going to be more or less
1288	* restricted to an instruction or two.
1289	*/
1290	typedef struct IEMNATIVEHSTREG
1291	{
1292	/** Set of guest registers this one shadows.
1293	*
1294	* Using a bitmap here so we can designate the same host register as a copy
1295	* for more than one guest register. This is expected to be useful in
1296	* situations where one value is copied to several registers in a sequence.
1297	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1298	* sequence we'd want to let this register follow to be a copy of and there
1299	* will always be places where we'd be picking the wrong one.
1300	*/
1301	uint64_t fGstRegShadows;
1302	/** What is being kept in this register. */
1303	IEMNATIVEWHAT enmWhat;
1304	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1305	uint8_t idxVar;
1306	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1307	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1308	* that scope. */
1309	uint8_t idxStackSlot;
1310	/** Alignment padding. */
1311	uint8_t abAlign[5];
1312	} IEMNATIVEHSTREG;
1313
1314
1315	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1316	/**
1317	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1318	* halves, on architectures where there is no 256-bit register available this entry will track
1319	* two adjacent 128-bit host registers.
1320	*
1321	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1322	*/
1323	typedef struct IEMNATIVEHSTSIMDREG
1324	{
1325	/** Set of guest registers this one shadows.
1326	*
1327	* Using a bitmap here so we can designate the same host register as a copy
1328	* for more than one guest register. This is expected to be useful in
1329	* situations where one value is copied to several registers in a sequence.
1330	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1331	* sequence we'd want to let this register follow to be a copy of and there
1332	* will always be places where we'd be picking the wrong one.
1333	*/
1334	uint64_t fGstRegShadows;
1335	/** What is being kept in this register. */
1336	IEMNATIVEWHAT enmWhat;
1337	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1338	uint8_t idxVar;
1339	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1340	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1341	/** Alignment padding. */
1342	uint8_t abAlign[5];
1343	} IEMNATIVEHSTSIMDREG;
1344	#endif
1345
1346
1347	/**
1348	* Core state for the native recompiler, that is, things that needs careful
1349	* handling when dealing with branches.
1350	*/
1351	typedef struct IEMNATIVECORESTATE
1352	{
1353	/** Allocation bitmap for aHstRegs. */
1354	uint32_t bmHstRegs;
1355
1356	/** Bitmap marking which host register contains guest register shadow copies.
1357	* This is used during register allocation to try preserve copies. */
1358	uint32_t bmHstRegsWithGstShadow;
1359	/** Bitmap marking valid entries in aidxGstRegShadows. */
1360	uint64_t bmGstRegShadows;
1361	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1362	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1363	uint64_t bmGstRegShadowDirty;
1364	#endif
1365
1366	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1367	/** The current instruction offset in bytes from when the guest program counter
1368	* was updated last. Used for delaying the write to the guest context program counter
1369	* as long as possible. */
1370	int64_t offPc;
1371	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1372	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1373	bool fDebugPcInitialized;
1374	# endif
1375	#endif
1376
1377	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1378	/** Allocation bitmap for aHstSimdRegs. */
1379	uint32_t bmHstSimdRegs;
1380
1381	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1382	* This is used during register allocation to try preserve copies. */
1383	uint32_t bmHstSimdRegsWithGstShadow;
1384	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1385	uint64_t bmGstSimdRegShadows;
1386	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1387	uint64_t bmGstSimdRegShadowDirtyLo128;
1388	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1389	uint64_t bmGstSimdRegShadowDirtyHi128;
1390	#endif
1391
1392	union
1393	{
1394	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1395	uint8_t aidxArgVars[8];
1396	/** For more efficient resetting. */
1397	uint64_t u64ArgVars;
1398	};
1399
1400	/** Allocation bitmap for the stack. */
1401	uint32_t bmStack;
1402	/** Allocation bitmap for aVars. */
1403	uint32_t bmVars;
1404
1405	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1406	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1407	* (A shadow copy of a guest register can only be held in a one host register,
1408	* there are no duplicate copies or ambiguities like that). */
1409	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1410	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1411	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1412	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1413	* (A shadow copy of a guest register can only be held in a one host register,
1414	* there are no duplicate copies or ambiguities like that). */
1415	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1416	#endif
1417
1418	/** Host register allocation tracking. */
1419	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1420	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1421	/** Host SIMD register allocation tracking. */
1422	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1423	#endif
1424
1425	/** Variables and arguments. */
1426	IEMNATIVEVAR aVars[9];
1427	} IEMNATIVECORESTATE;
1428	/** Pointer to core state. */
1429	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1430	/** Pointer to const core state. */
1431	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1432
1433	/** @def IEMNATIVE_VAR_IDX_UNPACK
1434	* @returns Index into IEMNATIVECORESTATE::aVars.
1435	* @param a_idxVar Variable index w/ magic (in strict builds).
1436	*/
1437	/** @def IEMNATIVE_VAR_IDX_PACK
1438	* @returns Variable index w/ magic (in strict builds).
1439	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1440	*/
1441	#ifdef VBOX_STRICT
1442	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1443	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1444	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1445	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1446	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1447	#else
1448	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1449	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1450	#endif
1451
1452
1453	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1454	/** Clear the dirty state of the given guest SIMD register. */
1455	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1456	do { \
1457	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1458	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1459	} while (0)
1460
1461	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1462	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1463	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1464	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1465	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1466	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1467	/** Returns whether the given guest SIMD register is dirty. */
1468	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1469	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1470
1471	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1472	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1473	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1474	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1475	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1476	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1477
1478	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1479	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1480	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1481	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1482	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1483	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1484	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1485	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1486	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1487	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1488	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1489	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1490	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1491	# endif
1492	#endif
1493
1494
1495	/**
1496	* Conditional stack entry.
1497	*/
1498	typedef struct IEMNATIVECOND
1499	{
1500	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1501	bool fInElse;
1502	union
1503	{
1504	RT_GCC_EXTENSION struct
1505	{
1506	/** Set if the if-block unconditionally exited the TB. */
1507	bool fIfExitTb;
1508	/** Set if the else-block unconditionally exited the TB. */
1509	bool fElseExitTb;
1510	};
1511	/** Indexed by fInElse. */
1512	bool afExitTb[2];
1513	};
1514	bool afPadding[5];
1515	/** The label for the IEM_MC_ELSE. */
1516	uint32_t idxLabelElse;
1517	/** The label for the IEM_MC_ENDIF. */
1518	uint32_t idxLabelEndIf;
1519	/** The initial state snapshot as the if-block starts executing. */
1520	IEMNATIVECORESTATE InitialState;
1521	/** The state snapshot at the end of the if-block. */
1522	IEMNATIVECORESTATE IfFinalState;
1523	} IEMNATIVECOND;
1524	/** Pointer to a condition stack entry. */
1525	typedef IEMNATIVECOND *PIEMNATIVECOND;
1526
1527
1528	/**
1529	* Native recompiler state.
1530	*/
1531	typedef struct IEMRECOMPILERSTATE
1532	{
1533	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1534	* IEMNATIVEINSTR units. */
1535	uint32_t cInstrBufAlloc;
1536	#ifdef VBOX_STRICT
1537	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1538	uint32_t offInstrBufChecked;
1539	#else
1540	uint32_t uPadding1; /* We don't keep track of the size here... */
1541	#endif
1542	/** Fixed temporary code buffer for native recompilation. */
1543	PIEMNATIVEINSTR pInstrBuf;
1544
1545	/** Bitmaps with the label types used. */
1546	uint64_t bmLabelTypes;
1547	/** Actual number of labels in paLabels. */
1548	uint32_t cLabels;
1549	/** Max number of entries allowed in paLabels before reallocating it. */
1550	uint32_t cLabelsAlloc;
1551	/** Labels defined while recompiling (referenced by fixups). */
1552	PIEMNATIVELABEL paLabels;
1553	/** Array with indexes of unique labels (uData always 0). */
1554	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1555
1556	/** Actual number of fixups paFixups. */
1557	uint32_t cFixups;
1558	/** Max number of entries allowed in paFixups before reallocating it. */
1559	uint32_t cFixupsAlloc;
1560	/** Buffer used by the recompiler for recording fixups when generating code. */
1561	PIEMNATIVEFIXUP paFixups;
1562
1563	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1564	/** Actual number of fixups in paTbExitFixups. */
1565	uint32_t cTbExitFixups;
1566	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1567	uint32_t cTbExitFixupsAlloc;
1568	/** Buffer used by the recompiler for recording fixups when generating code. */
1569	PIEMNATIVEEXITFIXUP paTbExitFixups;
1570	#endif
1571
1572	#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1573	/** Statistics: The idxInstr+1 value at the last PC update. */
1574	uint8_t idxInstrPlusOneOfLastPcUpdate;
1575	#endif
1576
1577	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1578	/** Number of debug info entries allocated for pDbgInfo. */
1579	uint32_t cDbgInfoAlloc;
1580	/** Debug info. */
1581	PIEMTBDBG pDbgInfo;
1582	#endif
1583
1584	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1585	/** The current call index (liveness array and threaded calls in TB). */
1586	uint32_t idxCurCall;
1587	/** Number of liveness entries allocated. */
1588	uint32_t cLivenessEntriesAlloc;
1589	/** Liveness entries for all the calls in the TB begin recompiled.
1590	* The entry for idxCurCall contains the info for what the next call will
1591	* require wrt registers. (Which means the last entry is the initial liveness
1592	* state.) */
1593	PIEMLIVENESSENTRY paLivenessEntries;
1594	#endif
1595
1596	/** The translation block being recompiled. */
1597	PCIEMTB pTbOrg;
1598	/** The VMCPU structure of the EMT. */
1599	PVMCPUCC pVCpu;
1600
1601	/** Condition sequence number (for generating unique labels). */
1602	uint16_t uCondSeqNo;
1603	/** Check IRQ sequence number (for generating unique labels). */
1604	uint16_t uCheckIrqSeqNo;
1605	/** TLB load sequence number (for generating unique labels). */
1606	uint16_t uTlbSeqNo;
1607	/** The current condition stack depth (aCondStack). */
1608	uint8_t cCondDepth;
1609
1610	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1611	uint8_t cArgsX;
1612	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1613	uint32_t fCImpl;
1614	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1615	uint32_t fMc;
1616	/** The expected IEMCPU::fExec value for the current call/instruction. */
1617	uint32_t fExec;
1618	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1619	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1620	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1621	*
1622	* This is an optimization because these control registers can only be changed from
1623	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1624	* consisting of multiple SIMD instructions.
1625	*/
1626	uint32_t fSimdRaiseXcptChecksEmitted;
1627	#endif
1628	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1629	uint32_t idxLastCheckIrqCallNo;
1630
1631	/** Core state requiring care with branches. */
1632	IEMNATIVECORESTATE Core;
1633
1634	/** The condition nesting stack. */
1635	IEMNATIVECOND aCondStack[2];
1636
1637	#ifndef IEM_WITH_THROW_CATCH
1638	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1639	* for recompilation error handling. */
1640	jmp_buf JmpBuf;
1641	#endif
1642	} IEMRECOMPILERSTATE;
1643	/** Pointer to a native recompiler state. */
1644	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1645
1646
1647	/** @def IEMNATIVE_TRY_SETJMP
1648	* Wrapper around setjmp / try, hiding all the ugly differences.
1649	*
1650	* @note Use with extreme care as this is a fragile macro.
1651	* @param a_pReNative The native recompile state.
1652	* @param a_rcTarget The variable that should receive the status code in case
1653	* of a longjmp/throw.
1654	*/
1655	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1656	* Start wrapper for catch / setjmp-else.
1657	*
1658	* This will set up a scope.
1659	*
1660	* @note Use with extreme care as this is a fragile macro.
1661	* @param a_pReNative The native recompile state.
1662	* @param a_rcTarget The variable that should receive the status code in case
1663	* of a longjmp/throw.
1664	*/
1665	/** @def IEMNATIVE_CATCH_LONGJMP_END
1666	* End wrapper for catch / setjmp-else.
1667	*
1668	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1669	* up the state.
1670	*
1671	* @note Use with extreme care as this is a fragile macro.
1672	* @param a_pReNative The native recompile state.
1673	*/
1674	/** @def IEMNATIVE_DO_LONGJMP
1675	*
1676	* Wrapper around longjmp / throw.
1677	*
1678	* @param a_pReNative The native recompile state.
1679	* @param a_rc The status code jump back with / throw.
1680	*/
1681	#ifdef IEM_WITH_THROW_CATCH
1682	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1683	a_rcTarget = VINF_SUCCESS; \
1684	try
1685	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1686	catch (int rcThrown) \
1687	{ \
1688	a_rcTarget = rcThrown
1689	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1690	} \
1691	((void)0)
1692	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1693	#else /* !IEM_WITH_THROW_CATCH */
1694	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1695	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1696	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1697	else \
1698	{ \
1699	((void)0)
1700	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1701	}
1702	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1703	#endif /* !IEM_WITH_THROW_CATCH */
1704
1705
1706	/**
1707	* Native recompiler worker for a threaded function.
1708	*
1709	* @returns New code buffer offset; throws VBox status code in case of a failure.
1710	* @param pReNative The native recompiler state.
1711	* @param off The current code buffer offset.
1712	* @param pCallEntry The threaded call entry.
1713	*
1714	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1715	*/
1716	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1717	/** Pointer to a native recompiler worker for a threaded function. */
1718	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1719
1720	/** Defines a native recompiler worker for a threaded function.
1721	* @see FNIEMNATIVERECOMPFUNC */
1722	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1723	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1724
1725	/** Prototypes a native recompiler function for a threaded function.
1726	* @see FNIEMNATIVERECOMPFUNC */
1727	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1728
1729
1730	/**
1731	* Native recompiler liveness analysis worker for a threaded function.
1732	*
1733	* @param pCallEntry The threaded call entry.
1734	* @param pIncoming The incoming liveness state entry.
1735	* @param pOutgoing The outgoing liveness state entry.
1736	*/
1737	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1738	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1739	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1740	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1741
1742	/** Defines a native recompiler liveness analysis worker for a threaded function.
1743	* @see FNIEMNATIVELIVENESSFUNC */
1744	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1745	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1746
1747	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1748	* @see FNIEMNATIVELIVENESSFUNC */
1749	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1750
1751
1752	/** Define a native recompiler helper function, safe to call from the TB code. */
1753	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1754	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1755	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1756	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1757	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1758	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1759	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1760	a_RetType (VBOXCALL *a_Name) a_ArgList
1761
1762
1763	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1764	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1765	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1766	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1767	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1768	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1769	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1770	uint8_t idxHstSimdReg = UINT8_MAX,
1771	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1772	# endif
1773	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1774	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1775	uint8_t idxGstReg, uint8_t idxHstReg);
1776	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1777	uint64_t fGstReg);
1778	# endif
1779	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1780	uint64_t offPc, uint32_t cInstrSkipped);
1781	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1782
1783	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1784	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1785	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1786	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1787	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1788	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1789	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1790	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1791	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1792	#endif
1793	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1794
1795	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1796	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1797	bool fPreferVolatile = true);
1798	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1799	bool fPreferVolatile = true);
1800	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1801	IEMNATIVEGSTREG enmGstReg,
1802	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1803	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1804	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1805	IEMNATIVEGSTREG enmGstReg);
1806
1807	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1808	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1809	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1810	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1811	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1812	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1813	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1814	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1815	# endif
1816	#endif
1817	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1818	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1819	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1820	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1821	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1822	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1823	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1824	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1825	# endif
1826	#endif
1827	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1828	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1829	uint32_t fKeepVars = 0);
1830	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1831	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1832	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1833	uint32_t fHstRegsActiveShadows);
1834	#ifdef VBOX_STRICT
1835	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1836	#endif
1837	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1838	uint64_t fGstSimdShwExcept);
1839	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1840	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1841	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1842	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
1843	# endif
1844	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1845	#endif
1846	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1847	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1848	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1849	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
1850	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1851	uint64_t fFlushGstReg = UINT64_MAX);
1852	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
1853	uint32_t off, uint8_t idxHstReg);
1854	#endif
1855
1856
1857	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1858	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1859	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1860	bool fPreferVolatile = true);
1861	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1862	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1863	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1864	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1865	bool fNoVolatileRegs = false);
1866	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1867	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1868	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1869	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1870	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1871	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1872	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1873	#endif
1874
1875	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1876	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1877	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1878	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1879	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1880	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1881	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1882	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1883	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1884	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1885	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1886	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1887	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1888	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1889	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1890	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1891	#endif
1892	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1893	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1894	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1895	uint32_t fHstRegsNotToSave);
1896	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1897	uint32_t fHstRegsNotToSave);
1898	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1899	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1900
1901	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1902	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1903	#ifdef VBOX_STRICT
1904	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1905	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1906	IEMNATIVEGSTREG enmGstReg);
1907	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1908	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1909	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1910	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1911	# endif
1912	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1913	#endif
1914	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1915	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1916	#endif
1917	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1918	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1919	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1920	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1921	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1922	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1923	PCIEMTHRDEDCALLENTRY pCallEntry);
1924	IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(iemNativeLivenessFunc_ThreadedCall);
1925	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1926	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1927
1928
1929	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1930	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1931	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1932	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1933	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1934	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1935	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1936	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1937	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1938	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1939	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1940	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1941	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1942
1943	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1944	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1945	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1946	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1947	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1948	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1949	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1950	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1951	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1952	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1953	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1954	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1955	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1956	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1957	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1958	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1959	#endif
1960	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1961	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1962	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1963	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1964	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1965	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1966	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1967	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1968	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1969	#endif
1970	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1971	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1972	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1973	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1974	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1975	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1976	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1977
1978	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1979	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1980	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1981	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1982	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1983	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1984	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1985	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1986	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1987	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1988	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1989	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1990	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1991	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1992	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1993	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1994	#endif
1995	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1996	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1997	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1998	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1999	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2000	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2001	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2002	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2003	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2004	#endif
2005	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2006	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2007	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2008	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2009	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2010	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2011	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2012
2013	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2014	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2015	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2016	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2017	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2018	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2019	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2020	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2021	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2022	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2023	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2024	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2025	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2026	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2027	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2028	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2029	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2030	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2031	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2032	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2033	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2034	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2035
2036	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2037	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2038	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2039	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2040	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2041	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2042	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2043	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2044	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2045	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2046	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2047	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2048	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2049	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2050	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2051	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2052	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2053	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2054	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2055	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2056	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2057	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2058
2059	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2060	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2061	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2062	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2063
2064
2065	/**
2066	* Info about shadowed guest register values.
2067	* @see IEMNATIVEGSTREG
2068	*/
2069	typedef struct IEMANTIVEGSTREGINFO
2070	{
2071	/** Offset in VMCPU. */
2072	uint32_t off;
2073	/** The field size. */
2074	uint8_t cb;
2075	/** Name (for logging). */
2076	const char *pszName;
2077	} IEMANTIVEGSTREGINFO;
2078	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2079	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2080	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2081	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2082	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2083
2084
2085
2086	/**
2087	* Ensures that there is sufficient space in the instruction output buffer.
2088	*
2089	* This will reallocate the buffer if needed and allowed.
2090	*
2091	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2092	* allocation size.
2093	*
2094	* @returns Pointer to the instruction output buffer on success; throws VBox
2095	* status code on failure, so no need to check it.
2096	* @param pReNative The native recompile state.
2097	* @param off Current instruction offset. Works safely for UINT32_MAX
2098	* as well.
2099	* @param cInstrReq Number of instruction about to be added. It's okay to
2100	* overestimate this a bit.
2101	*/
2102	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2103	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2104	{
2105	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2106	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2107	{
2108	#ifdef VBOX_STRICT
2109	pReNative->offInstrBufChecked = offChecked;
2110	#endif
2111	return pReNative->pInstrBuf;
2112	}
2113	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2114	}
2115
2116	/**
2117	* Checks that we didn't exceed the space requested in the last
2118	* iemNativeInstrBufEnsure() call.
2119	*/
2120	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2121	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2122	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2123
2124	/**
2125	* Checks that a variable index is valid.
2126	*/
2127	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2128	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2129	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2130	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2131	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2132	("%s=%#x\n", #a_idxVar, a_idxVar))
2133	#else
2134	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2135	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2136	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2137	#endif
2138
2139	/**
2140	* Checks that a variable index is valid and that the variable is assigned the
2141	* correct argument number.
2142	* This also adds a RT_NOREF of a_idxVar.
2143	*/
2144	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2145	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2146	RT_NOREF_PV(a_idxVar); \
2147	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2148	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2149	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2150	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2151	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2152	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2153	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2154	a_uArgNo)); \
2155	} while (0)
2156	#else
2157	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2158	RT_NOREF_PV(a_idxVar); \
2159	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2160	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2161	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2162	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2163	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2164	} while (0)
2165	#endif
2166
2167
2168	/**
2169	* Checks that a variable has the expected size.
2170	*/
2171	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2172	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2173	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2174	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2175
2176
2177	/**
2178	* Calculates the stack address of a variable as a [r]BP displacement value.
2179	*/
2180	DECL_FORCE_INLINE(int32_t)
2181	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2182	{
2183	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2184	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2185	}
2186
2187
2188	/**
2189	* Releases the variable's register.
2190	*
2191	* The register must have been previously acquired calling
2192	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2193	* iemNativeVarRegisterSetAndAcquire().
2194	*/
2195	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2196	{
2197	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2198	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2199	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2200	}
2201
2202
2203	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2204	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2205	{
2206	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2207	iemNativeVarRegisterRelease(pReNative, idxVar);
2208	}
2209	#endif
2210
2211
2212	/**
2213	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2214	*
2215	* @returns The flush mask.
2216	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2217	* @param fGstShwFlush The starting flush mask.
2218	*/
2219	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2220	{
2221	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2222	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2223	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2224	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2225	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2226	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2227	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2228	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2229	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2230	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2231	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2232	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2233	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2234	return fGstShwFlush;
2235	}
2236
2237
2238	/** Number of hidden arguments for CIMPL calls.
2239	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2240	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2241	# define IEM_CIMPL_HIDDEN_ARGS 3
2242	#else
2243	# define IEM_CIMPL_HIDDEN_ARGS 2
2244	#endif
2245
2246
2247	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2248	/** Number of hidden arguments for SSE_AIMPL calls. */
2249	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2250	/** Number of hidden arguments for AVX_AIMPL calls. */
2251	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2252	#endif
2253
2254
2255	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2256
2257	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2258	/**
2259	* Helper for iemNativeLivenessGetStateByGstReg.
2260	*
2261	* @returns IEMLIVENESS_STATE_XXX
2262	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2263	* ORed together.
2264	*/
2265	DECL_FORCE_INLINE(uint32_t)
2266	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2267	{
2268	/* INPUT trumps anything else. */
2269	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2270	return IEMLIVENESS_STATE_INPUT;
2271
2272	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2273	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2274	{
2275	/* If not all sub-fields are clobbered they must be considered INPUT. */
2276	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2277	return IEMLIVENESS_STATE_INPUT;
2278	return IEMLIVENESS_STATE_CLOBBERED;
2279	}
2280
2281	/* XCPT_OR_CALL trumps UNUSED. */
2282	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2283	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2284
2285	return IEMLIVENESS_STATE_UNUSED;
2286	}
2287	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2288
2289
2290	DECL_FORCE_INLINE(uint32_t)
2291	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2292	{
2293	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2294	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2295	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2296	# else
2297	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2298	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2299	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2300	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2301	# endif
2302	}
2303
2304
2305	DECL_FORCE_INLINE(uint32_t)
2306	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2307	{
2308	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2309	if (enmGstReg == kIemNativeGstReg_EFlags)
2310	{
2311	/* Merge the eflags states to one. */
2312	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2313	uRet = RT_BIT_32(uRet);
2314	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2315	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2316	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2317	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2318	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2319	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2320	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2321	# else
2322	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2323	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2324	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2325	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2326	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2327	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2328	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2329	# endif
2330	}
2331	return uRet;
2332	}
2333
2334
2335	# ifdef VBOX_STRICT
2336	/** For assertions only - caller checks that idxCurCall isn't zero. */
2337	DECL_FORCE_INLINE(uint32_t)
2338	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2339	{
2340	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2341	}
2342	# endif /* VBOX_STRICT */
2343
2344	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2345
2346
2347	/**
2348	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2349	*/
2350	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2351	{
2352	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2353	return IEM_CIMPL_HIDDEN_ARGS;
2354	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2355	return 1;
2356	return 0;
2357	}
2358
2359
2360	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2361	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2362	{
2363	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2364
2365	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2366	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2367	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2368	return (uint8_t)idxReg;
2369	}
2370
2371
2372
2373	/*********************************************************************************************************************************
2374	* Register Allocator (GPR) *
2375	*********************************************************************************************************************************/
2376
2377	/**
2378	* Marks host register @a idxHstReg as containing a shadow copy of guest
2379	* register @a enmGstReg.
2380	*
2381	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2382	* host register before calling.
2383	*/
2384	DECL_FORCE_INLINE(void)
2385	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2386	{
2387	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2388	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2389	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2390
2391	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2392	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2393	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2394	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2395	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2396	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2397	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2398	#else
2399	RT_NOREF(off);
2400	#endif
2401	}
2402
2403
2404	/**
2405	* Clear any guest register shadow claims from @a idxHstReg.
2406	*
2407	* The register does not need to be shadowing any guest registers.
2408	*/
2409	DECL_FORCE_INLINE(void)
2410	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2411	{
2412	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2413	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2414	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2415	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2416	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2417	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2418	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2419	#endif
2420
2421	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2422	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2423	if (fGstRegs)
2424	{
2425	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2426	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2427	while (fGstRegs)
2428	{
2429	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2430	fGstRegs &= ~RT_BIT_64(iGstReg);
2431	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2432	}
2433	}
2434	#else
2435	RT_NOREF(off);
2436	#endif
2437
2438	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2439	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2440	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2441	}
2442
2443
2444	/**
2445	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2446	* and global overview flags.
2447	*/
2448	DECL_FORCE_INLINE(void)
2449	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2450	{
2451	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2452	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2453	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2454	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2455	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2456	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2457	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2458	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2459	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2460	#endif
2461
2462	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2463	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2464	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2465	#else
2466	RT_NOREF(off);
2467	#endif
2468
2469	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2470	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2471	if (!fGstRegShadowsNew)
2472	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2473	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2474	}
2475
2476
2477	#if 0 /* unused */
2478	/**
2479	* Clear any guest register shadow claim for @a enmGstReg.
2480	*/
2481	DECL_FORCE_INLINE(void)
2482	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2483	{
2484	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2485	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2486	{
2487	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2488	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2489	}
2490	}
2491	#endif
2492
2493
2494	/**
2495	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2496	* as the new shadow of it.
2497	*
2498	* Unlike the other guest reg shadow helpers, this does the logging for you.
2499	* However, it is the liveness state is not asserted here, the caller must do
2500	* that.
2501	*/
2502	DECL_FORCE_INLINE(void)
2503	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2504	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2505	{
2506	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2507	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2508	{
2509	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2510	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2511	if (idxHstRegOld == idxHstRegNew)
2512	return;
2513	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2514	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2515	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2516	}
2517	else
2518	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2519	g_aGstShadowInfo[enmGstReg].pszName));
2520	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2521	}
2522
2523
2524	/**
2525	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2526	* to @a idxRegTo.
2527	*/
2528	DECL_FORCE_INLINE(void)
2529	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2530	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2531	{
2532	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2533	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2534	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2535	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2536	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2537	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2538	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2539	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2540	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2541
2542	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2543	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2544	if (!fGstRegShadowsFrom)
2545	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2546	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2547	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2548	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2549	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2550	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2551	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2552	#else
2553	RT_NOREF(off);
2554	#endif
2555	}
2556
2557
2558	/**
2559	* Flushes any delayed guest register writes.
2560	*
2561	* This must be called prior to calling CImpl functions and any helpers that use
2562	* the guest state (like raising exceptions) and such.
2563	*
2564	* This optimization has not yet been implemented. The first target would be
2565	* RIP updates, since these are the most common ones.
2566	*
2567	* @note This function does not flush any shadowing information for guest
2568	* registers. This needs to be done by the caller if it wishes to do so.
2569	*/
2570	DECL_INLINE_THROW(uint32_t)
2571	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
2572	uint64_t fGstSimdShwExcept = 0)
2573	{
2574	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2575	uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
2576	#else
2577	uint64_t const fWritebackPc = 0;
2578	#endif
2579	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2580	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2581	#else
2582	uint64_t const bmGstRegShadowDirty = 0;
2583	#endif
2584	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2585	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
2586	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2587	& ~fGstSimdShwExcept;
2588	#else
2589	uint64_t const bmGstSimdRegShadowDirty = 0;
2590	#endif
2591	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2592	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2593
2594	return off;
2595	}
2596
2597
2598
2599	/*********************************************************************************************************************************
2600	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2601	*********************************************************************************************************************************/
2602
2603	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2604
2605	DECL_FORCE_INLINE(uint8_t)
2606	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2607	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2608	{
2609	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2610
2611	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2612	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2613	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2614	return idxSimdReg;
2615	}
2616
2617
2618	/**
2619	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2620	* SIMD register @a enmGstSimdReg.
2621	*
2622	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2623	* host register before calling.
2624	*/
2625	DECL_FORCE_INLINE(void)
2626	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2627	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2628	{
2629	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2630	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2631	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2632
2633	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2634	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2635	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2636	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2637	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2638	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2639	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2640	#else
2641	RT_NOREF(off);
2642	#endif
2643	}
2644
2645
2646	/**
2647	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2648	* to @a idxSimdRegTo.
2649	*/
2650	DECL_FORCE_INLINE(void)
2651	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2652	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2653	{
2654	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2655	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2656	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2657	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2658	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2659	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2660	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2661	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2662	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2663	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2664	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2665
2666	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2667	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2668	if (!fGstRegShadowsFrom)
2669	{
2670	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2671	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2672	}
2673	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2674	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2675	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2676	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2677	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2678	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2679	#else
2680	RT_NOREF(off);
2681	#endif
2682	}
2683
2684
2685	/**
2686	* Clear any guest register shadow claims from @a idxHstSimdReg.
2687	*
2688	* The register does not need to be shadowing any guest registers.
2689	*/
2690	DECL_FORCE_INLINE(void)
2691	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2692	{
2693	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2694	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2695	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2696	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2697	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2698	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2699	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2700
2701	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2702	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2703	if (fGstRegs)
2704	{
2705	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2706	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2707	while (fGstRegs)
2708	{
2709	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2710	fGstRegs &= ~RT_BIT_64(iGstReg);
2711	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2712	}
2713	}
2714	#else
2715	RT_NOREF(off);
2716	#endif
2717
2718	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2719	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2720	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2721	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2722	}
2723
2724	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2725
2726
2727	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2728	/**
2729	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2730	*/
2731	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2732	{
2733	if (pReNative->Core.offPc)
2734	return iemNativeEmitPcWritebackSlow(pReNative, off);
2735	return off;
2736	}
2737	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2738
2739
2740	#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2741	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2742	* it saves us the trouble of a hidden parameter on MSC/amd64. */
2743	# ifdef RT_ARCH_AMD64
2744	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2745	# elif defined(RT_ARCH_ARM64)
2746	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2747	# endif
2748	#endif
2749
2750	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2751	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2752	#endif
2753
2754	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2755
2756	/** @} */
2757
2758	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2759

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106099

Download in other formats: