IEMN8veRecompiler.h@ 107623

Last change on this file since 107623 was 107200, checked in by vboxsync, 3 months ago
VMM/IEM: Deal with hidden pointer to VBOXSTRICTRC return struct on win.arm64. jiraref:VBP-1466
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 164.5 KB

Line
1	/* $Id: IEMN8veRecompiler.h 107200 2024-11-29 22:15:46Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41	#include <VBox/cdefs.h> /* for VBOXSTRICTRC_STRICT_ENABLED */
42
43	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
44	* Enables generating internal debug info for better TB disassembly dumping. */
45	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
46	# define IEMNATIVE_WITH_TB_DEBUG_INFO
47	#endif
48
49	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	* Enables liveness analysis. */
51	#if 1 \|\| defined(DOXYGEN_RUNNING)
52	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62	* Enables strict consistency checks around EFLAGS skipping.
63	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65	# ifdef VBOX_STRICT
66	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67	# endif
68	#elif defined(DOXYGEN_RUNNING)
69	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70	#endif
71
72	/** @def IEMNATIVE_WITH_EFLAGS_POSTPONING
73	* Enables delaying EFLAGS calculations/updating to conditional code paths
74	* that are (hopefully) not taken so frequently.
75	*
76	* This can only help with case where there is an conditional
77	* call/exception/tbexit that needs the flag, but in the default code stream the
78	* flag will be clobbered. Useful for TlbMiss scenarios and sequences of memory
79	* based instructions clobbering status flags. */
80	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
81	# if 1 \|\| defined(DOXYGEN_RUNNING)
82	# define IEMNATIVE_WITH_EFLAGS_POSTPONING
83	# endif
84	#endif
85	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
86	# ifndef IEMNATIVE_WITH_EFLAGS_SKIPPING
87	# error "IEMNATIVE_WITH_EFLAGS_POSTPONING requires IEMNATIVE_WITH_EFLAGS_SKIPPING at present"
88	# endif
89	#endif
90
91	/** @def IEMLIVENESS_EXTENDED_LAYOUT
92	* Enables the extended liveness data layout. */
93	#if defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \|\| defined(DOXYGEN_RUNNING) \|\| 0
94	# define IEMLIVENESS_EXTENDED_LAYOUT
95	#endif
96
97
98	#ifdef VBOX_WITH_STATISTICS
99	/** Always count instructions for now. */
100	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
101	#endif
102
103
104	/** @name Stack Frame Layout
105	*
106	* @{ */
107	/** The size of the area for stack variables and spills and stuff.
108	* @note This limit is duplicated in the python script(s). We add 0x40 for
109	* alignment padding. */
110	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
111	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
112	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
113	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
114
115	#ifdef RT_ARCH_AMD64
116	/** An stack alignment adjustment (between non-volatile register pushes and
117	* the stack variable area, so the latter better aligned). */
118	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
119
120	/** Number of stack arguments slots for calls made from the frame. */
121	# ifdef RT_OS_WINDOWS
122	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
123	# else
124	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
125	# endif
126	/** Number of any shadow arguments (spill area) for calls we make. */
127	# ifdef RT_OS_WINDOWS
128	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
129	# else
130	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
131	# endif
132
133	/** Frame pointer (RBP) relative offset of the last push. */
134	# ifdef RT_OS_WINDOWS
135	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
136	# else
137	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
138	# endif
139	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
140	* address for it). */
141	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
142	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
143	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
144	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
145	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
146	# ifdef RT_OS_WINDOWS
147	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
148	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
149	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
150	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
151	# endif
152
153	# ifdef RT_OS_WINDOWS
154	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
155	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
156	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
157	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
158	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
159	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
160	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
161	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
162	/** The offset to VBOXSTRICTRC on the stack. */
163	# define IEMNATIVE_FP_OFF_VBOXSTRICRC IEMNATIVE_FP_OFF_IN_SHADOW_ARG0
164	# endif
165
166	#elif RT_ARCH_ARM64
167	/** No alignment padding needed for arm64.
168	* @note HACK ALERT! We abuse this for keeping VBOXSTRICTRC on windows, since
169	* it isn't allowed to be returned by register. */
170	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
171	# ifdef VBOXSTRICTRC_STRICT_ENABLED
172	# ifdef RT_OS_WINDOWS
173	# undef IEMNATIVE_FRAME_ALIGN_SIZE
174	# define IEMNATIVE_FRAME_ALIGN_SIZE 16
175	/** The offset to VBOXSTRICTRC on the stack. */
176	# define IEMNATIVE_FP_OFF_VBOXSTRICRC (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE)
177	# endif
178	# endif
179	/** No stack argument slots, got 8 registers for arguments will suffice. */
180	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
181	/** There are no argument spill area. */
182	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
183
184	/** Number of saved registers at the top of our stack frame.
185	* This includes the return address and old frame pointer, so x19 thru x30. */
186	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
187	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
188	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
189
190	/** Frame pointer (BP) relative offset of the last push. */
191	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
192
193	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
194	* address for it). */
195	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
196
197	#else
198	# error "port me"
199	#endif
200	/** @} */
201
202
203	/** @name Fixed Register Allocation(s)
204	* @{ */
205	/** @def IEMNATIVE_REG_FIXED_PVMCPU
206	* The number of the register holding the pVCpu pointer. */
207	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
208	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
209	* @note This not available on AMD64, only ARM64. */
210	/** @def IEMNATIVE_REG_FIXED_TMP0
211	* Dedicated temporary register.
212	* @note This has extremely short lifetime, must be used with great care to make
213	* sure any calling code or code being called is making use of it.
214	* It will definitely not survive a call or anything of that nature.
215	* @todo replace this by a register allocator and content tracker. */
216	/** @def IEMNATIVE_REG_FIXED_MASK
217	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
218	* architecture. */
219	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
220	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
221	* architecture. */
222	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
223	* Dedicated temporary SIMD register. */
224	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
225	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
226	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
227	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
228	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
229	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
230	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
231	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
232	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
233	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
234	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
235	# else
236	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
237	# endif
238	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
239	\| RT_BIT_32(ARMV8_A64_REG_LR) \
240	\| RT_BIT_32(ARMV8_A64_REG_BP) \
241	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
242	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
243	\| RT_BIT_32(ARMV8_A64_REG_X18) \
244	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
245	\| IEMNATIVE_REG_FIXED_MASK_ADD)
246
247	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
248	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
249	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
250	# else
251	/** @note
252	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
253	* support emulating 256-bit registers we pair two real registers statically to
254	* one virtual for now, leaving us with only 16 256-bit registers. We always
255	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
256	* the register allocator assumes that it will be always free when the lower is
257	* picked.
258	*
259	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
260	* touch them in order to avoid having to save and restore them in the
261	* prologue/epilogue.
262	*/
263	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
264	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
265	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
266	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
267	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
268	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
269	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
270	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
271	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
272	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
273	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
274	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
275	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
276	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
277	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
278	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
279	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
280	\| RT_BIT_32(ARMV8_A64_REG_Q1))
281	# endif
282
283	#elif defined(RT_ARCH_AMD64)
284	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
285	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
286	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
287	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
288	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
289	\| RT_BIT_32(X86_GREG_xSP) \
290	\| RT_BIT_32(X86_GREG_xBP) )
291
292	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
293	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
294	# ifndef _MSC_VER
295	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
296	# endif
297	# endif
298	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
299	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
300	# else
301	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
302	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
303	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
304	# endif
305
306	#else
307	# error "port me"
308	#endif
309	/** @} */
310
311	/** @name Call related registers.
312	* @{ */
313	/** @def IEMNATIVE_CALL_RET_GREG
314	* The return value register. */
315	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
316	* Number of arguments in registers. */
317	/** @def IEMNATIVE_CALL_ARG0_GREG
318	* The general purpose register carrying argument \#0. */
319	/** @def IEMNATIVE_CALL_ARG1_GREG
320	* The general purpose register carrying argument \#1. */
321	/** @def IEMNATIVE_CALL_ARG2_GREG
322	* The general purpose register carrying argument \#2. */
323	/** @def IEMNATIVE_CALL_ARG3_GREG
324	* The general purpose register carrying argument \#3. */
325	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
326	* Mask of registers the callee will not save and may trash. */
327	#ifdef RT_ARCH_AMD64
328	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
329
330	# ifdef RT_OS_WINDOWS
331	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
332	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
333	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
334	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
335	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
336	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
337	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
338	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
339	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
340	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
341	\| RT_BIT_32(X86_GREG_xCX) \
342	\| RT_BIT_32(X86_GREG_xDX) \
343	\| RT_BIT_32(X86_GREG_x8) \
344	\| RT_BIT_32(X86_GREG_x9) \
345	\| RT_BIT_32(X86_GREG_x10) \
346	\| RT_BIT_32(X86_GREG_x11) )
347	/* xmm0 - xmm5 are marked as volatile. */
348	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
349
350	# else /* !RT_OS_WINDOWS */
351	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
352	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
353	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
354	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
355	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
356	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
357	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
358	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
359	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
360	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
361	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
362	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
363	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
364	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
365	\| RT_BIT_32(X86_GREG_xCX) \
366	\| RT_BIT_32(X86_GREG_xDX) \
367	\| RT_BIT_32(X86_GREG_xDI) \
368	\| RT_BIT_32(X86_GREG_xSI) \
369	\| RT_BIT_32(X86_GREG_x8) \
370	\| RT_BIT_32(X86_GREG_x9) \
371	\| RT_BIT_32(X86_GREG_x10) \
372	\| RT_BIT_32(X86_GREG_x11) )
373	/* xmm0 - xmm15 are marked as volatile. */
374	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
375	# endif /* !RT_OS_WINDOWS */
376
377	#elif defined(RT_ARCH_ARM64)
378	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
379	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
380	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
381	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
382	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
383	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
384	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
385	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
386	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
387	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
388	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
389	\| RT_BIT_32(ARMV8_A64_REG_X1) \
390	\| RT_BIT_32(ARMV8_A64_REG_X2) \
391	\| RT_BIT_32(ARMV8_A64_REG_X3) \
392	\| RT_BIT_32(ARMV8_A64_REG_X4) \
393	\| RT_BIT_32(ARMV8_A64_REG_X5) \
394	\| RT_BIT_32(ARMV8_A64_REG_X6) \
395	\| RT_BIT_32(ARMV8_A64_REG_X7) )
396	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
397	\| RT_BIT_32(ARMV8_A64_REG_X1) \
398	\| RT_BIT_32(ARMV8_A64_REG_X2) \
399	\| RT_BIT_32(ARMV8_A64_REG_X3) \
400	\| RT_BIT_32(ARMV8_A64_REG_X4) \
401	\| RT_BIT_32(ARMV8_A64_REG_X5) \
402	\| RT_BIT_32(ARMV8_A64_REG_X6) \
403	\| RT_BIT_32(ARMV8_A64_REG_X7) \
404	\| RT_BIT_32(ARMV8_A64_REG_X8) \
405	\| RT_BIT_32(ARMV8_A64_REG_X9) \
406	\| RT_BIT_32(ARMV8_A64_REG_X10) \
407	\| RT_BIT_32(ARMV8_A64_REG_X11) \
408	\| RT_BIT_32(ARMV8_A64_REG_X12) \
409	\| RT_BIT_32(ARMV8_A64_REG_X13) \
410	\| RT_BIT_32(ARMV8_A64_REG_X14) \
411	\| RT_BIT_32(ARMV8_A64_REG_X15) \
412	\| RT_BIT_32(ARMV8_A64_REG_X16) \
413	\| RT_BIT_32(ARMV8_A64_REG_X17) )
414	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
415	* so to simplify our life a bit we just mark everything as volatile. */
416	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK UINT32_C(0xffffffff)
417
418	#endif
419
420	/** This is the maximum argument count we'll ever be needing. */
421	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
422	#ifdef RT_OS_WINDOWS
423	# ifdef VBOXSTRICTRC_STRICT_ENABLED
424	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
425	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
426	# endif
427	#endif
428
429	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
430	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
431	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
432	#ifdef IEMNATIVE_REG_FIXED_TMP0
433	# ifdef IEMNATIVE_REG_FIXED_TMP1
434	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
435	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
436	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
437	# else
438	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
439	# endif
440	#else
441	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
442	#endif
443
444	/** @def IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
445	* The allocatable non-volatile general purpose register set. */
446	#define IEMNATIVE_CALL_NONVOLATILE_GREG_MASK \
447	(~IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
448	/** @} */
449
450
451	/** @def IEMNATIVE_HST_GREG_COUNT
452	* Number of host general purpose registers we tracker. */
453	/** @def IEMNATIVE_HST_GREG_MASK
454	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
455	* inverted register masks and such to get down to a correct set of regs. */
456	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
457	* Number of host SIMD registers we track. */
458	/** @def IEMNATIVE_HST_SIMD_REG_MASK
459	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
460	* inverted register masks and such to get down to a correct set of regs. */
461	#ifdef RT_ARCH_AMD64
462	# define IEMNATIVE_HST_GREG_COUNT 16
463	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
464
465	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
466	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
467
468	#elif defined(RT_ARCH_ARM64)
469	# define IEMNATIVE_HST_GREG_COUNT 32
470	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
471
472	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
473	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
474
475	#else
476	# error "Port me!"
477	#endif
478
479
480	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
481
482
483	/** Native code generator label types. */
484	typedef enum
485	{
486	kIemNativeLabelType_Invalid = 0,
487	/** @name Exit reasons - Labels w/o data, only once instance per TB.
488	*
489	* The labels requiring register inputs are documented.
490	*
491	* @note Jumps to these requires instructions that are capable of spanning the
492	* max TB length.
493	* @{
494	*/
495	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
496	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
497	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
498	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
499	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
500	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
501	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
502	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
503	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
504	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
505	kIemNativeLabelType_ObsoleteTb, /*< Calls iemNativeHlpObsoleteTb (no inputs). /
506	kIemNativeLabelType_NeedCsLimChecking, /*< Calls iemNativeHlpNeedCsLimChecking (no inputs). /
507	kIemNativeLabelType_CheckBranchMiss, /*< Calls iemNativeHlpCheckBranchMiss (no inputs). /
508	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
509
510	/* Manually defined labels: */
511	/*< Returns with VINF_SUCCESS, no inputs. /
512	kIemNativeLabelType_ReturnSuccess,
513	/** Returns with VINF_IEM_REEXEC_FINISH_WITH_FLAGS, no inputs. */
514	kIemNativeLabelType_ReturnWithFlags,
515	/** Returns with VINF_IEM_REEXEC_BREAK, no inputs. */
516	kIemNativeLabelType_ReturnBreak,
517	/** Returns with VINF_IEM_REEXEC_BREAK_FF, no inputs. */
518	kIemNativeLabelType_ReturnBreakFF,
519	/** The last TB exit label that doesn't have any input registers. */
520	kIemNativeLabelType_LastTbExitWithoutInputs = kIemNativeLabelType_ReturnBreakFF,
521
522	/** Argument registers 1, 2 & 3 are set up. */
523	kIemNativeLabelType_ReturnBreakViaLookup,
524	/** Argument registers 1, 2 & 3 are set up. */
525	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
526	/** Argument registers 1 & 2 are set up. */
527	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
528	/** Argument registers 1 & 2 are set up. */
529	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
530	/** Return register holds the RC and the instruction number is in CL/RCX
531	* on amd64 and the 2rd argument register elsewhere. */
532	kIemNativeLabelType_NonZeroRetOrPassUp,
533
534	/** The last fixup for branches that can span almost the whole TB length.
535	* @note Whether kIemNativeLabelType_Return needs to be one of these is
536	* a bit questionable, since nobody jumps to it except other tail code. */
537	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
538	/** The last fixup for branches that exits the TB. */
539	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_NonZeroRetOrPassUp,
540	/** @} */
541
542	/** Loop-jump target. */
543	kIemNativeLabelType_LoopJumpTarget,
544
545	/*
546	* Labels with data, potentially multiple instances per TB:
547	*
548	* These are localized labels, so no fixed jump type restrictions here.
549	*/
550	kIemNativeLabelType_FirstWithMultipleInstances,
551	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
552	kIemNativeLabelType_Else,
553	kIemNativeLabelType_Endif,
554	kIemNativeLabelType_CheckIrq,
555	kIemNativeLabelType_TlbLookup,
556	kIemNativeLabelType_TlbMiss,
557	kIemNativeLabelType_TlbDone,
558	kIemNativeLabelType_End
559	} IEMNATIVELABELTYPE;
560
561	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
562	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
563
564	#define IEMNATIVELABELTYPE_IS_EXIT_WITHOUT_INPUTS(a_enmLabel) \
565	((a_enmLabel) <= kIemNativeLabelType_LastTbExitWithoutInputs && (a_enmLabel) > kIemNativeLabelType_Invalid)
566
567	/**
568	* Get the mask of input registers for an TB exit label.
569	* This will return zero for any non-exit lable.
570	*/
571	#ifdef RT_ARCH_AMD64
572	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
573	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
574	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
575	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
576	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
577	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
578	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
579	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
580	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(X86_GREG_xCX) /* <-- the difference */ \
581	: 0)
582	# else
583	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
584	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
585	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
586	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
587	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
588	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
589	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
590	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
591	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
592	: 0)
593	#endif
594
595
596	/** Native code generator label definition. */
597	typedef struct IEMNATIVELABEL
598	{
599	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
600	* the epilog. */
601	uint32_t off;
602	/** The type of label (IEMNATIVELABELTYPE). */
603	uint16_t enmType;
604	/** Additional label data, type specific. */
605	uint16_t uData;
606	} IEMNATIVELABEL;
607	/** Pointer to a label. */
608	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
609
610
611
612	/** Native code generator fixup types. */
613	typedef enum
614	{
615	kIemNativeFixupType_Invalid = 0,
616	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
617	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
618	kIemNativeFixupType_Rel32,
619	#elif defined(RT_ARCH_ARM64)
620	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
621	kIemNativeFixupType_RelImm26At0,
622	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
623	kIemNativeFixupType_RelImm19At5,
624	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
625	kIemNativeFixupType_RelImm14At5,
626	#endif
627	kIemNativeFixupType_End
628	} IEMNATIVEFIXUPTYPE;
629
630	/** Native code generator fixup. */
631	typedef struct IEMNATIVEFIXUP
632	{
633	/** Code offset of the fixup location. */
634	uint32_t off;
635	/** The IEMNATIVELABEL this is a fixup for. */
636	uint16_t idxLabel;
637	/** The fixup type (IEMNATIVEFIXUPTYPE). */
638	uint8_t enmType;
639	/** Addend or other data. */
640	int8_t offAddend;
641	} IEMNATIVEFIXUP;
642	/** Pointer to a native code generator fixup. */
643	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
644
645
646
647	/** Native code generator fixup to per chunk TB tail code. */
648	typedef struct IEMNATIVEEXITFIXUP
649	{
650	/** Code offset of the fixup location. */
651	uint32_t off;
652	/** The exit reason. */
653	IEMNATIVELABELTYPE enmExitReason;
654	} IEMNATIVEEXITFIXUP;
655	/** Pointer to a native code generator TB exit fixup. */
656	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
657
658	/**
659	* Per executable memory chunk context with addresses for common code.
660	*/
661	typedef struct IEMNATIVEPERCHUNKCTX
662	{
663	/** Pointers to the exit labels */
664	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
665	} IEMNATIVEPERCHUNKCTX;
666	/** Pointer to per-chunk recompiler context. */
667	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
668	/** Pointer to const per-chunk recompiler context. */
669	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
670
671
672
673	/**
674	* One bit of the state.
675	*
676	* Each register state takes up two bits. We keep the two bits in two separate
677	* 64-bit words to simplify applying them to the guest shadow register mask in
678	* the register allocator.
679	*/
680	typedef union IEMLIVENESSBIT
681	{
682	uint64_t bm64;
683	RT_GCC_EXTENSION struct
684	{ /* bit no */
685	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
686	uint64_t fCr0 : 1; /*< 0x10 / 16: /
687	uint64_t fCr4 : 1; /*< 0x11 / 17: /
688	uint64_t fFcw : 1; /*< 0x12 / 18: /
689	uint64_t fFsw : 1; /*< 0x13 / 19: /
690	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
691	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
692	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
693	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
694	uint64_t fXcr0 : 1; /*< 0x2c / 44: /
695	uint64_t fMxCsr : 1; /*< 0x2d / 45: /
696	uint64_t fEflOther : 1; /*< 0x2e / 46: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
697	uint64_t fEflCf : 1; /*< 0x2f / 47: Carry flag (X86_EFL_CF / 0). /
698	uint64_t fEflPf : 1; /*< 0x30 / 48: Parity flag (X86_EFL_PF / 2). /
699	uint64_t fEflAf : 1; /*< 0x31 / 59: Auxilary carry flag (X86_EFL_AF / 4). /
700	uint64_t fEflZf : 1; /*< 0x32 / 50: Zero flag (X86_EFL_ZF / 6). /
701	uint64_t fEflSf : 1; /*< 0x33 / 51: Signed flag (X86_EFL_SF / 7). /
702	uint64_t fEflOf : 1; /*< 0x34 / 52: Overflow flag (X86_EFL_OF / 12). /
703	uint64_t fUnusedPc : 1; /*< 0x35 / 53: (PC in ) /
704	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
705	};
706	} IEMLIVENESSBIT;
707	AssertCompileSize(IEMLIVENESSBIT, 8);
708
709	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
710	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
711	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
712	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
713	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
714	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
715	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
716	#define IEMLIVENESSBIT_IDX_EFL_COUNT 7
717
718
719	/**
720	* A liveness state entry.
721	*
722	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
723	* Once we add a SSE register shadowing, we'll add another 64-bit element for
724	* that.
725	*/
726	typedef union IEMLIVENESSENTRY
727	{
728	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
729	uint64_t bm64[16 / 8];
730	uint16_t bm32[16 / 4];
731	uint16_t bm16[16 / 2];
732	uint8_t bm8[ 16 / 1];
733	IEMLIVENESSBIT aBits[2];
734	#else
735	uint64_t bm64[32 / 8];
736	uint16_t bm32[32 / 4];
737	uint16_t bm16[32 / 2];
738	uint8_t bm8[ 32 / 1];
739	IEMLIVENESSBIT aBits[4];
740	#endif
741	RT_GCC_EXTENSION struct
742	{
743	/** Bit \#0 of the register states. */
744	IEMLIVENESSBIT Bit0;
745	/** Bit \#1 of the register states. */
746	IEMLIVENESSBIT Bit1;
747	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
748	/** Bit \#2 of the register states. */
749	IEMLIVENESSBIT Bit2;
750	/** Bit \#3 of the register states. */
751	IEMLIVENESSBIT Bit3;
752	#endif
753	};
754	} IEMLIVENESSENTRY;
755	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
756	AssertCompileSize(IEMLIVENESSENTRY, 16);
757	#else
758	AssertCompileSize(IEMLIVENESSENTRY, 32);
759	#endif
760	/** Pointer to a liveness state entry. */
761	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
762	/** Pointer to a const liveness state entry. */
763	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
764
765	/** @name 64-bit value masks for IEMLIVENESSENTRY.
766	* @{ / / 0xzzzzyyyyxxxxwwww */
767	#define IEMLIVENESSBIT_MASK UINT64_C(0x001fffffffffffff)
768
769	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
770	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
771	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
772
773	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
774	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
775	#endif
776
777	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x001fc00000000000)
778	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x001f800000000000)
779
780	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
781	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
782	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
783	#endif
784	/** @} */
785
786
787	/** @name The liveness state for a register.
788	*
789	* The state values have been picked to with state accumulation in mind (what
790	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
791	* performance critical work done with the values.
792	*
793	* This is a compressed state that only requires 2 bits per register.
794	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
795	* 1. the incoming state from the following call,
796	* 2. the outgoing state for this call,
797	* 3. mask of the entries set in the 2nd.
798	*
799	* The mask entry (3rd one above) will be used both when updating the outgoing
800	* state and when merging in incoming state for registers not touched by the
801	* current call.
802	*
803	*
804	* Extended Layout:
805	*
806	* The extended layout variation differs from the above as it records the
807	* different register accesses as individual bits, and it is currently used for
808	* the delayed EFLAGS calculation experiments. The latter means that
809	* calls/tb-exits and potential calls/exceptions/tb-exits are recorded
810	* separately so the latter can be checked for in combination with clobbering.
811	*
812	* @{ */
813	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
814	/** The register will be clobbered and the current value thrown away.
815	*
816	* When this is applied to the state (2) we'll simply be AND'ing it with the
817	* (old) mask (3) and adding the register to the mask. This way we'll
818	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
819	* IEMLIVENESS_STATE_INPUT states. */
820	# define IEMLIVENESS_STATE_CLOBBERED 0
821	/** The register is unused in the remainder of the TB.
822	*
823	* This is an initial state and can not be set by any of the
824	* iemNativeLivenessFunc_xxxx callbacks. */
825	# define IEMLIVENESS_STATE_UNUSED 1
826	/** The register value is required in a potential call or exception.
827	*
828	* This means that the register value must be calculated and is best written to
829	* the state, but that any shadowing registers can be flushed thereafter as it's
830	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
831	*
832	* It is typically applied across the board, but we preserve incoming
833	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
834	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
835	* 1. r0 = old & ~mask;
836	* 2. r0 = t1 & (t1 >> 1);
837	* 3. state \|= r0 \| 0b10;
838	* 4. mask = ~0;
839	*/
840	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
841	/** The register value is used as input.
842	*
843	* This means that the register value must be calculated and it is best to keep
844	* it in a register. It does not need to be writtent out as such. This is the
845	* highest priority state.
846	*
847	* Whether the call modifies the register or not isn't relevant to earlier
848	* calls, so that's not recorded.
849	*
850	* When applying this state we just or in the value in the outgoing state and
851	* mask. */
852	# define IEMLIVENESS_STATE_INPUT 3
853	/** Mask of the state bits. */
854	# define IEMLIVENESS_STATE_MASK 3
855	/** The number of bits per state. */
856	# define IEMLIVENESS_STATE_BIT_COUNT 2
857
858	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state.
859	* @note only used in assertions. */
860	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
861	/** Check if we're expecting read accesses to a register with the given (previous) liveness state.
862	* @note only used in assertions. */
863	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
864	/** Check if a register clobbering is expected given the (previous) liveness state.
865	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
866	* include INPUT if the register is used in more than one place.
867	* @note only used in assertions. */
868	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
869
870	/** Check if all status flags are going to be clobbered and doesn't need
871	* calculating in the current step.
872	* @param a_pCurEntry The current liveness entry.
873	* @note Used by actual code. */
874	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
875	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
876
877	/***
878	* Construct a mask of what will be clobbered and never used.
879	*
880	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
881	* unnecessary EFLAGS calculations.
882	*
883	* @param a_pCurEntry The current liveness entry.
884	* @note Used by actual code.
885	*/
886	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
887	( ~((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_MASK )
888
889	/** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
890	* states, as these are no longer needed.
891	* @param a_pCurEntry The current liveness entry.
892	* @note Used by actual code. */
893	AssertCompile(IEMLIVENESS_STATE_UNUSED == 1 && IEMLIVENESS_STATE_XCPT_OR_CALL == 2);
894	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
895	( (a_pCurEntry)->Bit0.bm64 ^ (a_pCurEntry)->Bit1.bm64 )
896
897
898	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
899	/** The register is not used any more. */
900	# define IEMLIVENESS_STATE_UNUSED 0
901	/** Flag: The register is required in a potential call or/and exception. */
902	# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
903	# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
904	/** Flag: The register is read. */
905	# define IEMLIVENESS_STATE_READ 2
906	# define IEMLIVENESS_BIT_READ 1
907	/** Flag: The register is written. */
908	# define IEMLIVENESS_STATE_WRITE 4
909	# define IEMLIVENESS_BIT_WRITE 2
910	/** Flag: Unconditional call. */
911	# define IEMLIVENESS_STATE_CALL 8
912	# define IEMLIVENESS_BIT_CALL 3
913
914	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
915	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
916	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
917	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
918
919	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
920	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
921	&& !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
922	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
923	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
924	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
925
926	/** Construct a mask of the registers not in the read or write state.
927	* @note We could skips writes, if they aren't from us, as this is just a hack
928	* to prevent trashing registers that have just been written or will be
929	* written when we retire the current instruction.
930	* @param a_pCurEntry The current liveness entry.
931	* @note Used by actual code. */
932	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
933	( ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
934	& ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
935	& IEMLIVENESSBIT_MASK )
936
937	/***
938	* Construct a mask of what will be clobbered and never used.
939	*
940	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
941	* unnecessary EFLAGS calculations.
942	*
943	* @param a_pCurEntry The current liveness entry.
944	* @note Used by actual code.
945	*/
946	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
947	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
948	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
949	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
950	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
951
952	/**
953	* Construct a mask of what (EFLAGS) which can be postponed.
954	*
955	* The postponement is for the avoiding EFLAGS status bits calculations in the
956	* primary code stream whenever possible, and instead only do these in the TLB
957	* load and TB exit code paths which shouldn't be traveled quite as often.
958	* A requirement, though, is that the status bits will be clobbered later in the
959	* TB.
960	*
961	* User need to apply IEMLIVENESSBIT_STATUS_EFL_MASK if appropriate/necessary.
962	*
963	* @param a_pCurEntry The current liveness entry.
964	* @note Used by actual code.
965	*/
966	# define IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(a_pCurEntry) \
967	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
968	& (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
969	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
970	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
971
972	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
973	/** @} */
974
975	/** @name Liveness helpers for builtin functions and similar.
976	*
977	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
978	* own set of manipulator macros for those.
979	*
980	* @{ */
981	/** Initializing the state as all unused. */
982	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
983	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
984	do { \
985	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
986	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
987	} while (0)
988	#else
989	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
990	do { \
991	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
992	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
993	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
994	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
995	} while (0)
996	#endif
997
998	/** Initializing the outgoing state with a potential xcpt or call state.
999	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT.
1000	*
1001	* @note Must invoke IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL when done!
1002	*/
1003	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1004	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1005	do { \
1006	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1007	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1008	} while (0)
1009	#else
1010	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1011	do { \
1012	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1013	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1014	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1015	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
1016	} while (0)
1017	#endif
1018
1019	/** Completes IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL after applying any
1020	* other state modifications.
1021	*/
1022	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1023	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) ((void)0)
1024	#else
1025	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1026	do { \
1027	uint64_t const fInhMask = ~( (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL].bm64 \
1028	\| (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE].bm64); \
1029	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 & fInhMask; \
1030	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64 & fInhMask; \
1031	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & fInhMask; \
1032	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_CALL].bm64 & fInhMask; \
1033	} while (0)
1034	#endif
1035
1036	/** Initializing the outgoing state with an unconditional call state.
1037	* This should only really be used alone. */
1038	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1039	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1040	do { \
1041	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1042	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1043	} while (0)
1044	#else
1045	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1046	do { \
1047	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1048	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
1049	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1050	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1051	RT_NOREF(a_pIncoming); \
1052	} while (0)
1053	#endif
1054
1055	#if 0 /* unused */
1056	/** Initializing the outgoing state with an unconditional call state as well as
1057	* an potential call/exception preceeding it.
1058	* This should only really be used alone. */
1059	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1060	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1061	do { \
1062	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1063	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1064	} while (0)
1065	#else
1066	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1067	do { \
1068	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1069	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1070	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1071	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1072	} while (0)
1073	#endif
1074	#endif
1075
1076	/** Adds a segment base register as input to the outgoing state. */
1077	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1078	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1079	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
1080	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
1081	} while (0)
1082	#else
1083	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1084	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
1085	} while (0)
1086	#endif
1087
1088	/** Adds a segment attribute register as input to the outgoing state. */
1089	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1090	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1091	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1092	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1093	} while (0)
1094	#else
1095	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1096	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1097	} while (0)
1098	#endif
1099
1100	/** Adds a segment limit register as input to the outgoing state. */
1101	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1102	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1103	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1104	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1105	} while (0)
1106	#else
1107	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1108	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
1109	} while (0)
1110	#endif
1111
1112	/** Adds a segment limit register as input to the outgoing state. */
1113	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1114	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1115	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
1116	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
1117	} while (0)
1118	#else
1119	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1120	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
1121	} while (0)
1122	#endif
1123	/** @} */
1124
1125	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY
1126	* Debug assertion that the required flags are available and not incorrectly skipped.
1127	*/
1128	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
1129	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) \
1130	AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
1131	("%#x & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, \
1132	a_fEflNeeded, (a_pReNative)->fSkippingEFlags & (a_fEflNeeded) ))
1133	#else
1134	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1135	#endif
1136
1137	/** @def IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY
1138	* Debug assertion that the required flags are available and not incorrectly postponed.
1139	*/
1140	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1141	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
1142	AssertMsg(!((a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded)), \
1143	("%#x & %#x -> %#x\n", (a_pReNative)->PostponedEfl.fEFlags, \
1144	a_fEflNeeded, (a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded) ))
1145	#else
1146	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1147	#endif
1148
1149	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING
1150	* Debug assertion that the required flags are available and not incorrectly
1151	* skipped or postponed.
1152	*/
1153	#if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
1154	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1155	AssertMsg(!(((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded)), \
1156	("(%#x \| %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->PostponedEfl.fEFlags, \
1157	a_fEflNeeded, ((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded) ))
1158	#elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
1159	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1160	IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded)
1161	#elif defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \
1162	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1163	IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded)
1164	#else
1165	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) ((void)0)
1166	#endif
1167
1168	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
1169	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
1170	* calculated and up to date. This is to double check that we haven't skipped
1171	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
1172	* @note has to be placed in
1173	*/
1174	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1175	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { \
1176	(a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); \
1177	} while (0)
1178	#else
1179	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
1180	#endif
1181
1182
1183	/** @def IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS
1184	* Number of extra instructions to allocate for each TB exit to account for
1185	* postponed EFLAGS calculations.
1186	*/
1187	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1188	# ifdef RT_ARCH_AMD64
1189	# ifdef VBOX_STRICT
1190	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 64
1191	# else
1192	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1193	# endif
1194	# elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
1195	# ifdef VBOX_STRICT
1196	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 48
1197	# else
1198	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1199	# endif
1200	# else
1201	# error "port me"
1202	# endif
1203	#else
1204	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 0
1205	#endif
1206
1207	/** @def IEMNATIVE_CLEAR_POSTPONED_EFLAGS
1208	* Helper macro function for calling iemNativeClearPostponedEFlags() when
1209	* IEMNATIVE_WITH_EFLAGS_POSTPONING is enabled.
1210	*/
1211	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1212	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) iemNativeClearPostponedEFlags<a_fEflClobbered>(a_pReNative)
1213	#else
1214	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) ((void)0)
1215	#endif
1216
1217	/** @def IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS
1218	* Macro for testing whether there are currently any postponed EFLAGS calcs w/o
1219	* needing to \#ifdef the check.
1220	*/
1221	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1222	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) ((a_pReNative)->PostponedEfl.fEFlags != 0)
1223	#else
1224	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) false
1225	#endif
1226
1227
1228	/**
1229	* Translation block debug info entry type.
1230	*/
1231	typedef enum IEMTBDBGENTRYTYPE
1232	{
1233	kIemTbDbgEntryType_Invalid = 0,
1234	/** The entry is for marking a native code position.
1235	* Entries following this all apply to this position. */
1236	kIemTbDbgEntryType_NativeOffset,
1237	/** The entry is for a new guest instruction. */
1238	kIemTbDbgEntryType_GuestInstruction,
1239	/** Marks the start of a threaded call. */
1240	kIemTbDbgEntryType_ThreadedCall,
1241	/** Marks the location of a label. */
1242	kIemTbDbgEntryType_Label,
1243	/** Info about a host register shadowing a guest register. */
1244	kIemTbDbgEntryType_GuestRegShadowing,
1245	/** Info about a host SIMD register shadowing a guest SIMD register. */
1246	kIemTbDbgEntryType_GuestSimdRegShadowing,
1247	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1248	/** Info about a delayed RIP update. */
1249	kIemTbDbgEntryType_DelayedPcUpdate,
1250	#endif
1251	/** Info about a shadowed guest register becoming dirty. */
1252	kIemTbDbgEntryType_GuestRegDirty,
1253	/** Info about register writeback/flush oepration. */
1254	kIemTbDbgEntryType_GuestRegWriteback,
1255	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1256	/** Info about a delayed EFLAGS calculation. */
1257	kIemTbDbgEntryType_PostponedEFlagsCalc,
1258	#endif
1259	kIemTbDbgEntryType_End
1260	} IEMTBDBGENTRYTYPE;
1261
1262	/**
1263	* Translation block debug info entry.
1264	*/
1265	typedef union IEMTBDBGENTRY
1266	{
1267	/** Plain 32-bit view. */
1268	uint32_t u;
1269
1270	/** Generic view for getting at the type field. */
1271	struct
1272	{
1273	/** IEMTBDBGENTRYTYPE */
1274	uint32_t uType : 4;
1275	uint32_t uTypeSpecific : 28;
1276	} Gen;
1277
1278	struct
1279	{
1280	/** kIemTbDbgEntryType_ThreadedCall1. */
1281	uint32_t uType : 4;
1282	/** Native code offset. */
1283	uint32_t offNative : 28;
1284	} NativeOffset;
1285
1286	struct
1287	{
1288	/** kIemTbDbgEntryType_GuestInstruction. */
1289	uint32_t uType : 4;
1290	uint32_t uUnused : 4;
1291	/** The IEM_F_XXX flags. */
1292	uint32_t fExec : 24;
1293	} GuestInstruction;
1294
1295	struct
1296	{
1297	/* kIemTbDbgEntryType_ThreadedCall. */
1298	uint32_t uType : 4;
1299	/** Set if the call was recompiled to native code, clear if just calling
1300	* threaded function. */
1301	uint32_t fRecompiled : 1;
1302	uint32_t uUnused : 11;
1303	/** The threaded call number (IEMTHREADEDFUNCS). */
1304	uint32_t enmCall : 16;
1305	} ThreadedCall;
1306
1307	struct
1308	{
1309	/* kIemTbDbgEntryType_Label. */
1310	uint32_t uType : 4;
1311	uint32_t uUnused : 4;
1312	/** The label type (IEMNATIVELABELTYPE). */
1313	uint32_t enmLabel : 8;
1314	/** The label data. */
1315	uint32_t uData : 16;
1316	} Label;
1317
1318	struct
1319	{
1320	/* kIemTbDbgEntryType_GuestRegShadowing. */
1321	uint32_t uType : 4;
1322	uint32_t uUnused : 4;
1323	/** The guest register being shadowed (IEMNATIVEGSTREG). */
1324	uint32_t idxGstReg : 8;
1325	/** The host new register number, UINT8_MAX if dropped. */
1326	uint32_t idxHstReg : 8;
1327	/** The previous host register number, UINT8_MAX if new. */
1328	uint32_t idxHstRegPrev : 8;
1329	} GuestRegShadowing;
1330
1331	struct
1332	{
1333	/* kIemTbDbgEntryType_GuestSimdRegShadowing. */
1334	uint32_t uType : 4;
1335	uint32_t uUnused : 4;
1336	/** The guest register being shadowed (IEMNATIVEGSTSIMDREG). */
1337	uint32_t idxGstSimdReg : 8;
1338	/** The host new register number, UINT8_MAX if dropped. */
1339	uint32_t idxHstSimdReg : 8;
1340	/** The previous host register number, UINT8_MAX if new. */
1341	uint32_t idxHstSimdRegPrev : 8;
1342	} GuestSimdRegShadowing;
1343
1344	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1345	struct
1346	{
1347	/* kIemTbDbgEntryType_DelayedPcUpdate. */
1348	uint32_t uType : 4;
1349	/** Number of instructions skipped. */
1350	uint32_t cInstrSkipped : 8;
1351	/* The instruction offset added to the program counter. */
1352	int32_t offPc : 20;
1353	} DelayedPcUpdate;
1354	#endif
1355
1356	struct
1357	{
1358	/* kIemTbDbgEntryType_GuestRegDirty. */
1359	uint32_t uType : 4;
1360	uint32_t uUnused : 11;
1361	/** Flag whether this is about a SIMD (true) or general (false) register. */
1362	uint32_t fSimdReg : 1;
1363	/** The guest register index being marked as dirty. */
1364	uint32_t idxGstReg : 8;
1365	/** The host register number this register is shadowed in .*/
1366	uint32_t idxHstReg : 8;
1367	} GuestRegDirty;
1368
1369	struct
1370	{
1371	/* kIemTbDbgEntryType_GuestRegWriteback. */
1372	uint32_t uType : 4;
1373	/** Flag whether this is about a SIMD (true) or general (false) register flush. */
1374	uint32_t fSimdReg : 1;
1375	/** The mask shift. */
1376	uint32_t cShift : 2;
1377	/** The guest register mask being written back. */
1378	uint32_t fGstReg : 25;
1379	} GuestRegWriteback;
1380
1381	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1382	struct
1383	{
1384	/* kIemTbDbgEntryType_PostponedEFlagsCalc. */
1385	uint32_t uType : 4;
1386	/** The EFLAGS operation (IEMNATIVE_POSTPONED_EFL_OP_T). */
1387	uint32_t enmOp : 4;
1388	/** The mask shift. */
1389	uint32_t cOpBits : 8;
1390	/** The emit instance number (0-based). */
1391	uint32_t idxEmit : 8;
1392	/** Unused. */
1393	uint32_t uUnused : 8;
1394	} PostponedEflCalc;
1395	#endif
1396	} IEMTBDBGENTRY;
1397	AssertCompileSize(IEMTBDBGENTRY, sizeof(uint32_t));
1398	/** Pointer to a debug info entry. */
1399	typedef IEMTBDBGENTRY *PIEMTBDBGENTRY;
1400	/** Pointer to a const debug info entry. */
1401	typedef IEMTBDBGENTRY const *PCIEMTBDBGENTRY;
1402
1403	/**
1404	* Translation block debug info.
1405	*/
1406	typedef struct IEMTBDBG
1407	{
1408	/** This is the flat PC corresponding to IEMTB::GCPhysPc. */
1409	RTGCPTR FlatPc;
1410	/** Number of entries in aEntries. */
1411	uint32_t cEntries;
1412	/** The offset of the last kIemTbDbgEntryType_NativeOffset record. */
1413	uint32_t offNativeLast;
1414	/** Debug info entries. */
1415	RT_FLEXIBLE_ARRAY_EXTENSION
1416	IEMTBDBGENTRY aEntries[RT_FLEXIBLE_ARRAY];
1417	} IEMTBDBG;
1418	/** Pointer to TB debug info. */
1419	typedef IEMTBDBG *PIEMTBDBG;
1420	/** Pointer to const TB debug info. */
1421	typedef IEMTBDBG const *PCIEMTBDBG;
1422
1423	/**
1424	* Guest registers that can be shadowed in GPRs.
1425	*
1426	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
1427	* must be placed last, as the liveness state tracks it as 7 subcomponents and
1428	* we don't want to waste space here.
1429	*
1430	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
1431	* friends as well as IEMAllN8veLiveness.cpp.
1432	*/
1433	typedef enum IEMNATIVEGSTREG : uint8_t
1434	{
1435	kIemNativeGstReg_GprFirst = 0,
1436	kIemNativeGstReg_Rax = kIemNativeGstReg_GprFirst + 0,
1437	kIemNativeGstReg_Rcx = kIemNativeGstReg_GprFirst + 1,
1438	kIemNativeGstReg_Rdx = kIemNativeGstReg_GprFirst + 2,
1439	kIemNativeGstReg_Rbx = kIemNativeGstReg_GprFirst + 3,
1440	kIemNativeGstReg_Rsp = kIemNativeGstReg_GprFirst + 4,
1441	kIemNativeGstReg_Rbp = kIemNativeGstReg_GprFirst + 5,
1442	kIemNativeGstReg_Rsi = kIemNativeGstReg_GprFirst + 6,
1443	kIemNativeGstReg_Rdi = kIemNativeGstReg_GprFirst + 7,
1444	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
1445	kIemNativeGstReg_Cr0,
1446	kIemNativeGstReg_Cr4,
1447	kIemNativeGstReg_FpuFcw,
1448	kIemNativeGstReg_FpuFsw,
1449	kIemNativeGstReg_SegBaseFirst,
1450	kIemNativeGstReg_CsBase = kIemNativeGstReg_SegBaseFirst + X86_SREG_CS,
1451	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1452	kIemNativeGstReg_SegAttribFirst,
1453	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1454	kIemNativeGstReg_SegLimitFirst,
1455	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1456	kIemNativeGstReg_SegSelFirst,
1457	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1458	kIemNativeGstReg_Xcr0,
1459	kIemNativeGstReg_MxCsr,
1460	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags. /
1461	/* 6 entry gap for liveness EFlags subdivisions. */
1462	kIemNativeGstReg_Pc = kIemNativeGstReg_EFlags + 7,
1463	kIemNativeGstReg_End
1464	} IEMNATIVEGSTREG;
1465	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1466	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1467	AssertCompile(RT_BIT_64(kIemNativeGstReg_Pc) - UINT64_C(1) == IEMLIVENESSBIT_MASK);
1468
1469	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1470	* @{ */
1471	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1472	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1473	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1474	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1475	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1476	/** @} */
1477
1478
1479	/**
1480	* Guest registers that can be shadowed in host SIMD registers.
1481	*
1482	* @todo r=aeichner Liveness tracking
1483	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1484	*/
1485	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1486	{
1487	kIemNativeGstSimdReg_SimdRegFirst = 0,
1488	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1489	kIemNativeGstSimdReg_End
1490	} IEMNATIVEGSTSIMDREG;
1491
1492	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1493	* @{ */
1494	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1495	/** @} */
1496
1497	/**
1498	* The Load/store size for a SIMD guest register.
1499	*/
1500	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1501	{
1502	/** Invalid size. */
1503	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1504	/** Loads the low 128-bit of a guest SIMD register. */
1505	kIemNativeGstSimdRegLdStSz_Low128,
1506	/** Loads the high 128-bit of a guest SIMD register. */
1507	kIemNativeGstSimdRegLdStSz_High128,
1508	/** Loads the whole 256-bits of a guest SIMD register. */
1509	kIemNativeGstSimdRegLdStSz_256,
1510	/** End value. */
1511	kIemNativeGstSimdRegLdStSz_End
1512	} IEMNATIVEGSTSIMDREGLDSTSZ;
1513
1514
1515	/**
1516	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1517	*/
1518	typedef enum IEMNATIVEGSTREGUSE
1519	{
1520	/** The usage is read-only, the register holding the guest register
1521	* shadow copy will not be modified by the caller. */
1522	kIemNativeGstRegUse_ReadOnly = 0,
1523	/** The caller will update the guest register (think: PC += cbInstr).
1524	* The guest shadow copy will follow the returned register. */
1525	kIemNativeGstRegUse_ForUpdate,
1526	/** The call will put an entirely new value in the guest register, so
1527	* if new register is allocate it will be returned uninitialized. */
1528	kIemNativeGstRegUse_ForFullWrite,
1529	/** The caller will use the guest register value as input in a calculation
1530	* and the host register will be modified.
1531	* This means that the returned host register will not be marked as a shadow
1532	* copy of the guest register. */
1533	kIemNativeGstRegUse_Calculation
1534	} IEMNATIVEGSTREGUSE;
1535
1536	/**
1537	* Guest registers (classes) that can be referenced.
1538	*/
1539	typedef enum IEMNATIVEGSTREGREF : uint8_t
1540	{
1541	kIemNativeGstRegRef_Invalid = 0,
1542	kIemNativeGstRegRef_Gpr,
1543	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1544	kIemNativeGstRegRef_EFlags,
1545	kIemNativeGstRegRef_MxCsr,
1546	kIemNativeGstRegRef_FpuReg,
1547	kIemNativeGstRegRef_MReg,
1548	kIemNativeGstRegRef_XReg,
1549	kIemNativeGstRegRef_X87,
1550	kIemNativeGstRegRef_XState,
1551	//kIemNativeGstRegRef_YReg, - doesn't work.
1552	kIemNativeGstRegRef_End
1553	} IEMNATIVEGSTREGREF;
1554
1555
1556	/** Variable kinds. */
1557	typedef enum IEMNATIVEVARKIND : uint8_t
1558	{
1559	/** Customary invalid zero value. */
1560	kIemNativeVarKind_Invalid = 0,
1561	/** This is either in a register or on the stack. */
1562	kIemNativeVarKind_Stack,
1563	/** Immediate value - loaded into register when needed, or can live on the
1564	* stack if referenced (in theory). */
1565	kIemNativeVarKind_Immediate,
1566	/** Variable reference - loaded into register when needed, never stack. */
1567	kIemNativeVarKind_VarRef,
1568	/** Guest register reference - loaded into register when needed, never stack. */
1569	kIemNativeVarKind_GstRegRef,
1570	/** End of valid values. */
1571	kIemNativeVarKind_End
1572	} IEMNATIVEVARKIND;
1573
1574
1575	/** Variable or argument. */
1576	typedef struct IEMNATIVEVAR
1577	{
1578	RT_GCC_EXTENSION
1579	union
1580	{
1581	struct
1582	{
1583	/** The kind of variable. */
1584	IEMNATIVEVARKIND enmKind;
1585	/** The variable size in bytes. */
1586	uint8_t cbVar;
1587	/** Set if the registered is currently used exclusively, false if the
1588	* variable is idle and the register can be grabbed. */
1589	bool fRegAcquired;
1590	/** Flag whether this variable is held in a SIMD register (only supported for
1591	* 128-bit and 256-bit variables), only valid when idxReg is not UINT8_MAX. */
1592	bool fSimdReg;
1593	};
1594	uint32_t u32Init0; /*< Init optimzation - cbVar is set, the other are initialized with zeros. /
1595	};
1596
1597	RT_GCC_EXTENSION
1598	union
1599	{
1600	struct
1601	{
1602	/** The host register allocated for the variable, UINT8_MAX if not. */
1603	uint8_t idxReg;
1604	/** The argument number if argument, UINT8_MAX if regular variable. */
1605	uint8_t uArgNo;
1606	/** The first stack slot (uint64_t), except for immediate and references
1607	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1608	* has a stack slot it has been initialized and has a value. Unused variables
1609	* has neither a stack slot nor a host register assignment. */
1610	uint8_t idxStackSlot;
1611	/** If referenced, the index (unpacked) of the variable referencing this one,
1612	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1613	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1614	uint8_t idxReferrerVar;
1615	};
1616	uint32_t u32Init1; /*< Init optimization; all these are initialized to 0xff. /
1617	};
1618
1619	union
1620	{
1621	/** kIemNativeVarKind_Immediate: The immediate value. */
1622	uint64_t uValue;
1623	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1624	uint8_t idxRefVar;
1625	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1626	struct
1627	{
1628	/** The class of register. */
1629	IEMNATIVEGSTREGREF enmClass;
1630	/** Index within the class. */
1631	uint8_t idx;
1632	} GstRegRef;
1633	} u;
1634	} IEMNATIVEVAR;
1635	AssertCompileSize(IEMNATIVEVAR, 16);
1636	/** Pointer to a variable or argument. */
1637	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1638	/** Pointer to a const variable or argument. */
1639	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1640
1641	/** What is being kept in a host register. */
1642	typedef enum IEMNATIVEWHAT : uint8_t
1643	{
1644	/** The traditional invalid zero value. */
1645	kIemNativeWhat_Invalid = 0,
1646	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1647	kIemNativeWhat_Var,
1648	/** Temporary register, this is typically freed when a MC completes. */
1649	kIemNativeWhat_Tmp,
1650	/** Call argument w/o a variable mapping. This is free (via
1651	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1652	kIemNativeWhat_Arg,
1653	/** Return status code.
1654	* @todo not sure if we need this... */
1655	kIemNativeWhat_rc,
1656	/** The fixed pVCpu (PVMCPUCC) register.
1657	* @todo consider offsetting this on amd64 to use negative offsets to access
1658	* more members using 8-byte disp. */
1659	kIemNativeWhat_pVCpuFixed,
1660	/** The fixed pCtx (PCPUMCTX) register.
1661	* @todo consider offsetting this on amd64 to use negative offsets to access
1662	* more members using 8-byte disp. */
1663	kIemNativeWhat_pCtxFixed,
1664	/** Fixed temporary register. */
1665	kIemNativeWhat_FixedTmp,
1666	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1667	/** Shadow RIP for the delayed RIP updating debugging. */
1668	kIemNativeWhat_PcShadow,
1669	#endif
1670	/** Register reserved by the CPU or OS architecture. */
1671	kIemNativeWhat_FixedReserved,
1672	/** End of valid values. */
1673	kIemNativeWhat_End
1674	} IEMNATIVEWHAT;
1675
1676	/**
1677	* Host general register entry.
1678	*
1679	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1680	*
1681	* @todo Track immediate values in host registers similarlly to how we track the
1682	* guest register shadow copies. For it to be real helpful, though,
1683	* we probably need to know which will be reused and put them into
1684	* non-volatile registers, otherwise it's going to be more or less
1685	* restricted to an instruction or two.
1686	*/
1687	typedef struct IEMNATIVEHSTREG
1688	{
1689	/** Set of guest registers this one shadows.
1690	*
1691	* Using a bitmap here so we can designate the same host register as a copy
1692	* for more than one guest register. This is expected to be useful in
1693	* situations where one value is copied to several registers in a sequence.
1694	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1695	* sequence we'd want to let this register follow to be a copy of and there
1696	* will always be places where we'd be picking the wrong one.
1697	*/
1698	uint64_t fGstRegShadows;
1699	/** What is being kept in this register. */
1700	IEMNATIVEWHAT enmWhat;
1701	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1702	uint8_t idxVar;
1703	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1704	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1705	* that scope. */
1706	uint8_t idxStackSlot;
1707	/** Alignment padding. */
1708	uint8_t abAlign[5];
1709	} IEMNATIVEHSTREG;
1710
1711
1712	/**
1713	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1714	* halves, on architectures where there is no 256-bit register available this entry will track
1715	* two adjacent 128-bit host registers.
1716	*
1717	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1718	*/
1719	typedef struct IEMNATIVEHSTSIMDREG
1720	{
1721	/** Set of guest registers this one shadows.
1722	*
1723	* Using a bitmap here so we can designate the same host register as a copy
1724	* for more than one guest register. This is expected to be useful in
1725	* situations where one value is copied to several registers in a sequence.
1726	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1727	* sequence we'd want to let this register follow to be a copy of and there
1728	* will always be places where we'd be picking the wrong one.
1729	*/
1730	uint64_t fGstRegShadows;
1731	/** What is being kept in this register. */
1732	IEMNATIVEWHAT enmWhat;
1733	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1734	uint8_t idxVar;
1735	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1736	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1737	/** Alignment padding. */
1738	uint8_t abAlign[5];
1739	} IEMNATIVEHSTSIMDREG;
1740
1741
1742	/**
1743	* Core state for the native recompiler, that is, things that needs careful
1744	* handling when dealing with branches.
1745	*/
1746	typedef struct IEMNATIVECORESTATE
1747	{
1748	/** Allocation bitmap for aHstRegs. */
1749	uint32_t bmHstRegs;
1750
1751	/** Bitmap marking which host register contains guest register shadow copies.
1752	* This is used during register allocation to try preserve copies. */
1753	uint32_t bmHstRegsWithGstShadow;
1754	/** Bitmap marking valid entries in aidxGstRegShadows. */
1755	uint64_t bmGstRegShadows;
1756	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1757	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1758	uint64_t bmGstRegShadowDirty;
1759	#endif
1760
1761	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1762	/** The current instruction offset in bytes from when the guest program counter
1763	* was updated last. Used for delaying the write to the guest context program counter
1764	* as long as possible. */
1765	int64_t offPc;
1766	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1767	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1768	bool fDebugPcInitialized;
1769	# endif
1770	#endif
1771
1772	/** Allocation bitmap for aHstSimdRegs. */
1773	uint32_t bmHstSimdRegs;
1774
1775	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1776	* This is used during register allocation to try preserve copies. */
1777	uint32_t bmHstSimdRegsWithGstShadow;
1778	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1779	uint64_t bmGstSimdRegShadows;
1780	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1781	uint64_t bmGstSimdRegShadowDirtyLo128;
1782	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1783	uint64_t bmGstSimdRegShadowDirtyHi128;
1784
1785	union
1786	{
1787	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1788	uint8_t aidxArgVars[8];
1789	/** For more efficient resetting. */
1790	uint64_t u64ArgVars;
1791	};
1792
1793	/** Allocation bitmap for the stack. */
1794	uint32_t bmStack;
1795	/** Allocation bitmap for aVars. */
1796	uint32_t bmVars;
1797
1798	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1799	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1800	* (A shadow copy of a guest register can only be held in a one host register,
1801	* there are no duplicate copies or ambiguities like that). */
1802	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1803	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1804	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1805	* (A shadow copy of a guest register can only be held in a one host register,
1806	* there are no duplicate copies or ambiguities like that). */
1807	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1808
1809	/** Host register allocation tracking. */
1810	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1811	/** Host SIMD register allocation tracking. */
1812	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1813
1814	/** Variables and arguments. */
1815	IEMNATIVEVAR aVars[9];
1816	} IEMNATIVECORESTATE;
1817	/** Pointer to core state. */
1818	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1819	/** Pointer to const core state. */
1820	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1821
1822	/** @def IEMNATIVE_VAR_IDX_UNPACK
1823	* @returns Index into IEMNATIVECORESTATE::aVars.
1824	* @param a_idxVar Variable index w/ magic (in strict builds).
1825	*/
1826	/** @def IEMNATIVE_VAR_IDX_PACK
1827	* @returns Variable index w/ magic (in strict builds).
1828	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1829	*/
1830	#ifdef VBOX_STRICT
1831	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1832	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1833	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1834	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1835	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1836	#else
1837	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1838	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1839	#endif
1840
1841
1842	/** Clear the dirty state of the given guest SIMD register. */
1843	#define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1844	do { \
1845	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1846	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1847	} while (0)
1848
1849	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1850	#define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1851	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1852	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1853	#define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1854	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1855	/** Returns whether the given guest SIMD register is dirty. */
1856	#define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1857	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1858
1859	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1860	#define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1861	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1862	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1863	#define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1864	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1865
1866	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1867	#define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1868	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1869	#define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1870	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1871	#define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1872	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1873	#define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1874	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1875	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1876	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1877	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1878	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1879	#endif
1880
1881
1882	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1883	typedef enum IEMNATIVE_POSTPONED_EFL_OP_T : uint8_t
1884	{
1885	kIemNativePostponedEflOp_Invalid = 0,
1886	/** Logical operation.
1887	* Operands: result register.
1888	* @note This clears OF, CF and (undefined) AF, thus no need for inputs. */
1889	kIemNativePostponedEflOp_Logical,
1890	kIemNativePostponedEflOp_End
1891	} IEMNATIVE_POSTPONED_EFL_OP_T;
1892	#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
1893
1894	/**
1895	* Conditional stack entry.
1896	*/
1897	typedef struct IEMNATIVECOND
1898	{
1899	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1900	bool fInElse;
1901	union
1902	{
1903	RT_GCC_EXTENSION struct
1904	{
1905	/** Set if the if-block unconditionally exited the TB. */
1906	bool fIfExitTb;
1907	/** Set if the else-block unconditionally exited the TB. */
1908	bool fElseExitTb;
1909	};
1910	/** Indexed by fInElse. */
1911	bool afExitTb[2];
1912	};
1913	bool afPadding[5];
1914	/** The label for the IEM_MC_ELSE. */
1915	uint32_t idxLabelElse;
1916	/** The label for the IEM_MC_ENDIF. */
1917	uint32_t idxLabelEndIf;
1918	/** The initial state snapshot as the if-block starts executing. */
1919	IEMNATIVECORESTATE InitialState;
1920	/** The state snapshot at the end of the if-block. */
1921	IEMNATIVECORESTATE IfFinalState;
1922	} IEMNATIVECOND;
1923	/** Pointer to a condition stack entry. */
1924	typedef IEMNATIVECOND *PIEMNATIVECOND;
1925
1926
1927	/**
1928	* Native recompiler state.
1929	*/
1930	typedef struct IEMRECOMPILERSTATE
1931	{
1932	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1933	* IEMNATIVEINSTR units. */
1934	uint32_t cInstrBufAlloc;
1935	#ifdef VBOX_STRICT
1936	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1937	uint32_t offInstrBufChecked;
1938	#else
1939	uint32_t uPadding1; /* We don't keep track of the size here... */
1940	#endif
1941	/** Fixed temporary code buffer for native recompilation. */
1942	PIEMNATIVEINSTR pInstrBuf;
1943
1944	/** Bitmaps with the label types used. */
1945	uint64_t bmLabelTypes;
1946	/** Actual number of labels in paLabels. */
1947	uint32_t cLabels;
1948	/** Max number of entries allowed in paLabels before reallocating it. */
1949	uint32_t cLabelsAlloc;
1950	/** Labels defined while recompiling (referenced by fixups). */
1951	PIEMNATIVELABEL paLabels;
1952	/** Array with indexes of unique labels (uData always 0). */
1953	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1954
1955	/** Actual number of fixups paFixups. */
1956	uint32_t cFixups;
1957	/** Max number of entries allowed in paFixups before reallocating it. */
1958	uint32_t cFixupsAlloc;
1959	/** Buffer used by the recompiler for recording fixups when generating code. */
1960	PIEMNATIVEFIXUP paFixups;
1961
1962	/** Actual number of fixups in paTbExitFixups. */
1963	uint32_t cTbExitFixups;
1964	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1965	uint32_t cTbExitFixupsAlloc;
1966	/** Buffer used by the recompiler for recording fixups when generating code. */
1967	PIEMNATIVEEXITFIXUP paTbExitFixups;
1968
1969	#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1970	/** Statistics: The idxInstr+1 value at the last PC update. */
1971	uint8_t idxInstrPlusOneOfLastPcUpdate;
1972	#endif
1973
1974	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1975	/** Number of debug info entries allocated for pDbgInfo. */
1976	uint32_t cDbgInfoAlloc;
1977	/** Debug info. */
1978	PIEMTBDBG pDbgInfo;
1979	#endif
1980
1981	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1982	/** The current call index (liveness array and threaded calls in TB). */
1983	uint32_t idxCurCall;
1984	/** Number of liveness entries allocated. */
1985	uint32_t cLivenessEntriesAlloc;
1986	/** Liveness entries for all the calls in the TB begin recompiled.
1987	* The entry for idxCurCall contains the info for what the next call will
1988	* require wrt registers. (Which means the last entry is the initial liveness
1989	* state.) */
1990	PIEMLIVENESSENTRY paLivenessEntries;
1991	#endif
1992
1993	/** The translation block being recompiled. */
1994	PCIEMTB pTbOrg;
1995	/** The VMCPU structure of the EMT. */
1996	PVMCPUCC pVCpu;
1997
1998	/** Condition sequence number (for generating unique labels). */
1999	uint16_t uCondSeqNo;
2000	/** Check IRQ sequence number (for generating unique labels). */
2001	uint16_t uCheckIrqSeqNo;
2002	/** TLB load sequence number (for generating unique labels). */
2003	uint16_t uTlbSeqNo;
2004	/** The current condition stack depth (aCondStack). */
2005	uint8_t cCondDepth;
2006
2007	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
2008	uint8_t cArgsX;
2009	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
2010	uint32_t fCImpl;
2011	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
2012	uint32_t fMc;
2013	/** The expected IEMCPU::fExec value for the current call/instruction. */
2014	uint32_t fExec;
2015	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
2016	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
2017	*
2018	* This is an optimization because these control registers can only be changed from
2019	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
2020	* consisting of multiple SIMD instructions.
2021	*/
2022	uint32_t fSimdRaiseXcptChecksEmitted;
2023	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
2024	uint32_t idxLastCheckIrqCallNo;
2025	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
2026	uint32_t fSkippingEFlags;
2027	#endif
2028	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2029	struct
2030	{
2031	/** EFLAGS status bits that we're currently postponing the calculcation of. */
2032	uint32_t fEFlags;
2033	/** The postponed EFLAGS status bits calculation operation. */
2034	IEMNATIVE_POSTPONED_EFL_OP_T enmOp;
2035	/** The bit-width of the postponed EFLAGS calculation. */
2036	uint8_t cOpBits;
2037	/** Host register holding result or first source for the delayed operation,
2038	* UINT8_MAX if not in use. */
2039	uint8_t idxReg1;
2040	/** Host register holding second source for the delayed operation,
2041	* UINT8_MAX if not in use. */
2042	uint8_t idxReg2;
2043	# if defined(VBOX_WITH_STATISTICS) \|\| defined(IEMNATIVE_WITH_TB_DEBUG_INFO)
2044	/** Number of times the delayed calculation was emitted. */
2045	uint8_t cEmits;
2046	# endif
2047	} PostponedEfl;
2048	#endif
2049
2050	/** Core state requiring care with branches. */
2051	IEMNATIVECORESTATE Core;
2052
2053	/** The condition nesting stack. */
2054	IEMNATIVECOND aCondStack[2];
2055
2056	#ifndef IEM_WITH_THROW_CATCH
2057	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
2058	* for recompilation error handling. */
2059	jmp_buf JmpBuf;
2060	#endif
2061	} IEMRECOMPILERSTATE;
2062	/** Pointer to a native recompiler state. */
2063	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
2064
2065
2066	/** @def IEMNATIVE_TRY_SETJMP
2067	* Wrapper around setjmp / try, hiding all the ugly differences.
2068	*
2069	* @note Use with extreme care as this is a fragile macro.
2070	* @param a_pReNative The native recompile state.
2071	* @param a_rcTarget The variable that should receive the status code in case
2072	* of a longjmp/throw.
2073	*/
2074	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
2075	* Start wrapper for catch / setjmp-else.
2076	*
2077	* This will set up a scope.
2078	*
2079	* @note Use with extreme care as this is a fragile macro.
2080	* @param a_pReNative The native recompile state.
2081	* @param a_rcTarget The variable that should receive the status code in case
2082	* of a longjmp/throw.
2083	*/
2084	/** @def IEMNATIVE_CATCH_LONGJMP_END
2085	* End wrapper for catch / setjmp-else.
2086	*
2087	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
2088	* up the state.
2089	*
2090	* @note Use with extreme care as this is a fragile macro.
2091	* @param a_pReNative The native recompile state.
2092	*/
2093	/** @def IEMNATIVE_DO_LONGJMP
2094	*
2095	* Wrapper around longjmp / throw.
2096	*
2097	* @param a_pReNative The native recompile state.
2098	* @param a_rc The status code jump back with / throw.
2099	*/
2100	#ifdef IEM_WITH_THROW_CATCH
2101	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2102	a_rcTarget = VINF_SUCCESS; \
2103	try
2104	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2105	catch (int rcThrown) \
2106	{ \
2107	a_rcTarget = rcThrown
2108	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2109	} \
2110	((void)0)
2111	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
2112	#else /* !IEM_WITH_THROW_CATCH */
2113	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2114	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
2115	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2116	else \
2117	{ \
2118	((void)0)
2119	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2120	}
2121	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
2122	#endif /* !IEM_WITH_THROW_CATCH */
2123
2124
2125	/**
2126	* Native recompiler worker for a threaded function.
2127	*
2128	* @returns New code buffer offset; throws VBox status code in case of a failure.
2129	* @param pReNative The native recompiler state.
2130	* @param off The current code buffer offset.
2131	* @param pCallEntry The threaded call entry.
2132	*
2133	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
2134	*/
2135	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
2136	/** Pointer to a native recompiler worker for a threaded function. */
2137	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
2138
2139	/** Defines a native recompiler worker for a threaded function.
2140	* @see FNIEMNATIVERECOMPFUNC */
2141	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
2142	IEM_DECL_MSC_GUARD_IGNORE uint32_t VBOXCALL \
2143	a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
2144
2145	/** Prototypes a native recompiler function for a threaded function.
2146	* @see FNIEMNATIVERECOMPFUNC */
2147	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
2148
2149
2150	/**
2151	* Native recompiler liveness analysis worker for a threaded function.
2152	*
2153	* @param pCallEntry The threaded call entry.
2154	* @param pIncoming The incoming liveness state entry.
2155	* @param pOutgoing The outgoing liveness state entry.
2156	*/
2157	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
2158	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
2159	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
2160	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
2161
2162	/** Defines a native recompiler liveness analysis worker for a threaded function.
2163	* @see FNIEMNATIVELIVENESSFUNC */
2164	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
2165	IEM_DECL_MSC_GUARD_IGNORE DECLCALLBACK(void) \
2166	a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
2167
2168	/** Prototypes a native recompiler liveness analysis function for a threaded function.
2169	* @see FNIEMNATIVELIVENESSFUNC */
2170	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
2171
2172
2173	/** Define a native recompiler helper function, safe to call from the TB code. */
2174	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
2175	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2176	/** Prototype a native recompiler helper function, safe to call from the TB code. */
2177	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
2178	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2179	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
2180	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
2181	a_RetType (VBOXCALL *a_Name) a_ArgList
2182
2183
2184	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2185	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2186	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
2187	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
2188	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
2189	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2190	uint8_t idxHstSimdReg = UINT8_MAX,
2191	uint8_t idxHstSimdRegPrev = UINT8_MAX);
2192	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2193	uint8_t idxGstReg, uint8_t idxHstReg);
2194	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2195	uint64_t fGstReg);
2196	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
2197	uint64_t offPc, uint32_t cInstrSkipped);
2198	# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2199	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddPostponedEFlagsCalc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2200	IEMNATIVE_POSTPONED_EFL_OP_T enmOp, uint8_t cOpBits,
2201	uint8_t idxInstance);
2202	# endif
2203	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
2204
2205	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2206	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
2207	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
2208	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2209	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
2210	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
2211	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
2212	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere,
2213	IEMNATIVELABELTYPE enmExitReason);
2214	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
2215
2216	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2217	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2218	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2219	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpExPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2220	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm);
2221
2222	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2223	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2224	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWrite(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2225	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculation(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2226	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2227	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2228	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2229	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2230
2231	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2232	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2233	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2234	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2235	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2236	#endif
2237
2238	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2239	IEMNATIVEGSTREG enmGstReg);
2240	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2241	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2242	uint64_t fRead, uint64_t fWrite = 0);
2243	#else
2244	DECL_FORCE_INLINE_THROW(uint8_t)
2245	iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2246	uint64_t fRead, uint64_t fWrite = 0)
2247	{
2248	RT_NOREF(fRead, fWrite);
2249	return iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(pReNative, poff, kIemNativeGstReg_EFlags);
2250	}
2251	#endif
2252
2253	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
2254	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
2255	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
2256	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2257	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
2258	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2259	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
2260	#endif
2261	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2262	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2263	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2264	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
2265	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
2266	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2267	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
2268	#endif
2269	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
2270	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
2271	uint32_t fKeepVars = 0);
2272	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
2273	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
2274	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2275	uint32_t fHstRegsActiveShadows);
2276	#ifdef VBOX_STRICT
2277	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
2278	#endif
2279	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
2280	uint64_t fGstSimdShwExcept);
2281	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2282	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2283	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2284	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
2285	# endif
2286	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2287	#endif
2288	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2289	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
2290	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2291	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
2292	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2293	uint64_t fFlushGstReg = UINT64_MAX);
2294	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
2295	uint32_t off, uint8_t idxHstReg);
2296	#endif
2297
2298
2299	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
2300	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
2301	bool fPreferVolatile = true);
2302	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2303	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2304	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
2305	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
2306	bool fNoVolatileRegs = false);
2307	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
2308	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
2309	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2310	IEMNATIVEGSTSIMDREG enmGstSimdReg);
2311	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2312	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
2313	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2314
2315	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
2316	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
2317	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
2318	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
2319	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
2320	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
2321	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2322	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
2323	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2324	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
2325	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2326	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireSlow(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff);
2327	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireWithPrefSlow(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2328	uint32_t *poff, uint8_t idxRegPref);
2329	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireInitedSlow(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff);
2330	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireInitedWithPrefSlow(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2331	uint32_t *poff, uint8_t idxRegPref);
2332	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
2333	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
2334	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2335	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
2336	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2337	uint32_t fHstGprNotToSave);
2338	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2339	uint32_t fHstGprNotToSave);
2340	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2341	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
2342
2343	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2344	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2345	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowRegEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off,
2346	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2347	#ifdef VBOX_STRICT
2348	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
2349	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
2350	IEMNATIVEGSTREG enmGstReg);
2351	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheckEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf,
2352	uint32_t off, uint8_t idxReg, IEMNATIVEGSTREG enmGstReg);
2353	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
2354	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2355	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2356	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
2357	#endif
2358	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
2359	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
2360	#endif
2361	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
2362	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
2363	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
2364	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
2365	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
2366	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2367	PCIEMTHRDEDCALLENTRY pCallEntry);
2368	IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(iemNativeLivenessFunc_ThreadedCall);
2369	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
2370	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
2371
2372
2373	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
2374	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
2375	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
2376	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
2377	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
2378	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
2379	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
2380	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
2381	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
2382	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
2383	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
2384	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
2385	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
2386
2387	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2388	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2389	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2390	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2391	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2392	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2393	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2394	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2395	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2396	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2397	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2398	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2399	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2400	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2401	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2402	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
2403	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
2404	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
2405	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
2406	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2407	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2408	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2409	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2410	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2411	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2412	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2413	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2414	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2415	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2416	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2417
2418	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2419	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2420	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2421	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2422	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2423	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2424	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2425	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2426	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2427	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2428	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2429	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2430	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2431	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2432	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2433	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
2434	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2435	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2436	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2437	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2438	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2439	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2440	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2441	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2442	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2443	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2444	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2445	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2446	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2447	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2448
2449	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2450	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2451	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2452	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2453	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2454	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2455	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2456	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2457	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2458	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2459	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2460	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2461	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2462	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2463	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2464	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2465	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2466	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2467	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2468	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2469	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2470	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2471
2472	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2473	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2474	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2475	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2476	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2477	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2478	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2479	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2480	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2481	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2482	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2483	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2484	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2485	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2486	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2487	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2488	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2489	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2490	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2491	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2492	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2493	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2494
2495	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2496	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2497	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2498	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2499
2500
2501	/**
2502	* Info about shadowed guest register values.
2503	* @see IEMNATIVEGSTREG
2504	*/
2505	typedef struct IEMANTIVEGSTREGINFO
2506	{
2507	/** Offset in VMCPU. */
2508	uint32_t off;
2509	/** The field size. */
2510	uint8_t cb;
2511	/** Name (for logging). */
2512	const char *pszName;
2513	} IEMANTIVEGSTREGINFO;
2514	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2515	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2516	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2517	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2518	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2519
2520
2521
2522	/**
2523	* Ensures that there is sufficient space in the instruction output buffer.
2524	*
2525	* This will reallocate the buffer if needed and allowed.
2526	*
2527	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2528	* allocation size.
2529	*
2530	* @returns Pointer to the instruction output buffer on success; throws VBox
2531	* status code on failure, so no need to check it.
2532	* @param pReNative The native recompile state.
2533	* @param off Current instruction offset. Works safely for UINT32_MAX
2534	* as well.
2535	* @param cInstrReq Number of instruction about to be added. It's okay to
2536	* overestimate this a bit.
2537	*/
2538	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2539	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2540	{
2541	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2542	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2543	{
2544	#ifdef VBOX_STRICT
2545	pReNative->offInstrBufChecked = offChecked;
2546	#endif
2547	return pReNative->pInstrBuf;
2548	}
2549	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2550	}
2551
2552	/**
2553	* Checks that we didn't exceed the space requested in the last
2554	* iemNativeInstrBufEnsure() call.
2555	*/
2556	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2557	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2558	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2559
2560	/**
2561	* Checks that a variable index is valid.
2562	*/
2563	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2564	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2565	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2566	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2567	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2568	("%s=%#x\n", #a_idxVar, a_idxVar))
2569	#else
2570	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2571	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2572	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2573	#endif
2574
2575	/**
2576	* Checks that a variable index is valid and that the variable is assigned the
2577	* correct argument number.
2578	* This also adds a RT_NOREF of a_idxVar.
2579	*/
2580	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2581	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2582	RT_NOREF_PV(a_idxVar); \
2583	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2584	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2585	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2586	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2587	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2588	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2589	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2590	a_uArgNo)); \
2591	} while (0)
2592	#else
2593	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2594	RT_NOREF_PV(a_idxVar); \
2595	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2596	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2597	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2598	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2599	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2600	} while (0)
2601	#endif
2602
2603
2604	/**
2605	* Checks that a variable has the expected size.
2606	*/
2607	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2608	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2609	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2610	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2611
2612
2613	/**
2614	* Calculates the stack address of a variable as a [r]BP displacement value.
2615	*/
2616	DECL_FORCE_INLINE(int32_t)
2617	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2618	{
2619	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2620	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2621	}
2622
2623
2624	/**
2625	* Releases the variable's register.
2626	*
2627	* The register must have been previously acquired calling
2628	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2629	* iemNativeVarRegisterSetAndAcquire().
2630	*/
2631	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2632	{
2633	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2634	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2635	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2636	}
2637
2638
2639	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2640	{
2641	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2642	iemNativeVarRegisterRelease(pReNative, idxVar);
2643	}
2644
2645
2646	/**
2647	* Makes sure variable @a idxVar has a register assigned to it and that it stays
2648	* fixed till we call iemNativeVarRegisterRelease.
2649	*
2650	* @returns The host register number.
2651	* @param pReNative The recompiler state.
2652	* @param idxVar The variable.
2653	* @param poff Pointer to the instruction buffer offset.
2654	* In case a register needs to be freed up or the value
2655	* loaded off the stack.
2656	* @note Must not modify the host status flags!
2657	*/
2658	DECL_INLINE_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff)
2659	{
2660	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2661	PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
2662	Assert(pVar->cbVar <= 8);
2663	Assert(!pVar->fRegAcquired);
2664	uint8_t const idxReg = pVar->idxReg;
2665	if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
2666	{
2667	Assert( pVar->enmKind > kIemNativeVarKind_Invalid
2668	&& pVar->enmKind < kIemNativeVarKind_End);
2669	pVar->fRegAcquired = true;
2670	return idxReg;
2671	}
2672	return iemNativeVarRegisterAcquireSlow(pReNative, idxVar, poff);
2673	}
2674
2675
2676	/**
2677	* Makes sure variable @a idxVar has a register assigned to it and that it stays
2678	* fixed till we call iemNativeVarRegisterRelease.
2679	*
2680	* @returns The host register number.
2681	* @param pReNative The recompiler state.
2682	* @param idxVar The variable.
2683	* @param poff Pointer to the instruction buffer offset.
2684	* In case a register needs to be freed up or the value
2685	* loaded off the stack.
2686	* @param idxRegPref Preferred register number.
2687	* @note Must not modify the host status flags!
2688	*/
2689	DECL_INLINE_THROW(uint8_t)
2690	iemNativeVarRegisterAcquireWithPref(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff, uint8_t idxRegPref)
2691	{
2692	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2693	PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
2694	Assert(pVar->cbVar <= 8);
2695	Assert(!pVar->fRegAcquired);
2696	Assert(idxRegPref < RT_ELEMENTS(pReNative->Core.aHstRegs));
2697	uint8_t const idxReg = pVar->idxReg;
2698	if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
2699	{
2700	Assert( pVar->enmKind > kIemNativeVarKind_Invalid
2701	&& pVar->enmKind < kIemNativeVarKind_End);
2702	pVar->fRegAcquired = true;
2703	return idxReg;
2704	}
2705	return iemNativeVarRegisterAcquireWithPrefSlow(pReNative, idxVar, poff, idxRegPref);
2706	}
2707
2708
2709	/**
2710	* Makes sure variable @a idxVar has a register assigned to it and that it stays
2711	* fixed till we call iemNativeVarRegisterRelease.
2712	*
2713	* The variable must be initialized or VERR_IEM_VAR_NOT_INITIALIZED will be
2714	* thrown.
2715	*
2716	* @returns The host register number.
2717	* @param pReNative The recompiler state.
2718	* @param idxVar The variable.
2719	* @param poff Pointer to the instruction buffer offset.
2720	* In case a register needs to be freed up or the value
2721	* loaded off the stack.
2722	* @note Must not modify the host status flags!
2723	*/
2724	DECL_INLINE_THROW(uint8_t) iemNativeVarRegisterAcquireInited(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff)
2725	{
2726	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2727	PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
2728	Assert(pVar->cbVar <= 8);
2729	Assert(!pVar->fRegAcquired);
2730	uint8_t const idxReg = pVar->idxReg;
2731	if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
2732	{
2733	Assert( pVar->enmKind > kIemNativeVarKind_Invalid
2734	&& pVar->enmKind < kIemNativeVarKind_End);
2735	pVar->fRegAcquired = true;
2736	return idxReg;
2737	}
2738	return iemNativeVarRegisterAcquireInitedSlow(pReNative, idxVar, poff);
2739	}
2740
2741
2742	/**
2743	* Makes sure variable @a idxVar has a register assigned to it and that it stays
2744	* fixed till we call iemNativeVarRegisterRelease.
2745	*
2746	* The variable must be initialized or VERR_IEM_VAR_NOT_INITIALIZED will be
2747	* thrown.
2748	*
2749	* @returns The host register number.
2750	* @param pReNative The recompiler state.
2751	* @param idxVar The variable.
2752	* @param poff Pointer to the instruction buffer offset.
2753	* In case a register needs to be freed up or the value
2754	* loaded off the stack.
2755	* @param idxRegPref Preferred register number.
2756	* @note Must not modify the host status flags!
2757	*/
2758	DECL_INLINE_THROW(uint8_t)
2759	iemNativeVarRegisterAcquireInitedWithPref(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff, uint8_t idxRegPref)
2760	{
2761	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2762	PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
2763	Assert(pVar->cbVar <= 8);
2764	Assert(!pVar->fRegAcquired);
2765	Assert(idxRegPref < RT_ELEMENTS(pReNative->Core.aHstRegs));
2766	uint8_t const idxReg = pVar->idxReg;
2767	if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
2768	{
2769	Assert( pVar->enmKind > kIemNativeVarKind_Invalid
2770	&& pVar->enmKind < kIemNativeVarKind_End);
2771	pVar->fRegAcquired = true;
2772	return idxReg;
2773	}
2774	return iemNativeVarRegisterAcquireInitedWithPrefSlow(pReNative, idxVar, poff, idxRegPref);
2775	}
2776
2777
2778	/**
2779	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2780	*
2781	* @returns The flush mask.
2782	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2783	* @param fGstShwFlush The starting flush mask.
2784	*/
2785	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2786	{
2787	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2788	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2789	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2790	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2791	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2792	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2793	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2794	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2795	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2796	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2797	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2798	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2799	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2800	return fGstShwFlush;
2801	}
2802
2803
2804	/** Number of hidden arguments for CIMPL calls.
2805	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2806	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64))
2807	# define IEM_CIMPL_HIDDEN_ARGS 3
2808	#else
2809	# define IEM_CIMPL_HIDDEN_ARGS 2
2810	#endif
2811
2812
2813	/** Number of hidden arguments for SSE_AIMPL calls. */
2814	#define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2815	/** Number of hidden arguments for AVX_AIMPL calls. */
2816	#define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2817
2818
2819	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2820
2821	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2822	/**
2823	* Helper for iemNativeLivenessGetStateByGstReg.
2824	*
2825	* @returns IEMLIVENESS_STATE_XXX
2826	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2827	* ORed together.
2828	*/
2829	DECL_FORCE_INLINE(uint32_t)
2830	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2831	{
2832	/* INPUT trumps anything else. */
2833	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2834	return IEMLIVENESS_STATE_INPUT;
2835
2836	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2837	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2838	{
2839	/* If not all sub-fields are clobbered they must be considered INPUT. */
2840	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2841	return IEMLIVENESS_STATE_INPUT;
2842	return IEMLIVENESS_STATE_CLOBBERED;
2843	}
2844
2845	/* XCPT_OR_CALL trumps UNUSED. */
2846	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2847	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2848
2849	return IEMLIVENESS_STATE_UNUSED;
2850	}
2851	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2852
2853
2854	DECL_FORCE_INLINE(uint32_t)
2855	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2856	{
2857	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2858	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2859	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2860	# else
2861	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2862	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2863	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2864	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2865	# endif
2866	}
2867
2868
2869	DECL_FORCE_INLINE(uint32_t)
2870	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2871	{
2872	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2873	if (enmGstReg == kIemNativeGstReg_EFlags)
2874	{
2875	/* Merge the eflags states to one. */
2876	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2877	uRet = RT_BIT_32(uRet);
2878	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2879	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2880	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2881	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2882	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2883	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2884	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2885	# else
2886	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2887	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2888	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2889	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2890	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2891	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2892	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2893	# endif
2894	}
2895	return uRet;
2896	}
2897
2898	# ifdef VBOX_STRICT
2899
2900	/** For assertions only - caller checks that idxCurCall isn't zero. */
2901	DECL_FORCE_INLINE(uint32_t)
2902	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2903	{
2904	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2905	}
2906
2907
2908	/** For assertions only - caller checks that idxCurCall isn't zero. */
2909	DECL_FORCE_INLINE(uint32_t)
2910	iemNativeLivenessGetPrevStateByGstRegEx(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2911	{
2912	return iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2913	}
2914
2915	# endif /* VBOX_STRICT */
2916	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2917
2918
2919	/**
2920	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2921	*/
2922	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2923	{
2924	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2925	return IEM_CIMPL_HIDDEN_ARGS;
2926	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2927	return 1;
2928	return 0;
2929	}
2930
2931
2932	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2933	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2934	{
2935	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2936
2937	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2938	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2939	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2940	return (uint8_t)idxReg;
2941	}
2942
2943
2944
2945	/*********************************************************************************************************************************
2946	* Register Allocator (GPR) *
2947	*********************************************************************************************************************************/
2948
2949	#ifdef RT_ARCH_ARM64
2950	# include <iprt/armv8.h>
2951	#endif
2952
2953
2954	/**
2955	* Marks host register @a idxHstReg as containing a shadow copy of guest
2956	* register @a enmGstReg.
2957	*
2958	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2959	* host register before calling.
2960	*/
2961	DECL_FORCE_INLINE(void)
2962	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2963	{
2964	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2965	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2966	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2967
2968	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2969	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2970	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2971	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2972	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2973	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2974	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2975	#else
2976	RT_NOREF(off);
2977	#endif
2978	}
2979
2980
2981	/**
2982	* Clear any guest register shadow claims from @a idxHstReg.
2983	*
2984	* The register does not need to be shadowing any guest registers.
2985	*/
2986	DECL_FORCE_INLINE(void)
2987	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2988	{
2989	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2990	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2991	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2992	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2993	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2994	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2995	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2996	#endif
2997
2998	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2999	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
3000	if (fGstRegs)
3001	{
3002	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
3003	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3004	while (fGstRegs)
3005	{
3006	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
3007	fGstRegs &= ~RT_BIT_64(iGstReg);
3008	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
3009	}
3010	}
3011	#else
3012	RT_NOREF(off);
3013	#endif
3014
3015	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
3016	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
3017	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
3018	}
3019
3020
3021	/**
3022	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
3023	* and global overview flags.
3024	*/
3025	DECL_FORCE_INLINE(void)
3026	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
3027	{
3028	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3029	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
3030	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
3031	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3032	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
3033	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
3034	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
3035	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
3036	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
3037	#endif
3038
3039	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3040	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3041	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
3042	#else
3043	RT_NOREF(off);
3044	#endif
3045
3046	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
3047	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
3048	if (!fGstRegShadowsNew)
3049	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
3050	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
3051	}
3052
3053
3054	#if 0 /* unused */
3055	/**
3056	* Clear any guest register shadow claim for @a enmGstReg.
3057	*/
3058	DECL_FORCE_INLINE(void)
3059	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
3060	{
3061	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3062	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
3063	{
3064	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
3065	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
3066	}
3067	}
3068	#endif
3069
3070
3071	/**
3072	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
3073	* as the new shadow of it.
3074	*
3075	* Unlike the other guest reg shadow helpers, this does the logging for you.
3076	* However, it is the liveness state is not asserted here, the caller must do
3077	* that.
3078	*/
3079	DECL_FORCE_INLINE(void)
3080	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
3081	IEMNATIVEGSTREG enmGstReg, uint32_t off)
3082	{
3083	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3084	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
3085	{
3086	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
3087	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
3088	if (idxHstRegOld == idxHstRegNew)
3089	return;
3090	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
3091	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
3092	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
3093	}
3094	else
3095	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
3096	g_aGstShadowInfo[enmGstReg].pszName));
3097	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
3098	}
3099
3100
3101	/**
3102	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
3103	* to @a idxRegTo.
3104	*/
3105	DECL_FORCE_INLINE(void)
3106	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
3107	IEMNATIVEGSTREG enmGstReg, uint32_t off)
3108	{
3109	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
3110	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
3111	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
3112	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
3113	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3114	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
3115	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
3116	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
3117	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
3118
3119	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
3120	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3121	if (!fGstRegShadowsFrom)
3122	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
3123	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
3124	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
3125	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
3126	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3127	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3128	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
3129	#else
3130	RT_NOREF(off);
3131	#endif
3132	}
3133
3134
3135	/**
3136	* Flushes any delayed guest register writes.
3137	*
3138	* This must be called prior to calling CImpl functions and any helpers that use
3139	* the guest state (like raising exceptions) and such.
3140	*
3141	* This optimization has not yet been implemented. The first target would be
3142	* RIP updates, since these are the most common ones.
3143	*
3144	* @note This function does not flush any shadowing information for guest
3145	* registers. This needs to be done by the caller if it wishes to do so.
3146	*/
3147	DECL_INLINE_THROW(uint32_t)
3148	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
3149	uint64_t fGstSimdShwExcept = 0)
3150	{
3151	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3152	uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
3153	#else
3154	uint64_t const fWritebackPc = 0;
3155	#endif
3156	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
3157	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
3158	#else
3159	uint64_t const bmGstRegShadowDirty = 0;
3160	#endif
3161	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
3162	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
3163	& ~fGstSimdShwExcept;
3164	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
3165	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
3166
3167	return off;
3168	}
3169
3170
3171	/**
3172	* Allocates a temporary host general purpose register for keeping a guest
3173	* register value.
3174	*
3175	* Since we may already have a register holding the guest register value,
3176	* code will be emitted to do the loading if that's not the case. Code may also
3177	* be emitted if we have to free up a register to satify the request.
3178	*
3179	* @returns The host register number; throws VBox status code on failure, so no
3180	* need to check the return value.
3181	* @param pReNative The native recompile state.
3182	* @param poff Pointer to the variable with the code buffer
3183	* position. This will be update if we need to move a
3184	* variable from register to stack in order to satisfy
3185	* the request.
3186	* @param enmGstReg The guest register that will is to be updated.
3187	* @param enmIntendedUse How the caller will be using the host register.
3188	* @param fNoVolatileRegs Set if no volatile register allowed, clear if any
3189	* register is okay (default). The ASSUMPTION here is
3190	* that the caller has already flushed all volatile
3191	* registers, so this is only applied if we allocate a
3192	* new register.
3193	* @sa iemNativeRegAllocTmpForGuestEFlags
3194	* iemNativeRegAllocTmpForGuestRegIfAlreadyPresent
3195	* iemNativeRegAllocTmpForGuestRegInt
3196	*/
3197	DECL_FORCE_INLINE_THROW(uint8_t)
3198	iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg,
3199	IEMNATIVEGSTREGUSE const enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
3200	bool const fNoVolatileRegs = false)
3201	{
3202	if (enmIntendedUse == kIemNativeGstRegUse_ReadOnly)
3203	return !fNoVolatileRegs
3204	? iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, enmGstReg)
3205	: iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(pReNative, poff, enmGstReg);
3206	if (enmIntendedUse == kIemNativeGstRegUse_ForUpdate)
3207	return !fNoVolatileRegs
3208	? iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, enmGstReg)
3209	: iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(pReNative, poff, enmGstReg);
3210	if (enmIntendedUse == kIemNativeGstRegUse_ForFullWrite)
3211	return !fNoVolatileRegs
3212	? iemNativeRegAllocTmpForGuestRegFullWrite(pReNative, poff, enmGstReg)
3213	: iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(pReNative, poff, enmGstReg);
3214	Assert(enmIntendedUse == kIemNativeGstRegUse_Calculation);
3215	return !fNoVolatileRegs
3216	? iemNativeRegAllocTmpForGuestRegCalculation(pReNative, poff, enmGstReg)
3217	: iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(pReNative, poff, enmGstReg);
3218	}
3219
3220	#if !defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| !defined(VBOX_STRICT)
3221
3222	DECL_FORCE_INLINE_THROW(uint8_t)
3223	iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3224	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3225	{
3226	RT_NOREF(fRead, fWrite, fPotentialCall);
3227	return iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, kIemNativeGstReg_EFlags);
3228	}
3229
3230	DECL_FORCE_INLINE_THROW(uint8_t)
3231	iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3232	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3233	{
3234	RT_NOREF(fRead, fWrite, fPotentialCall);
3235	return iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, kIemNativeGstReg_EFlags);
3236	}
3237
3238	#endif
3239
3240
3241
3242	/*********************************************************************************************************************************
3243	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
3244	*********************************************************************************************************************************/
3245
3246	DECL_FORCE_INLINE(uint8_t)
3247	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
3248	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
3249	{
3250	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
3251
3252	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
3253	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
3254	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
3255	return idxSimdReg;
3256	}
3257
3258
3259	/**
3260	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
3261	* SIMD register @a enmGstSimdReg.
3262	*
3263	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
3264	* host register before calling.
3265	*/
3266	DECL_FORCE_INLINE(void)
3267	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
3268	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3269	{
3270	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
3271	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
3272	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
3273
3274	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
3275	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3276	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
3277	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
3278	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3279	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3280	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
3281	#else
3282	RT_NOREF(off);
3283	#endif
3284	}
3285
3286
3287	/**
3288	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
3289	* to @a idxSimdRegTo.
3290	*/
3291	DECL_FORCE_INLINE(void)
3292	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
3293	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3294	{
3295	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
3296	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
3297	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
3298	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
3299	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3300	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
3301	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
3302	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
3303	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
3304	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
3305	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
3306
3307	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
3308	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3309	if (!fGstRegShadowsFrom)
3310	{
3311	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
3312	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3313	}
3314	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
3315	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3316	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
3317	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3318	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3319	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
3320	#else
3321	RT_NOREF(off);
3322	#endif
3323	}
3324
3325
3326	/**
3327	* Clear any guest register shadow claims from @a idxHstSimdReg.
3328	*
3329	* The register does not need to be shadowing any guest registers.
3330	*/
3331	DECL_FORCE_INLINE(void)
3332	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
3333	{
3334	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
3335	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
3336	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
3337	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
3338	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
3339	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
3340	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
3341
3342	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3343	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3344	if (fGstRegs)
3345	{
3346	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
3347	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3348	while (fGstRegs)
3349	{
3350	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
3351	fGstRegs &= ~RT_BIT_64(iGstReg);
3352	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
3353	}
3354	}
3355	#else
3356	RT_NOREF(off);
3357	#endif
3358
3359	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
3360	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3361	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
3362	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3363	}
3364
3365
3366
3367	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3368	/**
3369	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
3370	*/
3371	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
3372	{
3373	if (pReNative->Core.offPc)
3374	return iemNativeEmitPcWritebackSlow(pReNative, off);
3375	return off;
3376	}
3377	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
3378
3379
3380	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
3381	* it saves us the trouble of a hidden parameter on MSC/amd64. */
3382	#ifdef RT_ARCH_AMD64
3383	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
3384	#elif defined(RT_ARCH_ARM64)
3385	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
3386	#endif
3387
3388	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
3389	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
3390	#endif
3391
3392	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3393
3394	/** @} */
3395
3396	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3397

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 107623

Download in other formats: