IEMN8veRecompiler.h@ 104797

Last change on this file since 104797 was 104797, checked in by vboxsync, 11 months ago
VMM/IEM: Introduce special helpers for generating code to exit a TB in order to be able to experiment with different approaches more easily and convert the code emitters to make use of them, bugref:10677
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 129.2 KB

Line
1	/* $Id: IEMN8veRecompiler.h 104797 2024-05-28 05:50:30Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	/# define IEMLIVENESS_EXTENDED_LAYOUT/
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61
62	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
63	* Enables strict consistency checks around EFLAGS skipping.
64	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
65	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
66	# ifdef VBOX_STRICT
67	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
68	# endif
69	#elif defined(DOXYGEN_RUNNING)
70	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
71	#endif
72
73	#ifdef VBOX_WITH_STATISTICS
74	/** Always count instructions for now. */
75	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
76	#endif
77
78	/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
79	* Enables having only a single prologue for native TBs. */
80	#if 1 \|\| defined(DOXYGEN_RUNNING)
81	# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
82	#endif
83
84
85	/** @name Stack Frame Layout
86	*
87	* @{ */
88	/** The size of the area for stack variables and spills and stuff.
89	* @note This limit is duplicated in the python script(s). We add 0x40 for
90	* alignment padding. */
91	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
92	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
93	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
94	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
95
96	#ifdef RT_ARCH_AMD64
97	/** An stack alignment adjustment (between non-volatile register pushes and
98	* the stack variable area, so the latter better aligned). */
99	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
100
101	/** Number of stack arguments slots for calls made from the frame. */
102	# ifdef RT_OS_WINDOWS
103	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
104	# else
105	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
106	# endif
107	/** Number of any shadow arguments (spill area) for calls we make. */
108	# ifdef RT_OS_WINDOWS
109	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
110	# else
111	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
112	# endif
113
114	/** Frame pointer (RBP) relative offset of the last push. */
115	# ifdef RT_OS_WINDOWS
116	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
117	# else
118	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
119	# endif
120	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
121	* address for it). */
122	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
123	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
124	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
125	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
126	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
127	# ifdef RT_OS_WINDOWS
128	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
129	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
130	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
131	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
132	# endif
133
134	# ifdef RT_OS_WINDOWS
135	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
136	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
137	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
138	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
139	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
140	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
141	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
142	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
143	# endif
144
145	#elif RT_ARCH_ARM64
146	/** No alignment padding needed for arm64. */
147	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
148	/** No stack argument slots, got 8 registers for arguments will suffice. */
149	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
150	/** There are no argument spill area. */
151	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
152
153	/** Number of saved registers at the top of our stack frame.
154	* This includes the return address and old frame pointer, so x19 thru x30. */
155	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
156	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
157	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
158
159	/** Frame pointer (BP) relative offset of the last push. */
160	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
161
162	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
163	* address for it). */
164	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
165
166	#else
167	# error "port me"
168	#endif
169	/** @} */
170
171
172	/** @name Fixed Register Allocation(s)
173	* @{ */
174	/** @def IEMNATIVE_REG_FIXED_PVMCPU
175	* The number of the register holding the pVCpu pointer. */
176	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
177	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
178	* @note This not available on AMD64, only ARM64. */
179	/** @def IEMNATIVE_REG_FIXED_TMP0
180	* Dedicated temporary register.
181	* @todo replace this by a register allocator and content tracker. */
182	/** @def IEMNATIVE_REG_FIXED_MASK
183	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
184	* architecture. */
185	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
186	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
187	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
188	* architecture. */
189	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
190	* Dedicated temporary SIMD register. */
191	#endif
192	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
193	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
194	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
195	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
196	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
197	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
198	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
199	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
200	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
201	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
202	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
203	# else
204	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
205	# endif
206	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
207	\| RT_BIT_32(ARMV8_A64_REG_LR) \
208	\| RT_BIT_32(ARMV8_A64_REG_BP) \
209	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
210	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
211	\| RT_BIT_32(ARMV8_A64_REG_X18) \
212	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
213	\| IEMNATIVE_REG_FIXED_MASK_ADD)
214
215	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
216	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
217	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
218	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
219	# else
220	/** @note
221	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
222	* support emulating 256-bit registers we pair two real registers statically to
223	* one virtual for now, leaving us with only 16 256-bit registers. We always
224	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
225	* the register allocator assumes that it will be always free when the lower is
226	* picked.
227	*
228	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
229	* touch them in order to avoid having to save and restore them in the
230	* prologue/epilogue.
231	*/
232	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
233	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
234	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
235	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
236	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
237	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
238	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
239	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
240	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q1))
250	# endif
251	# endif
252
253	#elif defined(RT_ARCH_AMD64)
254	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
255	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
256	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
257	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
258	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
259	\| RT_BIT_32(X86_GREG_xSP) \
260	\| RT_BIT_32(X86_GREG_xBP) )
261
262	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
263	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
264	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
265	# ifndef _MSC_VER
266	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
267	# endif
268	# endif
269	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
270	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
271	# else
272	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
273	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
274	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
275	# endif
276	# endif
277
278	#else
279	# error "port me"
280	#endif
281	/** @} */
282
283	/** @name Call related registers.
284	* @{ */
285	/** @def IEMNATIVE_CALL_RET_GREG
286	* The return value register. */
287	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
288	* Number of arguments in registers. */
289	/** @def IEMNATIVE_CALL_ARG0_GREG
290	* The general purpose register carrying argument \#0. */
291	/** @def IEMNATIVE_CALL_ARG1_GREG
292	* The general purpose register carrying argument \#1. */
293	/** @def IEMNATIVE_CALL_ARG2_GREG
294	* The general purpose register carrying argument \#2. */
295	/** @def IEMNATIVE_CALL_ARG3_GREG
296	* The general purpose register carrying argument \#3. */
297	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
298	* Mask of registers the callee will not save and may trash. */
299	#ifdef RT_ARCH_AMD64
300	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
301
302	# ifdef RT_OS_WINDOWS
303	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
304	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
305	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
306	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
307	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
308	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
309	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
310	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
311	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
312	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
313	\| RT_BIT_32(X86_GREG_xCX) \
314	\| RT_BIT_32(X86_GREG_xDX) \
315	\| RT_BIT_32(X86_GREG_x8) \
316	\| RT_BIT_32(X86_GREG_x9) \
317	\| RT_BIT_32(X86_GREG_x10) \
318	\| RT_BIT_32(X86_GREG_x11) )
319	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
320	/* xmm0 - xmm5 are marked as volatile. */
321	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
322	# endif
323
324	# else /* !RT_OS_WINDOWS */
325	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
326	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
327	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
328	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
329	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
330	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
331	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
332	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
333	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
334	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
335	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
336	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
337	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
338	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
339	\| RT_BIT_32(X86_GREG_xCX) \
340	\| RT_BIT_32(X86_GREG_xDX) \
341	\| RT_BIT_32(X86_GREG_xDI) \
342	\| RT_BIT_32(X86_GREG_xSI) \
343	\| RT_BIT_32(X86_GREG_x8) \
344	\| RT_BIT_32(X86_GREG_x9) \
345	\| RT_BIT_32(X86_GREG_x10) \
346	\| RT_BIT_32(X86_GREG_x11) )
347	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
348	/* xmm0 - xmm15 are marked as volatile. */
349	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
350	# endif
351	# endif /* !RT_OS_WINDOWS */
352
353	#elif defined(RT_ARCH_ARM64)
354	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
355	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
356	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
357	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
358	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
359	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
360	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
361	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
362	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
363	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
364	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
365	\| RT_BIT_32(ARMV8_A64_REG_X1) \
366	\| RT_BIT_32(ARMV8_A64_REG_X2) \
367	\| RT_BIT_32(ARMV8_A64_REG_X3) \
368	\| RT_BIT_32(ARMV8_A64_REG_X4) \
369	\| RT_BIT_32(ARMV8_A64_REG_X5) \
370	\| RT_BIT_32(ARMV8_A64_REG_X6) \
371	\| RT_BIT_32(ARMV8_A64_REG_X7) )
372	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373	\| RT_BIT_32(ARMV8_A64_REG_X1) \
374	\| RT_BIT_32(ARMV8_A64_REG_X2) \
375	\| RT_BIT_32(ARMV8_A64_REG_X3) \
376	\| RT_BIT_32(ARMV8_A64_REG_X4) \
377	\| RT_BIT_32(ARMV8_A64_REG_X5) \
378	\| RT_BIT_32(ARMV8_A64_REG_X6) \
379	\| RT_BIT_32(ARMV8_A64_REG_X7) \
380	\| RT_BIT_32(ARMV8_A64_REG_X8) \
381	\| RT_BIT_32(ARMV8_A64_REG_X9) \
382	\| RT_BIT_32(ARMV8_A64_REG_X10) \
383	\| RT_BIT_32(ARMV8_A64_REG_X11) \
384	\| RT_BIT_32(ARMV8_A64_REG_X12) \
385	\| RT_BIT_32(ARMV8_A64_REG_X13) \
386	\| RT_BIT_32(ARMV8_A64_REG_X14) \
387	\| RT_BIT_32(ARMV8_A64_REG_X15) \
388	\| RT_BIT_32(ARMV8_A64_REG_X16) \
389	\| RT_BIT_32(ARMV8_A64_REG_X17) )
390	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
391	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
392	* so to simplify our life a bit we just mark everything as volatile. */
393	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
394	# endif
395
396	#endif
397
398	/** This is the maximum argument count we'll ever be needing. */
399	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
400	#ifdef RT_OS_WINDOWS
401	# ifdef VBOXSTRICTRC_STRICT_ENABLED
402	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
403	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
404	# endif
405	#endif
406	/** @} */
407
408
409	/** @def IEMNATIVE_HST_GREG_COUNT
410	* Number of host general purpose registers we tracker. */
411	/** @def IEMNATIVE_HST_GREG_MASK
412	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
413	* inverted register masks and such to get down to a correct set of regs. */
414	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
415	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
416	* Number of host SIMD registers we track. */
417	/** @def IEMNATIVE_HST_SIMD_REG_MASK
418	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
419	* inverted register masks and such to get down to a correct set of regs. */
420	#endif
421	#ifdef RT_ARCH_AMD64
422	# define IEMNATIVE_HST_GREG_COUNT 16
423	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
424
425	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
426	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
427	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
428	# endif
429
430	#elif defined(RT_ARCH_ARM64)
431	# define IEMNATIVE_HST_GREG_COUNT 32
432	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
433
434	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
435	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
436	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
437	# endif
438
439	#else
440	# error "Port me!"
441	#endif
442
443
444	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
445
446
447	/** TB exit reasons. */
448	typedef enum
449	{
450	kIemNativeExitReason_Invalid = 0,
451	kIemNativeExitReason_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
452	kIemNativeExitReason_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
453	kIemNativeExitReason_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
454	kIemNativeExitReason_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
455	kIemNativeExitReason_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
456	kIemNativeExitReason_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
457	kIemNativeExitReason_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
458	kIemNativeExitReason_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
459	kIemNativeExitReason_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
460	kIemNativeExitReason_ObsoleteTb,
461	kIemNativeExitReason_NeedCsLimChecking,
462	kIemNativeExitReason_CheckBranchMiss,
463	kIemNativeExitReason_Return, /** @todo Eliminate (needed for the compile assertion below). */
464	kIemNativeExitReason_ReturnBreak,
465	kIemNativeExitReason_ReturnBreakFF,
466	kIemNativeExitReason_ReturnBreakViaLookup,
467	kIemNativeExitReason_ReturnBreakViaLookupWithIrq,
468	kIemNativeExitReason_ReturnBreakViaLookupWithTlb,
469	kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq,
470	kIemNativeExitReason_ReturnWithFlags,
471	kIemNativeExitReason_NonZeroRetOrPassUp,
472	} IEMNATIVEEXITREASON;
473
474
475	/** Native code generator label types. */
476	typedef enum
477	{
478	kIemNativeLabelType_Invalid = 0,
479	/*
480	* Labels w/o data, only once instance per TB.
481	*
482	* Note! Jumps to these requires instructions that are capable of spanning
483	* the max TB length.
484	*/
485	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
486	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
487	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
488	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
489	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
490	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
491	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
492	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
493	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
494	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
495	kIemNativeLabelType_ObsoleteTb,
496	kIemNativeLabelType_NeedCsLimChecking,
497	kIemNativeLabelType_CheckBranchMiss,
498	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
499	/* Manually defined labels. */
500	kIemNativeLabelType_Return,
501	kIemNativeLabelType_ReturnBreak,
502	kIemNativeLabelType_ReturnBreakFF,
503	kIemNativeLabelType_ReturnBreakViaLookup,
504	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
505	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
506	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
507	kIemNativeLabelType_ReturnWithFlags,
508	kIemNativeLabelType_NonZeroRetOrPassUp,
509	/** The last fixup for branches that can span almost the whole TB length. */
510	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
511
512	/*
513	* Labels with data, potentially multiple instances per TB:
514	*
515	* These are localized labels, so no fixed jump type restrictions here.
516	*/
517	kIemNativeLabelType_FirstWithMultipleInstances,
518	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
519	kIemNativeLabelType_Else,
520	kIemNativeLabelType_Endif,
521	kIemNativeLabelType_CheckIrq,
522	kIemNativeLabelType_TlbLookup,
523	kIemNativeLabelType_TlbMiss,
524	kIemNativeLabelType_TlbDone,
525	kIemNativeLabelType_End
526	} IEMNATIVELABELTYPE;
527
528	/* Temporary kludge until all jumps to TB exit labels are converted to the new TB exiting style,
529	* see @bugref{10677}. */
530	#define IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(a_Reason) \
531	((int)kIemNativeLabelType_ ## a_Reason == (int)kIemNativeExitReason_ ## a_Reason)
532	AssertCompile( IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseDe)
533	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseUd)
534	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseRelated)
535	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseAvxRelated)
536	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseAvxFpRelated)
537	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseNm)
538	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseGp0)
539	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseMf)
540	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseXf)
541	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ObsoleteTb)
542	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NeedCsLimChecking)
543	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(CheckBranchMiss)
544	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(Return)
545	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreak)
546	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakFF)
547	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookup)
548	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithIrq)
549	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlb)
550	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlbAndIrq)
551	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnWithFlags)
552	&& IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NonZeroRetOrPassUp));
553
554
555	/** Native code generator label definition. */
556	typedef struct IEMNATIVELABEL
557	{
558	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
559	* the epilog. */
560	uint32_t off;
561	/** The type of label (IEMNATIVELABELTYPE). */
562	uint16_t enmType;
563	/** Additional label data, type specific. */
564	uint16_t uData;
565	} IEMNATIVELABEL;
566	/** Pointer to a label. */
567	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
568
569
570	/** Native code generator fixup types. */
571	typedef enum
572	{
573	kIemNativeFixupType_Invalid = 0,
574	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
575	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
576	kIemNativeFixupType_Rel32,
577	#elif defined(RT_ARCH_ARM64)
578	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
579	kIemNativeFixupType_RelImm26At0,
580	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
581	kIemNativeFixupType_RelImm19At5,
582	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
583	kIemNativeFixupType_RelImm14At5,
584	#endif
585	kIemNativeFixupType_End
586	} IEMNATIVEFIXUPTYPE;
587
588	/** Native code generator fixup. */
589	typedef struct IEMNATIVEFIXUP
590	{
591	/** Code offset of the fixup location. */
592	uint32_t off;
593	/** The IEMNATIVELABEL this is a fixup for. */
594	uint16_t idxLabel;
595	/** The fixup type (IEMNATIVEFIXUPTYPE). */
596	uint8_t enmType;
597	/** Addend or other data. */
598	int8_t offAddend;
599	} IEMNATIVEFIXUP;
600	/** Pointer to a native code generator fixup. */
601	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
602
603
604	/**
605	* One bit of the state.
606	*
607	* Each register state takes up two bits. We keep the two bits in two separate
608	* 64-bit words to simplify applying them to the guest shadow register mask in
609	* the register allocator.
610	*/
611	typedef union IEMLIVENESSBIT
612	{
613	uint64_t bm64;
614	RT_GCC_EXTENSION struct
615	{ /* bit no */
616	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
617	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
618	uint64_t fCr0 : 1; /*< 0x11 / 17: /
619	uint64_t fFcw : 1; /*< 0x12 / 18: /
620	uint64_t fFsw : 1; /*< 0x13 / 19: /
621	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
622	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
623	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
624	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
625	uint64_t fCr4 : 1; /*< 0x2c / 44: /
626	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
627	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
628	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
629	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
630	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
631	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
632	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
633	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
634	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
635	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
636	};
637	} IEMLIVENESSBIT;
638	AssertCompileSize(IEMLIVENESSBIT, 8);
639
640	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
641	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
642	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
643	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
644	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
645	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
646	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
647
648
649	/**
650	* A liveness state entry.
651	*
652	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
653	* Once we add a SSE register shadowing, we'll add another 64-bit element for
654	* that.
655	*/
656	typedef union IEMLIVENESSENTRY
657	{
658	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
659	uint64_t bm64[16 / 8];
660	uint16_t bm32[16 / 4];
661	uint16_t bm16[16 / 2];
662	uint8_t bm8[ 16 / 1];
663	IEMLIVENESSBIT aBits[2];
664	#else
665	uint64_t bm64[32 / 8];
666	uint16_t bm32[32 / 4];
667	uint16_t bm16[32 / 2];
668	uint8_t bm8[ 32 / 1];
669	IEMLIVENESSBIT aBits[4];
670	#endif
671	RT_GCC_EXTENSION struct
672	{
673	/** Bit \#0 of the register states. */
674	IEMLIVENESSBIT Bit0;
675	/** Bit \#1 of the register states. */
676	IEMLIVENESSBIT Bit1;
677	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
678	/** Bit \#2 of the register states. */
679	IEMLIVENESSBIT Bit2;
680	/** Bit \#3 of the register states. */
681	IEMLIVENESSBIT Bit3;
682	#endif
683	};
684	} IEMLIVENESSENTRY;
685	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
686	AssertCompileSize(IEMLIVENESSENTRY, 16);
687	#else
688	AssertCompileSize(IEMLIVENESSENTRY, 32);
689	#endif
690	/** Pointer to a liveness state entry. */
691	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
692	/** Pointer to a const liveness state entry. */
693	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
694
695	/** @name 64-bit value masks for IEMLIVENESSENTRY.
696	* @{ / / 0xzzzzyyyyxxxxwwww */
697	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
698
699	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
700	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
701	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
702
703	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
704	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
705	#endif
706
707	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
708	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
709
710	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
711	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
712	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
713	#endif
714	/** @} */
715
716
717	/** @name The liveness state for a register.
718	*
719	* The state values have been picked to with state accumulation in mind (what
720	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
721	* performance critical work done with the values.
722	*
723	* This is a compressed state that only requires 2 bits per register.
724	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
725	* 1. the incoming state from the following call,
726	* 2. the outgoing state for this call,
727	* 3. mask of the entries set in the 2nd.
728	*
729	* The mask entry (3rd one above) will be used both when updating the outgoing
730	* state and when merging in incoming state for registers not touched by the
731	* current call.
732	*
733	* @{ */
734	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
735	/** The register will be clobbered and the current value thrown away.
736	*
737	* When this is applied to the state (2) we'll simply be AND'ing it with the
738	* (old) mask (3) and adding the register to the mask. This way we'll
739	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
740	* IEMLIVENESS_STATE_INPUT states. */
741	# define IEMLIVENESS_STATE_CLOBBERED 0
742	/** The register is unused in the remainder of the TB.
743	*
744	* This is an initial state and can not be set by any of the
745	* iemNativeLivenessFunc_xxxx callbacks. */
746	# define IEMLIVENESS_STATE_UNUSED 1
747	/** The register value is required in a potential call or exception.
748	*
749	* This means that the register value must be calculated and is best written to
750	* the state, but that any shadowing registers can be flushed thereafter as it's
751	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
752	*
753	* It is typically applied across the board, but we preserve incoming
754	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
755	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
756	* 1. r0 = old & ~mask;
757	* 2. r0 = t1 & (t1 >> 1)'
758	* 3. state \|= r0 \| 0b10;
759	* 4. mask = ~0;
760	*/
761	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
762	/** The register value is used as input.
763	*
764	* This means that the register value must be calculated and it is best to keep
765	* it in a register. It does not need to be writtent out as such. This is the
766	* highest priority state.
767	*
768	* Whether the call modifies the register or not isn't relevant to earlier
769	* calls, so that's not recorded.
770	*
771	* When applying this state we just or in the value in the outgoing state and
772	* mask. */
773	# define IEMLIVENESS_STATE_INPUT 3
774	/** Mask of the state bits. */
775	# define IEMLIVENESS_STATE_MASK 3
776	/** The number of bits per state. */
777	# define IEMLIVENESS_STATE_BIT_COUNT 2
778	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
779	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
780	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
781	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
782	/** Check if a register clobbering is expected given the (previous) liveness state.
783	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
784	* include INPUT if the register is used in more than one place. */
785	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
786
787	/** Check if all status flags are going to be clobbered and doesn't need
788	* calculating in the current step.
789	* @param a_pCurEntry The current liveness entry. */
790	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
791	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
792
793	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
794	/** The register is not used any more. */
795	# define IEMLIVENESS_STATE_UNUSED 0
796	/** Flag: The register is required in a potential exception or call. */
797	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
798	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
799	/** Flag: The register is read. */
800	# define IEMLIVENESS_STATE_READ 2
801	# define IEMLIVENESS_BIT_READ 1
802	/** Flag: The register is written. */
803	# define IEMLIVENESS_STATE_WRITE 4
804	# define IEMLIVENESS_BIT_WRITE 2
805	/** Flag: Unconditional call (not needed, can be redefined for research). */
806	# define IEMLIVENESS_STATE_CALL 8
807	# define IEMLIVENESS_BIT_CALL 3
808	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
809	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
810	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
811	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
812	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
813
814	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
815	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
816	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
817	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
818
819	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
820	/** @} */
821
822	/** @name Liveness helpers for builtin functions and similar.
823	*
824	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
825	* own set of manimulator macros for those.
826	*
827	* @{ */
828	/** Initializing the state as all unused. */
829	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
830	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
831	do { \
832	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
833	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
834	} while (0)
835	#else
836	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
837	do { \
838	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
839	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
840	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
841	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
842	} while (0)
843	#endif
844
845	/** Initializing the outgoing state with a potential xcpt or call state.
846	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
847	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
848	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
849	do { \
850	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
851	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
852	} while (0)
853	#else
854	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
855	do { \
856	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
857	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
858	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
859	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
860	} while (0)
861	#endif
862
863	/** Adds a segment base register as input to the outgoing state. */
864	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
865	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
866	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
867	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
868	} while (0)
869	#else
870	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
871	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
872	} while (0)
873	#endif
874
875	/** Adds a segment attribute register as input to the outgoing state. */
876	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
877	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
878	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
879	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
880	} while (0)
881	#else
882	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
883	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
884	} while (0)
885	#endif
886
887	/** Adds a segment limit register as input to the outgoing state. */
888	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
889	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
890	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
891	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
892	} while (0)
893	#else
894	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
895	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
896	} while (0)
897	#endif
898
899	/** Adds a segment limit register as input to the outgoing state. */
900	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
901	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
902	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
903	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
904	} while (0)
905	#else
906	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
907	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
908	} while (0)
909	#endif
910	/** @} */
911
912	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
913	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
914	* calculated and up to date. This is to double check that we haven't skipped
915	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
916	* @note has to be placed in
917	*/
918	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
919	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
920	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
921	#else
922	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
923	#endif
924
925
926	/**
927	* Guest registers that can be shadowed in GPRs.
928	*
929	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
930	* must be placed last, as the liveness state tracks it as 7 subcomponents and
931	* we don't want to waste space here.
932	*
933	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
934	* friends as well as IEMAllN8veLiveness.cpp.
935	*/
936	typedef enum IEMNATIVEGSTREG : uint8_t
937	{
938	kIemNativeGstReg_GprFirst = 0,
939	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
940	kIemNativeGstReg_Pc,
941	kIemNativeGstReg_Cr0,
942	kIemNativeGstReg_FpuFcw,
943	kIemNativeGstReg_FpuFsw,
944	kIemNativeGstReg_SegBaseFirst,
945	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
946	kIemNativeGstReg_SegAttribFirst,
947	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
948	kIemNativeGstReg_SegLimitFirst,
949	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
950	kIemNativeGstReg_SegSelFirst,
951	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
952	kIemNativeGstReg_Cr4,
953	kIemNativeGstReg_Xcr0,
954	kIemNativeGstReg_MxCsr,
955	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
956	kIemNativeGstReg_End
957	} IEMNATIVEGSTREG;
958	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
959	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
960
961	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
962	* @{ */
963	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
964	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
965	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
966	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
967	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
968	/** @} */
969
970	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
971
972	/**
973	* Guest registers that can be shadowed in host SIMD registers.
974	*
975	* @todo r=aeichner Liveness tracking
976	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
977	*/
978	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
979	{
980	kIemNativeGstSimdReg_SimdRegFirst = 0,
981	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
982	kIemNativeGstSimdReg_End
983	} IEMNATIVEGSTSIMDREG;
984
985	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
986	* @{ */
987	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
988	/** @} */
989
990	/**
991	* The Load/store size for a SIMD guest register.
992	*/
993	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
994	{
995	/** Invalid size. */
996	kIemNativeGstSimdRegLdStSz_Invalid = 0,
997	/** Loads the low 128-bit of a guest SIMD register. */
998	kIemNativeGstSimdRegLdStSz_Low128,
999	/** Loads the high 128-bit of a guest SIMD register. */
1000	kIemNativeGstSimdRegLdStSz_High128,
1001	/** Loads the whole 256-bits of a guest SIMD register. */
1002	kIemNativeGstSimdRegLdStSz_256,
1003	/** End value. */
1004	kIemNativeGstSimdRegLdStSz_End
1005	} IEMNATIVEGSTSIMDREGLDSTSZ;
1006
1007	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1008
1009	/**
1010	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1011	*/
1012	typedef enum IEMNATIVEGSTREGUSE
1013	{
1014	/** The usage is read-only, the register holding the guest register
1015	* shadow copy will not be modified by the caller. */
1016	kIemNativeGstRegUse_ReadOnly = 0,
1017	/** The caller will update the guest register (think: PC += cbInstr).
1018	* The guest shadow copy will follow the returned register. */
1019	kIemNativeGstRegUse_ForUpdate,
1020	/** The call will put an entirely new value in the guest register, so
1021	* if new register is allocate it will be returned uninitialized. */
1022	kIemNativeGstRegUse_ForFullWrite,
1023	/** The caller will use the guest register value as input in a calculation
1024	* and the host register will be modified.
1025	* This means that the returned host register will not be marked as a shadow
1026	* copy of the guest register. */
1027	kIemNativeGstRegUse_Calculation
1028	} IEMNATIVEGSTREGUSE;
1029
1030	/**
1031	* Guest registers (classes) that can be referenced.
1032	*/
1033	typedef enum IEMNATIVEGSTREGREF : uint8_t
1034	{
1035	kIemNativeGstRegRef_Invalid = 0,
1036	kIemNativeGstRegRef_Gpr,
1037	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1038	kIemNativeGstRegRef_EFlags,
1039	kIemNativeGstRegRef_MxCsr,
1040	kIemNativeGstRegRef_FpuReg,
1041	kIemNativeGstRegRef_MReg,
1042	kIemNativeGstRegRef_XReg,
1043	kIemNativeGstRegRef_X87,
1044	kIemNativeGstRegRef_XState,
1045	//kIemNativeGstRegRef_YReg, - doesn't work.
1046	kIemNativeGstRegRef_End
1047	} IEMNATIVEGSTREGREF;
1048
1049
1050	/** Variable kinds. */
1051	typedef enum IEMNATIVEVARKIND : uint8_t
1052	{
1053	/** Customary invalid zero value. */
1054	kIemNativeVarKind_Invalid = 0,
1055	/** This is either in a register or on the stack. */
1056	kIemNativeVarKind_Stack,
1057	/** Immediate value - loaded into register when needed, or can live on the
1058	* stack if referenced (in theory). */
1059	kIemNativeVarKind_Immediate,
1060	/** Variable reference - loaded into register when needed, never stack. */
1061	kIemNativeVarKind_VarRef,
1062	/** Guest register reference - loaded into register when needed, never stack. */
1063	kIemNativeVarKind_GstRegRef,
1064	/** End of valid values. */
1065	kIemNativeVarKind_End
1066	} IEMNATIVEVARKIND;
1067
1068
1069	/** Variable or argument. */
1070	typedef struct IEMNATIVEVAR
1071	{
1072	/** The kind of variable. */
1073	IEMNATIVEVARKIND enmKind;
1074	/** The variable size in bytes. */
1075	uint8_t cbVar;
1076	/** The first stack slot (uint64_t), except for immediate and references
1077	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1078	* has a stack slot it has been initialized and has a value. Unused variables
1079	* has neither a stack slot nor a host register assignment. */
1080	uint8_t idxStackSlot;
1081	/** The host register allocated for the variable, UINT8_MAX if not. */
1082	uint8_t idxReg;
1083	/** The argument number if argument, UINT8_MAX if regular variable. */
1084	uint8_t uArgNo;
1085	/** If referenced, the index (unpacked) of the variable referencing this one,
1086	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1087	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1088	uint8_t idxReferrerVar;
1089	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1090	* @todo not sure what this really is for... */
1091	IEMNATIVEGSTREG enmGstReg;
1092	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1093	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1094	* only valid when idxReg is not UINT8_MAX. */
1095	bool fSimdReg : 1;
1096	/** Set if the registered is currently used exclusively, false if the
1097	* variable is idle and the register can be grabbed. */
1098	bool fRegAcquired : 1;
1099	#else
1100	/** Set if the registered is currently used exclusively, false if the
1101	* variable is idle and the register can be grabbed. */
1102	bool fRegAcquired;
1103	#endif
1104
1105	union
1106	{
1107	/** kIemNativeVarKind_Immediate: The immediate value. */
1108	uint64_t uValue;
1109	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1110	uint8_t idxRefVar;
1111	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1112	struct
1113	{
1114	/** The class of register. */
1115	IEMNATIVEGSTREGREF enmClass;
1116	/** Index within the class. */
1117	uint8_t idx;
1118	} GstRegRef;
1119	} u;
1120	} IEMNATIVEVAR;
1121	/** Pointer to a variable or argument. */
1122	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1123	/** Pointer to a const variable or argument. */
1124	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1125
1126	/** What is being kept in a host register. */
1127	typedef enum IEMNATIVEWHAT : uint8_t
1128	{
1129	/** The traditional invalid zero value. */
1130	kIemNativeWhat_Invalid = 0,
1131	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1132	kIemNativeWhat_Var,
1133	/** Temporary register, this is typically freed when a MC completes. */
1134	kIemNativeWhat_Tmp,
1135	/** Call argument w/o a variable mapping. This is free (via
1136	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1137	kIemNativeWhat_Arg,
1138	/** Return status code.
1139	* @todo not sure if we need this... */
1140	kIemNativeWhat_rc,
1141	/** The fixed pVCpu (PVMCPUCC) register.
1142	* @todo consider offsetting this on amd64 to use negative offsets to access
1143	* more members using 8-byte disp. */
1144	kIemNativeWhat_pVCpuFixed,
1145	/** The fixed pCtx (PCPUMCTX) register.
1146	* @todo consider offsetting this on amd64 to use negative offsets to access
1147	* more members using 8-byte disp. */
1148	kIemNativeWhat_pCtxFixed,
1149	/** Fixed temporary register. */
1150	kIemNativeWhat_FixedTmp,
1151	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1152	/** Shadow RIP for the delayed RIP updating debugging. */
1153	kIemNativeWhat_PcShadow,
1154	#endif
1155	/** Register reserved by the CPU or OS architecture. */
1156	kIemNativeWhat_FixedReserved,
1157	/** End of valid values. */
1158	kIemNativeWhat_End
1159	} IEMNATIVEWHAT;
1160
1161	/**
1162	* Host general register entry.
1163	*
1164	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1165	*
1166	* @todo Track immediate values in host registers similarlly to how we track the
1167	* guest register shadow copies. For it to be real helpful, though,
1168	* we probably need to know which will be reused and put them into
1169	* non-volatile registers, otherwise it's going to be more or less
1170	* restricted to an instruction or two.
1171	*/
1172	typedef struct IEMNATIVEHSTREG
1173	{
1174	/** Set of guest registers this one shadows.
1175	*
1176	* Using a bitmap here so we can designate the same host register as a copy
1177	* for more than one guest register. This is expected to be useful in
1178	* situations where one value is copied to several registers in a sequence.
1179	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1180	* sequence we'd want to let this register follow to be a copy of and there
1181	* will always be places where we'd be picking the wrong one.
1182	*/
1183	uint64_t fGstRegShadows;
1184	/** What is being kept in this register. */
1185	IEMNATIVEWHAT enmWhat;
1186	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1187	uint8_t idxVar;
1188	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1189	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1190	* that scope. */
1191	uint8_t idxStackSlot;
1192	/** Alignment padding. */
1193	uint8_t abAlign[5];
1194	} IEMNATIVEHSTREG;
1195
1196
1197	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1198	/**
1199	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1200	* halves, on architectures where there is no 256-bit register available this entry will track
1201	* two adjacent 128-bit host registers.
1202	*
1203	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1204	*/
1205	typedef struct IEMNATIVEHSTSIMDREG
1206	{
1207	/** Set of guest registers this one shadows.
1208	*
1209	* Using a bitmap here so we can designate the same host register as a copy
1210	* for more than one guest register. This is expected to be useful in
1211	* situations where one value is copied to several registers in a sequence.
1212	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1213	* sequence we'd want to let this register follow to be a copy of and there
1214	* will always be places where we'd be picking the wrong one.
1215	*/
1216	uint64_t fGstRegShadows;
1217	/** What is being kept in this register. */
1218	IEMNATIVEWHAT enmWhat;
1219	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1220	uint8_t idxVar;
1221	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1222	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1223	/** Alignment padding. */
1224	uint8_t abAlign[5];
1225	} IEMNATIVEHSTSIMDREG;
1226	#endif
1227
1228
1229	/**
1230	* Core state for the native recompiler, that is, things that needs careful
1231	* handling when dealing with branches.
1232	*/
1233	typedef struct IEMNATIVECORESTATE
1234	{
1235	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1236	/** The current instruction offset in bytes from when the guest program counter
1237	* was updated last. Used for delaying the write to the guest context program counter
1238	* as long as possible. */
1239	uint32_t offPc;
1240	/** Number of instructions where we could skip the updating. */
1241	uint32_t cInstrPcUpdateSkipped;
1242	#endif
1243	/** Allocation bitmap for aHstRegs. */
1244	uint32_t bmHstRegs;
1245
1246	/** Bitmap marking which host register contains guest register shadow copies.
1247	* This is used during register allocation to try preserve copies. */
1248	uint32_t bmHstRegsWithGstShadow;
1249	/** Bitmap marking valid entries in aidxGstRegShadows. */
1250	uint64_t bmGstRegShadows;
1251	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1252	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1253	uint64_t bmGstRegShadowDirty;
1254	#endif
1255
1256	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1257	/** Allocation bitmap for aHstSimdRegs. */
1258	uint32_t bmHstSimdRegs;
1259
1260	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1261	* This is used during register allocation to try preserve copies. */
1262	uint32_t bmHstSimdRegsWithGstShadow;
1263	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1264	uint64_t bmGstSimdRegShadows;
1265	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1266	uint64_t bmGstSimdRegShadowDirtyLo128;
1267	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1268	uint64_t bmGstSimdRegShadowDirtyHi128;
1269	#endif
1270
1271	union
1272	{
1273	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1274	uint8_t aidxArgVars[8];
1275	/** For more efficient resetting. */
1276	uint64_t u64ArgVars;
1277	};
1278
1279	/** Allocation bitmap for the stack. */
1280	uint32_t bmStack;
1281	/** Allocation bitmap for aVars. */
1282	uint32_t bmVars;
1283
1284	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1285	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1286	* (A shadow copy of a guest register can only be held in a one host register,
1287	* there are no duplicate copies or ambiguities like that). */
1288	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1289	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1290	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1291	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1292	* (A shadow copy of a guest register can only be held in a one host register,
1293	* there are no duplicate copies or ambiguities like that). */
1294	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1295	#endif
1296
1297	/** Host register allocation tracking. */
1298	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1299	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1300	/** Host SIMD register allocation tracking. */
1301	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1302	#endif
1303
1304	/** Variables and arguments. */
1305	IEMNATIVEVAR aVars[9];
1306	} IEMNATIVECORESTATE;
1307	/** Pointer to core state. */
1308	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1309	/** Pointer to const core state. */
1310	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1311
1312	/** @def IEMNATIVE_VAR_IDX_UNPACK
1313	* @returns Index into IEMNATIVECORESTATE::aVars.
1314	* @param a_idxVar Variable index w/ magic (in strict builds).
1315	*/
1316	/** @def IEMNATIVE_VAR_IDX_PACK
1317	* @returns Variable index w/ magic (in strict builds).
1318	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1319	*/
1320	#ifdef VBOX_STRICT
1321	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1322	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1323	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1324	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1325	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1326	#else
1327	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1328	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1329	#endif
1330
1331
1332	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1333	/** Clear the dirty state of the given guest SIMD register. */
1334	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1335	do { \
1336	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1337	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1338	} while (0)
1339
1340	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1341	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1342	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1343	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1344	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1345	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1346	/** Returns whether the given guest SIMD register is dirty. */
1347	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1348	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1349
1350	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1351	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1352	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1353	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1354	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1355	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1356
1357	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1358	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1359	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1360	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1361	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1362	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1363	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1364	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1365	#endif
1366
1367
1368	/**
1369	* Conditional stack entry.
1370	*/
1371	typedef struct IEMNATIVECOND
1372	{
1373	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1374	bool fInElse;
1375	/** The label for the IEM_MC_ELSE. */
1376	uint32_t idxLabelElse;
1377	/** The label for the IEM_MC_ENDIF. */
1378	uint32_t idxLabelEndIf;
1379	/** The initial state snapshot as the if-block starts executing. */
1380	IEMNATIVECORESTATE InitialState;
1381	/** The state snapshot at the end of the if-block. */
1382	IEMNATIVECORESTATE IfFinalState;
1383	} IEMNATIVECOND;
1384	/** Pointer to a condition stack entry. */
1385	typedef IEMNATIVECOND *PIEMNATIVECOND;
1386
1387
1388	/**
1389	* Native recompiler state.
1390	*/
1391	typedef struct IEMRECOMPILERSTATE
1392	{
1393	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1394	* IEMNATIVEINSTR units. */
1395	uint32_t cInstrBufAlloc;
1396	#ifdef VBOX_STRICT
1397	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1398	uint32_t offInstrBufChecked;
1399	#else
1400	uint32_t uPadding1; /* We don't keep track of the size here... */
1401	#endif
1402	/** Fixed temporary code buffer for native recompilation. */
1403	PIEMNATIVEINSTR pInstrBuf;
1404
1405	/** Bitmaps with the label types used. */
1406	uint64_t bmLabelTypes;
1407	/** Actual number of labels in paLabels. */
1408	uint32_t cLabels;
1409	/** Max number of entries allowed in paLabels before reallocating it. */
1410	uint32_t cLabelsAlloc;
1411	/** Labels defined while recompiling (referenced by fixups). */
1412	PIEMNATIVELABEL paLabels;
1413	/** Array with indexes of unique labels (uData always 0). */
1414	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1415
1416	/** Actual number of fixups paFixups. */
1417	uint32_t cFixups;
1418	/** Max number of entries allowed in paFixups before reallocating it. */
1419	uint32_t cFixupsAlloc;
1420	/** Buffer used by the recompiler for recording fixups when generating code. */
1421	PIEMNATIVEFIXUP paFixups;
1422
1423	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1424	/** Number of debug info entries allocated for pDbgInfo. */
1425	uint32_t cDbgInfoAlloc;
1426	uint32_t uPadding;
1427	/** Debug info. */
1428	PIEMTBDBG pDbgInfo;
1429	#endif
1430
1431	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1432	/** The current call index (liveness array and threaded calls in TB). */
1433	uint32_t idxCurCall;
1434	/** Number of liveness entries allocated. */
1435	uint32_t cLivenessEntriesAlloc;
1436	/** Liveness entries for all the calls in the TB begin recompiled.
1437	* The entry for idxCurCall contains the info for what the next call will
1438	* require wrt registers. (Which means the last entry is the initial liveness
1439	* state.) */
1440	PIEMLIVENESSENTRY paLivenessEntries;
1441	#endif
1442
1443	/** The translation block being recompiled. */
1444	PCIEMTB pTbOrg;
1445	/** The VMCPU structure of the EMT. */
1446	PVMCPUCC pVCpu;
1447
1448	/** Condition sequence number (for generating unique labels). */
1449	uint16_t uCondSeqNo;
1450	/** Check IRQ seqeunce number (for generating unique labels). */
1451	uint16_t uCheckIrqSeqNo;
1452	/** TLB load sequence number (for generating unique labels). */
1453	uint16_t uTlbSeqNo;
1454	/** The current condition stack depth (aCondStack). */
1455	uint8_t cCondDepth;
1456
1457	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1458	uint8_t cArgsX;
1459	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1460	uint32_t fCImpl;
1461	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1462	uint32_t fMc;
1463	/** The expected IEMCPU::fExec value for the current call/instruction. */
1464	uint32_t fExec;
1465	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1466	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1467	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1468	*
1469	* This is an optimization because these control registers can only be changed from
1470	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1471	* consisting of multiple SIMD instructions.
1472	*/
1473	uint32_t fSimdRaiseXcptChecksEmitted;
1474	#endif
1475	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1476	uint32_t idxLastCheckIrqCallNo;
1477
1478	/** Core state requiring care with branches. */
1479	IEMNATIVECORESTATE Core;
1480
1481	/** The condition nesting stack. */
1482	IEMNATIVECOND aCondStack[2];
1483
1484	#ifndef IEM_WITH_THROW_CATCH
1485	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1486	* for recompilation error handling. */
1487	jmp_buf JmpBuf;
1488	#endif
1489	} IEMRECOMPILERSTATE;
1490	/** Pointer to a native recompiler state. */
1491	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1492
1493
1494	/** @def IEMNATIVE_TRY_SETJMP
1495	* Wrapper around setjmp / try, hiding all the ugly differences.
1496	*
1497	* @note Use with extreme care as this is a fragile macro.
1498	* @param a_pReNative The native recompile state.
1499	* @param a_rcTarget The variable that should receive the status code in case
1500	* of a longjmp/throw.
1501	*/
1502	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1503	* Start wrapper for catch / setjmp-else.
1504	*
1505	* This will set up a scope.
1506	*
1507	* @note Use with extreme care as this is a fragile macro.
1508	* @param a_pReNative The native recompile state.
1509	* @param a_rcTarget The variable that should receive the status code in case
1510	* of a longjmp/throw.
1511	*/
1512	/** @def IEMNATIVE_CATCH_LONGJMP_END
1513	* End wrapper for catch / setjmp-else.
1514	*
1515	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1516	* up the state.
1517	*
1518	* @note Use with extreme care as this is a fragile macro.
1519	* @param a_pReNative The native recompile state.
1520	*/
1521	/** @def IEMNATIVE_DO_LONGJMP
1522	*
1523	* Wrapper around longjmp / throw.
1524	*
1525	* @param a_pReNative The native recompile state.
1526	* @param a_rc The status code jump back with / throw.
1527	*/
1528	#ifdef IEM_WITH_THROW_CATCH
1529	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1530	a_rcTarget = VINF_SUCCESS; \
1531	try
1532	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1533	catch (int rcThrown) \
1534	{ \
1535	a_rcTarget = rcThrown
1536	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1537	} \
1538	((void)0)
1539	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1540	#else /* !IEM_WITH_THROW_CATCH */
1541	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1542	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1543	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1544	else \
1545	{ \
1546	((void)0)
1547	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1548	}
1549	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1550	#endif /* !IEM_WITH_THROW_CATCH */
1551
1552
1553	/**
1554	* Native recompiler worker for a threaded function.
1555	*
1556	* @returns New code buffer offset; throws VBox status code in case of a failure.
1557	* @param pReNative The native recompiler state.
1558	* @param off The current code buffer offset.
1559	* @param pCallEntry The threaded call entry.
1560	*
1561	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1562	*/
1563	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1564	/** Pointer to a native recompiler worker for a threaded function. */
1565	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1566
1567	/** Defines a native recompiler worker for a threaded function.
1568	* @see FNIEMNATIVERECOMPFUNC */
1569	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1570	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1571
1572	/** Prototypes a native recompiler function for a threaded function.
1573	* @see FNIEMNATIVERECOMPFUNC */
1574	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1575
1576
1577	/**
1578	* Native recompiler liveness analysis worker for a threaded function.
1579	*
1580	* @param pCallEntry The threaded call entry.
1581	* @param pIncoming The incoming liveness state entry.
1582	* @param pOutgoing The outgoing liveness state entry.
1583	*/
1584	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1585	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1586	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1587	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1588
1589	/** Defines a native recompiler liveness analysis worker for a threaded function.
1590	* @see FNIEMNATIVELIVENESSFUNC */
1591	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1592	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1593
1594	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1595	* @see FNIEMNATIVELIVENESSFUNC */
1596	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1597
1598
1599	/** Define a native recompiler helper function, safe to call from the TB code. */
1600	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1601	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1602	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1603	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1604	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1605	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1606	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1607	a_RetType (VBOXCALL *a_Name) a_ArgList
1608
1609
1610	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1611	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1612	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1613	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1614	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1615	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1616	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1617	uint8_t idxHstSimdReg = UINT8_MAX,
1618	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1619	# endif
1620	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1621	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1622	uint8_t idxGstReg, uint8_t idxHstReg);
1623	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1624	uint64_t fGstReg);
1625	# endif
1626	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1627	uint32_t offPc, uint32_t cInstrSkipped);
1628	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1629
1630	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1631	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1632	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1633	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1634	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1635	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1636
1637	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1638	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1639	bool fPreferVolatile = true);
1640	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1641	bool fPreferVolatile = true);
1642	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1643	IEMNATIVEGSTREG enmGstReg,
1644	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1645	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1646	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1647	IEMNATIVEGSTREG enmGstReg);
1648
1649	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1650	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1651	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1652	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1653	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1654	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1655	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1656	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1657	# endif
1658	#endif
1659	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1660	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1661	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1662	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1663	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1664	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1665	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1666	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1667	# endif
1668	#endif
1669	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1670	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1671	uint32_t fKeepVars = 0);
1672	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1673	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1674	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1675	uint32_t fHstRegsActiveShadows);
1676	#ifdef VBOX_STRICT
1677	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1678	#endif
1679	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1680	uint64_t fGstSimdShwExcept);
1681	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1682	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1683	#endif
1684	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1685	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1686	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1687	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1688	#endif
1689
1690
1691	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1692	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1693	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1694	bool fPreferVolatile = true);
1695	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1696	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1697	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1698	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1699	bool fNoVolatileRegs = false);
1700	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1701	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1702	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1703	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1704	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1705	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1706	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1707	#endif
1708
1709	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1710	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1711	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1712	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1713	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1714	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1715	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1716	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1717	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1718	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1719	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1720	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1721	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1722	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1723	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1724	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1725	#endif
1726	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1727	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1728	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1729	uint32_t fHstRegsNotToSave);
1730	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1731	uint32_t fHstRegsNotToSave);
1732	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1733	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1734
1735	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1736	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1737	#ifdef VBOX_STRICT
1738	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1739	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1740	IEMNATIVEGSTREG enmGstReg);
1741	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1742	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1743	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1744	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1745	# endif
1746	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1747	#endif
1748	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1749	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1750	#endif
1751	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1752	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1753	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1754	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1755	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1756	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1757	PCIEMTHRDEDCALLENTRY pCallEntry);
1758	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1759	uint8_t idxAddrReg, uint8_t idxInstr);
1760	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1761	uint8_t idxAddrReg, uint8_t idxInstr);
1762	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1763	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1764
1765
1766	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1767	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1768	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1769	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1770	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1771	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1772	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1773	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1774	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1775	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1776
1777	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1778	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1779	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1780	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1781	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1782	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1783	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1784	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1785	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1786	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1787	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1788	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1789	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1790	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1791	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1792	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1793	#endif
1794	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1795	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1796	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1797	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1798	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1799	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1800	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1801	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1802	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1803	#endif
1804	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1805	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1806	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1807	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1808	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1809	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1810	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1811
1812	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1813	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1814	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1815	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1816	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1817	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1818	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1819	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1820	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1821	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1822	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1823	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1824	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1825	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1826	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1827	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1828	#endif
1829	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1830	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1831	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1832	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1833	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1834	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1835	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1836	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1837	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1838	#endif
1839	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1840	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1841	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1842	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1843	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1844	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1845	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1846
1847	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1848	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1849	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1850	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1851	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1852	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1853	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1854	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1855	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1856	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1857	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1858	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1859	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1860	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1861	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1862	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1863	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1864	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1865	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1866	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1867	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1868	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1869
1870	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1871	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1872	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1873	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1874	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1875	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1876	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1877	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1878	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1879	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1880	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1881	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1882	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1883	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1884	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1885	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1886	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1887	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1888	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1889	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1890	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1891	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1892
1893	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1894	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1895	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1896	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1897
1898
1899	/**
1900	* Info about shadowed guest register values.
1901	* @see IEMNATIVEGSTREG
1902	*/
1903	typedef struct IEMANTIVEGSTREGINFO
1904	{
1905	/** Offset in VMCPU. */
1906	uint32_t off;
1907	/** The field size. */
1908	uint8_t cb;
1909	/** Name (for logging). */
1910	const char *pszName;
1911	} IEMANTIVEGSTREGINFO;
1912	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1913	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1914	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1915	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1916	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1917
1918
1919
1920	/**
1921	* Ensures that there is sufficient space in the instruction output buffer.
1922	*
1923	* This will reallocate the buffer if needed and allowed.
1924	*
1925	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1926	* allocation size.
1927	*
1928	* @returns Pointer to the instruction output buffer on success; throws VBox
1929	* status code on failure, so no need to check it.
1930	* @param pReNative The native recompile state.
1931	* @param off Current instruction offset. Works safely for UINT32_MAX
1932	* as well.
1933	* @param cInstrReq Number of instruction about to be added. It's okay to
1934	* overestimate this a bit.
1935	*/
1936	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1937	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1938	{
1939	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1940	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1941	{
1942	#ifdef VBOX_STRICT
1943	pReNative->offInstrBufChecked = offChecked;
1944	#endif
1945	return pReNative->pInstrBuf;
1946	}
1947	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1948	}
1949
1950	/**
1951	* Checks that we didn't exceed the space requested in the last
1952	* iemNativeInstrBufEnsure() call.
1953	*/
1954	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1955	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1956	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1957
1958	/**
1959	* Checks that a variable index is valid.
1960	*/
1961	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1962	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1963	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1964	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1965	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1966	("%s=%#x\n", #a_idxVar, a_idxVar))
1967	#else
1968	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1969	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1970	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1971	#endif
1972
1973	/**
1974	* Checks that a variable index is valid and that the variable is assigned the
1975	* correct argument number.
1976	* This also adds a RT_NOREF of a_idxVar.
1977	*/
1978	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1979	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1980	RT_NOREF_PV(a_idxVar); \
1981	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1982	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1983	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1984	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1985	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1986	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1987	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1988	a_uArgNo)); \
1989	} while (0)
1990	#else
1991	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1992	RT_NOREF_PV(a_idxVar); \
1993	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1994	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1995	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1996	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1997	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1998	} while (0)
1999	#endif
2000
2001
2002	/**
2003	* Checks that a variable has the expected size.
2004	*/
2005	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2006	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2007	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2008	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
2009
2010
2011	/**
2012	* Calculates the stack address of a variable as a [r]BP displacement value.
2013	*/
2014	DECL_FORCE_INLINE(int32_t)
2015	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2016	{
2017	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2018	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2019	}
2020
2021
2022	/**
2023	* Releases the variable's register.
2024	*
2025	* The register must have been previously acquired calling
2026	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2027	* iemNativeVarRegisterSetAndAcquire().
2028	*/
2029	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2030	{
2031	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2032	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2033	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2034	}
2035
2036
2037	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2038	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2039	{
2040	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2041	iemNativeVarRegisterRelease(pReNative, idxVar);
2042	}
2043	#endif
2044
2045
2046	/**
2047	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2048	*
2049	* @returns The flush mask.
2050	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2051	* @param fGstShwFlush The starting flush mask.
2052	*/
2053	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2054	{
2055	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2056	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2057	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2058	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2059	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2060	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2061	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2062	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2063	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2064	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2065	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2066	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2067	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2068	return fGstShwFlush;
2069	}
2070
2071
2072	/** Number of hidden arguments for CIMPL calls.
2073	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2074	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2075	# define IEM_CIMPL_HIDDEN_ARGS 3
2076	#else
2077	# define IEM_CIMPL_HIDDEN_ARGS 2
2078	#endif
2079
2080
2081	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2082	/** Number of hidden arguments for SSE_AIMPL calls. */
2083	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2084	/** Number of hidden arguments for AVX_AIMPL calls. */
2085	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2086	#endif
2087
2088
2089	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2090
2091	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2092	/**
2093	* Helper for iemNativeLivenessGetStateByGstReg.
2094	*
2095	* @returns IEMLIVENESS_STATE_XXX
2096	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2097	* ORed together.
2098	*/
2099	DECL_FORCE_INLINE(uint32_t)
2100	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2101	{
2102	/* INPUT trumps anything else. */
2103	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2104	return IEMLIVENESS_STATE_INPUT;
2105
2106	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2107	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2108	{
2109	/* If not all sub-fields are clobbered they must be considered INPUT. */
2110	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2111	return IEMLIVENESS_STATE_INPUT;
2112	return IEMLIVENESS_STATE_CLOBBERED;
2113	}
2114
2115	/* XCPT_OR_CALL trumps UNUSED. */
2116	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2117	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2118
2119	return IEMLIVENESS_STATE_UNUSED;
2120	}
2121	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2122
2123
2124	DECL_FORCE_INLINE(uint32_t)
2125	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2126	{
2127	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2128	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2129	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2130	# else
2131	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2132	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2133	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2134	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2135	# endif
2136	}
2137
2138
2139	DECL_FORCE_INLINE(uint32_t)
2140	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2141	{
2142	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2143	if (enmGstReg == kIemNativeGstReg_EFlags)
2144	{
2145	/* Merge the eflags states to one. */
2146	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2147	uRet = RT_BIT_32(uRet);
2148	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2149	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2150	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2151	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2152	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2153	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2154	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2155	# else
2156	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2157	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2158	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2159	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2160	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2161	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2162	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2163	# endif
2164	}
2165	return uRet;
2166	}
2167
2168
2169	# ifdef VBOX_STRICT
2170	/** For assertions only, user checks that idxCurCall isn't zerow. */
2171	DECL_FORCE_INLINE(uint32_t)
2172	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2173	{
2174	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2175	}
2176	# endif /* VBOX_STRICT */
2177
2178	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2179
2180
2181	/**
2182	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2183	*/
2184	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2185	{
2186	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2187	return IEM_CIMPL_HIDDEN_ARGS;
2188	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2189	return 1;
2190	return 0;
2191	}
2192
2193
2194	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2195	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2196	{
2197	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2198
2199	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2200	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2201	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2202	return (uint8_t)idxReg;
2203	}
2204
2205
2206
2207	/*********************************************************************************************************************************
2208	* Register Allocator (GPR) *
2209	*********************************************************************************************************************************/
2210
2211	/**
2212	* Marks host register @a idxHstReg as containing a shadow copy of guest
2213	* register @a enmGstReg.
2214	*
2215	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2216	* host register before calling.
2217	*/
2218	DECL_FORCE_INLINE(void)
2219	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2220	{
2221	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2222	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2223	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2224
2225	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2226	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2227	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2228	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2229	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2230	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2231	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2232	#else
2233	RT_NOREF(off);
2234	#endif
2235	}
2236
2237
2238	/**
2239	* Clear any guest register shadow claims from @a idxHstReg.
2240	*
2241	* The register does not need to be shadowing any guest registers.
2242	*/
2243	DECL_FORCE_INLINE(void)
2244	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2245	{
2246	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2247	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2248	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2249	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2250	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2251	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2252	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2253	#endif
2254
2255	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2256	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2257	if (fGstRegs)
2258	{
2259	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2260	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2261	while (fGstRegs)
2262	{
2263	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2264	fGstRegs &= ~RT_BIT_64(iGstReg);
2265	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2266	}
2267	}
2268	#else
2269	RT_NOREF(off);
2270	#endif
2271
2272	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2273	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2274	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2275	}
2276
2277
2278	/**
2279	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2280	* and global overview flags.
2281	*/
2282	DECL_FORCE_INLINE(void)
2283	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2284	{
2285	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2286	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2287	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2288	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2289	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2290	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2291	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2292	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2293	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2294	#endif
2295
2296	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2297	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2298	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2299	#else
2300	RT_NOREF(off);
2301	#endif
2302
2303	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2304	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2305	if (!fGstRegShadowsNew)
2306	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2307	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2308	}
2309
2310
2311	#if 0 /* unused */
2312	/**
2313	* Clear any guest register shadow claim for @a enmGstReg.
2314	*/
2315	DECL_FORCE_INLINE(void)
2316	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2317	{
2318	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2319	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2320	{
2321	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2322	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2323	}
2324	}
2325	#endif
2326
2327
2328	/**
2329	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2330	* as the new shadow of it.
2331	*
2332	* Unlike the other guest reg shadow helpers, this does the logging for you.
2333	* However, it is the liveness state is not asserted here, the caller must do
2334	* that.
2335	*/
2336	DECL_FORCE_INLINE(void)
2337	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2338	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2339	{
2340	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2341	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2342	{
2343	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2344	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2345	if (idxHstRegOld == idxHstRegNew)
2346	return;
2347	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2348	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2349	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2350	}
2351	else
2352	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2353	g_aGstShadowInfo[enmGstReg].pszName));
2354	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2355	}
2356
2357
2358	/**
2359	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2360	* to @a idxRegTo.
2361	*/
2362	DECL_FORCE_INLINE(void)
2363	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2364	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2365	{
2366	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2367	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2368	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2369	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2370	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2371	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2372	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2373	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2374	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2375
2376	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2377	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2378	if (!fGstRegShadowsFrom)
2379	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2380	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2381	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2382	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2383	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2384	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2385	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2386	#else
2387	RT_NOREF(off);
2388	#endif
2389	}
2390
2391
2392	/**
2393	* Flushes any delayed guest register writes.
2394	*
2395	* This must be called prior to calling CImpl functions and any helpers that use
2396	* the guest state (like raising exceptions) and such.
2397	*
2398	* This optimization has not yet been implemented. The first target would be
2399	* RIP updates, since these are the most common ones.
2400	*
2401	* @note This function does not flush any shadowing information for guest registers. This needs to be done by
2402	* the caller if it wishes to do so.
2403	*/
2404	DECL_INLINE_THROW(uint32_t)
2405	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2406	{
2407	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2408	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2409	#else
2410	uint64_t const bmGstRegShadowDirty = 0;
2411	#endif
2412	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2413	uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 \| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2414	& ~fGstSimdShwExcept;
2415	#else
2416	uint64_t const bmGstSimdRegShadowDirty = 0;
2417	#endif
2418	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2419	uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2420	#else
2421	uint64_t const fWritebackPc = 0;
2422	#endif
2423	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2424	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2425
2426	return off;
2427	}
2428
2429
2430
2431	/*********************************************************************************************************************************
2432	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2433	*********************************************************************************************************************************/
2434
2435	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2436
2437	DECL_FORCE_INLINE(uint8_t)
2438	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2439	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2440	{
2441	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2442
2443	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2444	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2445	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2446	return idxSimdReg;
2447	}
2448
2449
2450	/**
2451	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2452	* SIMD register @a enmGstSimdReg.
2453	*
2454	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2455	* host register before calling.
2456	*/
2457	DECL_FORCE_INLINE(void)
2458	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2459	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2460	{
2461	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2462	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2463	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2464
2465	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2466	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2467	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2468	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2469	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2470	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2471	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2472	#else
2473	RT_NOREF(off);
2474	#endif
2475	}
2476
2477
2478	/**
2479	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2480	* to @a idxSimdRegTo.
2481	*/
2482	DECL_FORCE_INLINE(void)
2483	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2484	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2485	{
2486	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2487	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2488	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2489	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2490	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2491	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2492	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2493	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2494	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2495	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2496	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2497
2498	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2499	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2500	if (!fGstRegShadowsFrom)
2501	{
2502	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2503	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2504	}
2505	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2506	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2507	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2508	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2509	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2510	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2511	#else
2512	RT_NOREF(off);
2513	#endif
2514	}
2515
2516
2517	/**
2518	* Clear any guest register shadow claims from @a idxHstSimdReg.
2519	*
2520	* The register does not need to be shadowing any guest registers.
2521	*/
2522	DECL_FORCE_INLINE(void)
2523	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2524	{
2525	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2526	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2527	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2528	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2529	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2530	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2531	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2532
2533	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2534	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2535	if (fGstRegs)
2536	{
2537	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2538	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2539	while (fGstRegs)
2540	{
2541	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2542	fGstRegs &= ~RT_BIT_64(iGstReg);
2543	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2544	}
2545	}
2546	#else
2547	RT_NOREF(off);
2548	#endif
2549
2550	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2551	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2552	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2553	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2554	}
2555
2556	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2557
2558
2559	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2560	/**
2561	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2562	*/
2563	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2564	{
2565	if (pReNative->Core.offPc)
2566	return iemNativeEmitPcWritebackSlow(pReNative, off);
2567	return off;
2568	}
2569	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2570
2571
2572	#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2573	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2574	* it saves us the trouble of a hidden parameter on MSC/amd64. */
2575	# ifdef RT_ARCH_AMD64
2576	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2577	# elif defined(RT_ARCH_ARM64)
2578	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2579	# endif
2580	#endif
2581
2582	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2583
2584	/** @} */
2585
2586	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2587

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 104797

Download in other formats: