IEMN8veRecompiler.h@ 105855

Last change on this file since 105855 was 105855, checked in by vboxsync, 3 months ago
VMM/IEM: Don't update PC during IEM_MC_REL_JMP_S[8\|32]_AND_FINISH if we can avoid it. This extends the offPc to 64-bit and tries to make the stats more accurate. This is more on todo 4 in bugref:10720. bugref:10373
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 131.5 KB

Line
1	/* $Id: IEMN8veRecompiler.h 105855 2024-08-23 23:12:23Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	/# define IEMLIVENESS_EXTENDED_LAYOUT/
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62	* Enables strict consistency checks around EFLAGS skipping.
63	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65	# ifdef VBOX_STRICT
66	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67	# endif
68	#elif defined(DOXYGEN_RUNNING)
69	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70	#endif
71
72	#ifdef VBOX_WITH_STATISTICS
73	/** Always count instructions for now. */
74	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75	#endif
76
77	/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78	* Enables having only a single prologue for native TBs. */
79	#if 1 \|\| defined(DOXYGEN_RUNNING)
80	# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81	#endif
82
83	/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84	* Enable this to use common epilogue and tail code for all TBs in a chunk. */
85	#if 1 \|\| defined(DOXYGEN_RUNNING)
86	# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87	#endif
88
89
90	/** @name Stack Frame Layout
91	*
92	* @{ */
93	/** The size of the area for stack variables and spills and stuff.
94	* @note This limit is duplicated in the python script(s). We add 0x40 for
95	* alignment padding. */
96	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101	#ifdef RT_ARCH_AMD64
102	/** An stack alignment adjustment (between non-volatile register pushes and
103	* the stack variable area, so the latter better aligned). */
104	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106	/** Number of stack arguments slots for calls made from the frame. */
107	# ifdef RT_OS_WINDOWS
108	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109	# else
110	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111	# endif
112	/** Number of any shadow arguments (spill area) for calls we make. */
113	# ifdef RT_OS_WINDOWS
114	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115	# else
116	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117	# endif
118
119	/** Frame pointer (RBP) relative offset of the last push. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122	# else
123	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124	# endif
125	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126	* address for it). */
127	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132	# ifdef RT_OS_WINDOWS
133	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137	# endif
138
139	# ifdef RT_OS_WINDOWS
140	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148	# endif
149
150	#elif RT_ARCH_ARM64
151	/** No alignment padding needed for arm64. */
152	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153	/** No stack argument slots, got 8 registers for arguments will suffice. */
154	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155	/** There are no argument spill area. */
156	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158	/** Number of saved registers at the top of our stack frame.
159	* This includes the return address and old frame pointer, so x19 thru x30. */
160	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164	/** Frame pointer (BP) relative offset of the last push. */
165	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168	* address for it). */
169	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171	#else
172	# error "port me"
173	#endif
174	/** @} */
175
176
177	/** @name Fixed Register Allocation(s)
178	* @{ */
179	/** @def IEMNATIVE_REG_FIXED_PVMCPU
180	* The number of the register holding the pVCpu pointer. */
181	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183	* @note This not available on AMD64, only ARM64. */
184	/** @def IEMNATIVE_REG_FIXED_TMP0
185	* Dedicated temporary register.
186	* @note This has extremely short lifetime, must be used with great care to make
187	* sure any calling code or code being called is making use of it.
188	* It will definitely not survive a call or anything of that nature.
189	* @todo replace this by a register allocator and content tracker. */
190	/** @def IEMNATIVE_REG_FIXED_MASK
191	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
192	* architecture. */
193	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
196	* architecture. */
197	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
198	* Dedicated temporary SIMD register. */
199	#endif
200	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
201	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
202	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
203	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
204	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
205	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
206	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
207	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
208	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
209	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
210	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
211	# else
212	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
213	# endif
214	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
215	\| RT_BIT_32(ARMV8_A64_REG_LR) \
216	\| RT_BIT_32(ARMV8_A64_REG_BP) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
218	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
219	\| RT_BIT_32(ARMV8_A64_REG_X18) \
220	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
221	\| IEMNATIVE_REG_FIXED_MASK_ADD)
222
223	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
224	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
225	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
227	# else
228	/** @note
229	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
230	* support emulating 256-bit registers we pair two real registers statically to
231	* one virtual for now, leaving us with only 16 256-bit registers. We always
232	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
233	* the register allocator assumes that it will be always free when the lower is
234	* picked.
235	*
236	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
237	* touch them in order to avoid having to save and restore them in the
238	* prologue/epilogue.
239	*/
240	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
252	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
253	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
255	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
256	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
257	\| RT_BIT_32(ARMV8_A64_REG_Q1))
258	# endif
259	# endif
260
261	#elif defined(RT_ARCH_AMD64)
262	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
263	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
264	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
265	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
266	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
267	\| RT_BIT_32(X86_GREG_xSP) \
268	\| RT_BIT_32(X86_GREG_xBP) )
269
270	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
271	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
272	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
273	# ifndef _MSC_VER
274	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275	# endif
276	# endif
277	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
278	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
279	# else
280	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
281	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
282	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
283	# endif
284	# endif
285
286	#else
287	# error "port me"
288	#endif
289	/** @} */
290
291	/** @name Call related registers.
292	* @{ */
293	/** @def IEMNATIVE_CALL_RET_GREG
294	* The return value register. */
295	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
296	* Number of arguments in registers. */
297	/** @def IEMNATIVE_CALL_ARG0_GREG
298	* The general purpose register carrying argument \#0. */
299	/** @def IEMNATIVE_CALL_ARG1_GREG
300	* The general purpose register carrying argument \#1. */
301	/** @def IEMNATIVE_CALL_ARG2_GREG
302	* The general purpose register carrying argument \#2. */
303	/** @def IEMNATIVE_CALL_ARG3_GREG
304	* The general purpose register carrying argument \#3. */
305	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
306	* Mask of registers the callee will not save and may trash. */
307	#ifdef RT_ARCH_AMD64
308	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
309
310	# ifdef RT_OS_WINDOWS
311	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
312	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
313	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
314	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
315	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
316	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
317	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
318	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
319	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
320	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
321	\| RT_BIT_32(X86_GREG_xCX) \
322	\| RT_BIT_32(X86_GREG_xDX) \
323	\| RT_BIT_32(X86_GREG_x8) \
324	\| RT_BIT_32(X86_GREG_x9) \
325	\| RT_BIT_32(X86_GREG_x10) \
326	\| RT_BIT_32(X86_GREG_x11) )
327	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
328	/* xmm0 - xmm5 are marked as volatile. */
329	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
330	# endif
331
332	# else /* !RT_OS_WINDOWS */
333	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
334	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
335	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
336	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
337	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
338	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
339	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
340	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
341	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
342	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
343	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
344	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
345	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
346	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
347	\| RT_BIT_32(X86_GREG_xCX) \
348	\| RT_BIT_32(X86_GREG_xDX) \
349	\| RT_BIT_32(X86_GREG_xDI) \
350	\| RT_BIT_32(X86_GREG_xSI) \
351	\| RT_BIT_32(X86_GREG_x8) \
352	\| RT_BIT_32(X86_GREG_x9) \
353	\| RT_BIT_32(X86_GREG_x10) \
354	\| RT_BIT_32(X86_GREG_x11) )
355	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
356	/* xmm0 - xmm15 are marked as volatile. */
357	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
358	# endif
359	# endif /* !RT_OS_WINDOWS */
360
361	#elif defined(RT_ARCH_ARM64)
362	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
363	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
364	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
365	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
366	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
367	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
368	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
369	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
370	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
371	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
372	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373	\| RT_BIT_32(ARMV8_A64_REG_X1) \
374	\| RT_BIT_32(ARMV8_A64_REG_X2) \
375	\| RT_BIT_32(ARMV8_A64_REG_X3) \
376	\| RT_BIT_32(ARMV8_A64_REG_X4) \
377	\| RT_BIT_32(ARMV8_A64_REG_X5) \
378	\| RT_BIT_32(ARMV8_A64_REG_X6) \
379	\| RT_BIT_32(ARMV8_A64_REG_X7) )
380	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
381	\| RT_BIT_32(ARMV8_A64_REG_X1) \
382	\| RT_BIT_32(ARMV8_A64_REG_X2) \
383	\| RT_BIT_32(ARMV8_A64_REG_X3) \
384	\| RT_BIT_32(ARMV8_A64_REG_X4) \
385	\| RT_BIT_32(ARMV8_A64_REG_X5) \
386	\| RT_BIT_32(ARMV8_A64_REG_X6) \
387	\| RT_BIT_32(ARMV8_A64_REG_X7) \
388	\| RT_BIT_32(ARMV8_A64_REG_X8) \
389	\| RT_BIT_32(ARMV8_A64_REG_X9) \
390	\| RT_BIT_32(ARMV8_A64_REG_X10) \
391	\| RT_BIT_32(ARMV8_A64_REG_X11) \
392	\| RT_BIT_32(ARMV8_A64_REG_X12) \
393	\| RT_BIT_32(ARMV8_A64_REG_X13) \
394	\| RT_BIT_32(ARMV8_A64_REG_X14) \
395	\| RT_BIT_32(ARMV8_A64_REG_X15) \
396	\| RT_BIT_32(ARMV8_A64_REG_X16) \
397	\| RT_BIT_32(ARMV8_A64_REG_X17) )
398	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
399	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
400	* so to simplify our life a bit we just mark everything as volatile. */
401	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
402	# endif
403
404	#endif
405
406	/** This is the maximum argument count we'll ever be needing. */
407	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
408	#ifdef RT_OS_WINDOWS
409	# ifdef VBOXSTRICTRC_STRICT_ENABLED
410	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
411	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
412	# endif
413	#endif
414
415	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
416	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
417	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
418	#ifdef IEMNATIVE_REG_FIXED_TMP0
419	# ifdef IEMNATIVE_REG_FIXED_TMP1
420	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
421	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
422	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
423	# else
424	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
425	# endif
426	#else
427	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
428	#endif
429	/** @} */
430
431
432	/** @def IEMNATIVE_HST_GREG_COUNT
433	* Number of host general purpose registers we tracker. */
434	/** @def IEMNATIVE_HST_GREG_MASK
435	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
436	* inverted register masks and such to get down to a correct set of regs. */
437	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
438	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
439	* Number of host SIMD registers we track. */
440	/** @def IEMNATIVE_HST_SIMD_REG_MASK
441	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
442	* inverted register masks and such to get down to a correct set of regs. */
443	#endif
444	#ifdef RT_ARCH_AMD64
445	# define IEMNATIVE_HST_GREG_COUNT 16
446	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
447
448	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
449	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
450	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
451	# endif
452
453	#elif defined(RT_ARCH_ARM64)
454	# define IEMNATIVE_HST_GREG_COUNT 32
455	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
456
457	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
458	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
459	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
460	# endif
461
462	#else
463	# error "Port me!"
464	#endif
465
466
467	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
468
469
470	/** Native code generator label types. */
471	typedef enum
472	{
473	kIemNativeLabelType_Invalid = 0,
474	/*
475	* Labels w/o data, only once instance per TB - aka exit reasons.
476	*
477	* Note! Jumps to these requires instructions that are capable of spanning
478	* the max TB length.
479	*/
480	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
481	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
482	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
483	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
484	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
485	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
486	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
487	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
488	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
489	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
490	kIemNativeLabelType_ObsoleteTb,
491	kIemNativeLabelType_NeedCsLimChecking,
492	kIemNativeLabelType_CheckBranchMiss,
493	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
494	/* Manually defined labels. */
495	kIemNativeLabelType_ReturnBreak,
496	kIemNativeLabelType_ReturnBreakFF,
497	kIemNativeLabelType_ReturnBreakViaLookup,
498	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
499	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
500	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
501	kIemNativeLabelType_ReturnWithFlags,
502	kIemNativeLabelType_NonZeroRetOrPassUp,
503	kIemNativeLabelType_Return,
504	/** The last fixup for branches that can span almost the whole TB length.
505	* @note Whether kIemNativeLabelType_Return needs to be one of these is
506	* a bit questionable, since nobody jumps to it except other tail code. */
507	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
508	/** The last fixup for branches that exits the TB. */
509	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
510
511	/** Loop-jump target. */
512	kIemNativeLabelType_LoopJumpTarget,
513
514	/*
515	* Labels with data, potentially multiple instances per TB:
516	*
517	* These are localized labels, so no fixed jump type restrictions here.
518	*/
519	kIemNativeLabelType_FirstWithMultipleInstances,
520	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
521	kIemNativeLabelType_Else,
522	kIemNativeLabelType_Endif,
523	kIemNativeLabelType_CheckIrq,
524	kIemNativeLabelType_TlbLookup,
525	kIemNativeLabelType_TlbMiss,
526	kIemNativeLabelType_TlbDone,
527	kIemNativeLabelType_End
528	} IEMNATIVELABELTYPE;
529
530	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
531	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
532
533
534	/** Native code generator label definition. */
535	typedef struct IEMNATIVELABEL
536	{
537	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
538	* the epilog. */
539	uint32_t off;
540	/** The type of label (IEMNATIVELABELTYPE). */
541	uint16_t enmType;
542	/** Additional label data, type specific. */
543	uint16_t uData;
544	} IEMNATIVELABEL;
545	/** Pointer to a label. */
546	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
547
548
549	/** Native code generator fixup types. */
550	typedef enum
551	{
552	kIemNativeFixupType_Invalid = 0,
553	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
554	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
555	kIemNativeFixupType_Rel32,
556	#elif defined(RT_ARCH_ARM64)
557	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
558	kIemNativeFixupType_RelImm26At0,
559	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
560	kIemNativeFixupType_RelImm19At5,
561	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
562	kIemNativeFixupType_RelImm14At5,
563	#endif
564	kIemNativeFixupType_End
565	} IEMNATIVEFIXUPTYPE;
566
567	/** Native code generator fixup. */
568	typedef struct IEMNATIVEFIXUP
569	{
570	/** Code offset of the fixup location. */
571	uint32_t off;
572	/** The IEMNATIVELABEL this is a fixup for. */
573	uint16_t idxLabel;
574	/** The fixup type (IEMNATIVEFIXUPTYPE). */
575	uint8_t enmType;
576	/** Addend or other data. */
577	int8_t offAddend;
578	} IEMNATIVEFIXUP;
579	/** Pointer to a native code generator fixup. */
580	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
581
582	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
583
584	/** Native code generator fixup to per chunk TB tail code. */
585	typedef struct IEMNATIVEEXITFIXUP
586	{
587	/** Code offset of the fixup location. */
588	uint32_t off;
589	/** The exit reason. */
590	IEMNATIVELABELTYPE enmExitReason;
591	} IEMNATIVEEXITFIXUP;
592	/** Pointer to a native code generator TB exit fixup. */
593	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
594
595	/**
596	* Per executable memory chunk context with addresses for common code.
597	*/
598	typedef struct IEMNATIVEPERCHUNKCTX
599	{
600	/** Pointers to the exit labels */
601	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
602	} IEMNATIVEPERCHUNKCTX;
603	/** Pointer to per-chunk recompiler context. */
604	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
605	/** Pointer to const per-chunk recompiler context. */
606	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
607
608	#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
609
610
611	/**
612	* One bit of the state.
613	*
614	* Each register state takes up two bits. We keep the two bits in two separate
615	* 64-bit words to simplify applying them to the guest shadow register mask in
616	* the register allocator.
617	*/
618	typedef union IEMLIVENESSBIT
619	{
620	uint64_t bm64;
621	RT_GCC_EXTENSION struct
622	{ /* bit no */
623	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
624	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
625	uint64_t fCr0 : 1; /*< 0x11 / 17: /
626	uint64_t fFcw : 1; /*< 0x12 / 18: /
627	uint64_t fFsw : 1; /*< 0x13 / 19: /
628	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
629	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
630	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
631	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
632	uint64_t fCr4 : 1; /*< 0x2c / 44: /
633	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
634	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
635	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
636	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
637	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
638	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
639	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
640	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
641	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
642	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
643	};
644	} IEMLIVENESSBIT;
645	AssertCompileSize(IEMLIVENESSBIT, 8);
646
647	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
648	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
649	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
650	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
651	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
652	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
653	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
654
655
656	/**
657	* A liveness state entry.
658	*
659	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
660	* Once we add a SSE register shadowing, we'll add another 64-bit element for
661	* that.
662	*/
663	typedef union IEMLIVENESSENTRY
664	{
665	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
666	uint64_t bm64[16 / 8];
667	uint16_t bm32[16 / 4];
668	uint16_t bm16[16 / 2];
669	uint8_t bm8[ 16 / 1];
670	IEMLIVENESSBIT aBits[2];
671	#else
672	uint64_t bm64[32 / 8];
673	uint16_t bm32[32 / 4];
674	uint16_t bm16[32 / 2];
675	uint8_t bm8[ 32 / 1];
676	IEMLIVENESSBIT aBits[4];
677	#endif
678	RT_GCC_EXTENSION struct
679	{
680	/** Bit \#0 of the register states. */
681	IEMLIVENESSBIT Bit0;
682	/** Bit \#1 of the register states. */
683	IEMLIVENESSBIT Bit1;
684	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
685	/** Bit \#2 of the register states. */
686	IEMLIVENESSBIT Bit2;
687	/** Bit \#3 of the register states. */
688	IEMLIVENESSBIT Bit3;
689	#endif
690	};
691	} IEMLIVENESSENTRY;
692	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
693	AssertCompileSize(IEMLIVENESSENTRY, 16);
694	#else
695	AssertCompileSize(IEMLIVENESSENTRY, 32);
696	#endif
697	/** Pointer to a liveness state entry. */
698	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
699	/** Pointer to a const liveness state entry. */
700	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
701
702	/** @name 64-bit value masks for IEMLIVENESSENTRY.
703	* @{ / / 0xzzzzyyyyxxxxwwww */
704	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
705
706	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
707	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
708	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
709
710	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
711	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
712	#endif
713
714	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
715	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
716
717	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
718	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
719	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
720	#endif
721	/** @} */
722
723
724	/** @name The liveness state for a register.
725	*
726	* The state values have been picked to with state accumulation in mind (what
727	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
728	* performance critical work done with the values.
729	*
730	* This is a compressed state that only requires 2 bits per register.
731	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
732	* 1. the incoming state from the following call,
733	* 2. the outgoing state for this call,
734	* 3. mask of the entries set in the 2nd.
735	*
736	* The mask entry (3rd one above) will be used both when updating the outgoing
737	* state and when merging in incoming state for registers not touched by the
738	* current call.
739	*
740	* @{ */
741	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
742	/** The register will be clobbered and the current value thrown away.
743	*
744	* When this is applied to the state (2) we'll simply be AND'ing it with the
745	* (old) mask (3) and adding the register to the mask. This way we'll
746	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
747	* IEMLIVENESS_STATE_INPUT states. */
748	# define IEMLIVENESS_STATE_CLOBBERED 0
749	/** The register is unused in the remainder of the TB.
750	*
751	* This is an initial state and can not be set by any of the
752	* iemNativeLivenessFunc_xxxx callbacks. */
753	# define IEMLIVENESS_STATE_UNUSED 1
754	/** The register value is required in a potential call or exception.
755	*
756	* This means that the register value must be calculated and is best written to
757	* the state, but that any shadowing registers can be flushed thereafter as it's
758	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
759	*
760	* It is typically applied across the board, but we preserve incoming
761	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
762	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
763	* 1. r0 = old & ~mask;
764	* 2. r0 = t1 & (t1 >> 1)'
765	* 3. state \|= r0 \| 0b10;
766	* 4. mask = ~0;
767	*/
768	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
769	/** The register value is used as input.
770	*
771	* This means that the register value must be calculated and it is best to keep
772	* it in a register. It does not need to be writtent out as such. This is the
773	* highest priority state.
774	*
775	* Whether the call modifies the register or not isn't relevant to earlier
776	* calls, so that's not recorded.
777	*
778	* When applying this state we just or in the value in the outgoing state and
779	* mask. */
780	# define IEMLIVENESS_STATE_INPUT 3
781	/** Mask of the state bits. */
782	# define IEMLIVENESS_STATE_MASK 3
783	/** The number of bits per state. */
784	# define IEMLIVENESS_STATE_BIT_COUNT 2
785	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
786	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
787	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
788	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
789	/** Check if a register clobbering is expected given the (previous) liveness state.
790	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
791	* include INPUT if the register is used in more than one place. */
792	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
793
794	/** Check if all status flags are going to be clobbered and doesn't need
795	* calculating in the current step.
796	* @param a_pCurEntry The current liveness entry. */
797	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
798	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
799
800	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
801	/** The register is not used any more. */
802	# define IEMLIVENESS_STATE_UNUSED 0
803	/** Flag: The register is required in a potential exception or call. */
804	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
805	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
806	/** Flag: The register is read. */
807	# define IEMLIVENESS_STATE_READ 2
808	# define IEMLIVENESS_BIT_READ 1
809	/** Flag: The register is written. */
810	# define IEMLIVENESS_STATE_WRITE 4
811	# define IEMLIVENESS_BIT_WRITE 2
812	/** Flag: Unconditional call (not needed, can be redefined for research). */
813	# define IEMLIVENESS_STATE_CALL 8
814	# define IEMLIVENESS_BIT_CALL 3
815	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
816	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
817	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
818	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
819	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
820
821	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
822	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
823	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
824	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
825
826	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
827	/** @} */
828
829	/** @name Liveness helpers for builtin functions and similar.
830	*
831	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
832	* own set of manimulator macros for those.
833	*
834	* @{ */
835	/** Initializing the state as all unused. */
836	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
837	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
838	do { \
839	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
840	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
841	} while (0)
842	#else
843	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
844	do { \
845	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
846	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
847	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
848	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
849	} while (0)
850	#endif
851
852	/** Initializing the outgoing state with a potential xcpt or call state.
853	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
854	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
855	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
856	do { \
857	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
858	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
859	} while (0)
860	#else
861	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
862	do { \
863	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
864	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
865	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
866	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
867	} while (0)
868	#endif
869
870	/** Adds a segment base register as input to the outgoing state. */
871	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
872	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
873	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
874	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
875	} while (0)
876	#else
877	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
878	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
879	} while (0)
880	#endif
881
882	/** Adds a segment attribute register as input to the outgoing state. */
883	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
884	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
885	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
886	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
887	} while (0)
888	#else
889	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
890	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
891	} while (0)
892	#endif
893
894	/** Adds a segment limit register as input to the outgoing state. */
895	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
896	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
897	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
898	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
899	} while (0)
900	#else
901	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
902	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
903	} while (0)
904	#endif
905
906	/** Adds a segment limit register as input to the outgoing state. */
907	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
908	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
909	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
910	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
911	} while (0)
912	#else
913	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
914	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
915	} while (0)
916	#endif
917	/** @} */
918
919	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
920	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
921	* calculated and up to date. This is to double check that we haven't skipped
922	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
923	* @note has to be placed in
924	*/
925	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
926	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
927	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
928	#else
929	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
930	#endif
931
932
933	/**
934	* Guest registers that can be shadowed in GPRs.
935	*
936	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
937	* must be placed last, as the liveness state tracks it as 7 subcomponents and
938	* we don't want to waste space here.
939	*
940	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
941	* friends as well as IEMAllN8veLiveness.cpp.
942	*/
943	typedef enum IEMNATIVEGSTREG : uint8_t
944	{
945	kIemNativeGstReg_GprFirst = 0,
946	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
947	kIemNativeGstReg_Pc,
948	kIemNativeGstReg_Cr0,
949	kIemNativeGstReg_FpuFcw,
950	kIemNativeGstReg_FpuFsw,
951	kIemNativeGstReg_SegBaseFirst,
952	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
953	kIemNativeGstReg_SegAttribFirst,
954	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
955	kIemNativeGstReg_SegLimitFirst,
956	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
957	kIemNativeGstReg_SegSelFirst,
958	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
959	kIemNativeGstReg_Cr4,
960	kIemNativeGstReg_Xcr0,
961	kIemNativeGstReg_MxCsr,
962	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
963	kIemNativeGstReg_End
964	} IEMNATIVEGSTREG;
965	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
966	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
967
968	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
969	* @{ */
970	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
971	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
972	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
973	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
974	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
975	/** @} */
976
977	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
978
979	/**
980	* Guest registers that can be shadowed in host SIMD registers.
981	*
982	* @todo r=aeichner Liveness tracking
983	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
984	*/
985	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
986	{
987	kIemNativeGstSimdReg_SimdRegFirst = 0,
988	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
989	kIemNativeGstSimdReg_End
990	} IEMNATIVEGSTSIMDREG;
991
992	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
993	* @{ */
994	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
995	/** @} */
996
997	/**
998	* The Load/store size for a SIMD guest register.
999	*/
1000	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1001	{
1002	/** Invalid size. */
1003	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1004	/** Loads the low 128-bit of a guest SIMD register. */
1005	kIemNativeGstSimdRegLdStSz_Low128,
1006	/** Loads the high 128-bit of a guest SIMD register. */
1007	kIemNativeGstSimdRegLdStSz_High128,
1008	/** Loads the whole 256-bits of a guest SIMD register. */
1009	kIemNativeGstSimdRegLdStSz_256,
1010	/** End value. */
1011	kIemNativeGstSimdRegLdStSz_End
1012	} IEMNATIVEGSTSIMDREGLDSTSZ;
1013
1014	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1015
1016	/**
1017	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1018	*/
1019	typedef enum IEMNATIVEGSTREGUSE
1020	{
1021	/** The usage is read-only, the register holding the guest register
1022	* shadow copy will not be modified by the caller. */
1023	kIemNativeGstRegUse_ReadOnly = 0,
1024	/** The caller will update the guest register (think: PC += cbInstr).
1025	* The guest shadow copy will follow the returned register. */
1026	kIemNativeGstRegUse_ForUpdate,
1027	/** The call will put an entirely new value in the guest register, so
1028	* if new register is allocate it will be returned uninitialized. */
1029	kIemNativeGstRegUse_ForFullWrite,
1030	/** The caller will use the guest register value as input in a calculation
1031	* and the host register will be modified.
1032	* This means that the returned host register will not be marked as a shadow
1033	* copy of the guest register. */
1034	kIemNativeGstRegUse_Calculation
1035	} IEMNATIVEGSTREGUSE;
1036
1037	/**
1038	* Guest registers (classes) that can be referenced.
1039	*/
1040	typedef enum IEMNATIVEGSTREGREF : uint8_t
1041	{
1042	kIemNativeGstRegRef_Invalid = 0,
1043	kIemNativeGstRegRef_Gpr,
1044	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1045	kIemNativeGstRegRef_EFlags,
1046	kIemNativeGstRegRef_MxCsr,
1047	kIemNativeGstRegRef_FpuReg,
1048	kIemNativeGstRegRef_MReg,
1049	kIemNativeGstRegRef_XReg,
1050	kIemNativeGstRegRef_X87,
1051	kIemNativeGstRegRef_XState,
1052	//kIemNativeGstRegRef_YReg, - doesn't work.
1053	kIemNativeGstRegRef_End
1054	} IEMNATIVEGSTREGREF;
1055
1056
1057	/** Variable kinds. */
1058	typedef enum IEMNATIVEVARKIND : uint8_t
1059	{
1060	/** Customary invalid zero value. */
1061	kIemNativeVarKind_Invalid = 0,
1062	/** This is either in a register or on the stack. */
1063	kIemNativeVarKind_Stack,
1064	/** Immediate value - loaded into register when needed, or can live on the
1065	* stack if referenced (in theory). */
1066	kIemNativeVarKind_Immediate,
1067	/** Variable reference - loaded into register when needed, never stack. */
1068	kIemNativeVarKind_VarRef,
1069	/** Guest register reference - loaded into register when needed, never stack. */
1070	kIemNativeVarKind_GstRegRef,
1071	/** End of valid values. */
1072	kIemNativeVarKind_End
1073	} IEMNATIVEVARKIND;
1074
1075
1076	/** Variable or argument. */
1077	typedef struct IEMNATIVEVAR
1078	{
1079	/** The kind of variable. */
1080	IEMNATIVEVARKIND enmKind;
1081	/** The variable size in bytes. */
1082	uint8_t cbVar;
1083	/** The first stack slot (uint64_t), except for immediate and references
1084	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1085	* has a stack slot it has been initialized and has a value. Unused variables
1086	* has neither a stack slot nor a host register assignment. */
1087	uint8_t idxStackSlot;
1088	/** The host register allocated for the variable, UINT8_MAX if not. */
1089	uint8_t idxReg;
1090	/** The argument number if argument, UINT8_MAX if regular variable. */
1091	uint8_t uArgNo;
1092	/** If referenced, the index (unpacked) of the variable referencing this one,
1093	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1094	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1095	uint8_t idxReferrerVar;
1096	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1097	* @todo not sure what this really is for... */
1098	IEMNATIVEGSTREG enmGstReg;
1099	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1100	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1101	* only valid when idxReg is not UINT8_MAX. */
1102	bool fSimdReg : 1;
1103	/** Set if the registered is currently used exclusively, false if the
1104	* variable is idle and the register can be grabbed. */
1105	bool fRegAcquired : 1;
1106	#else
1107	/** Set if the registered is currently used exclusively, false if the
1108	* variable is idle and the register can be grabbed. */
1109	bool fRegAcquired;
1110	#endif
1111
1112	union
1113	{
1114	/** kIemNativeVarKind_Immediate: The immediate value. */
1115	uint64_t uValue;
1116	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1117	uint8_t idxRefVar;
1118	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1119	struct
1120	{
1121	/** The class of register. */
1122	IEMNATIVEGSTREGREF enmClass;
1123	/** Index within the class. */
1124	uint8_t idx;
1125	} GstRegRef;
1126	} u;
1127	} IEMNATIVEVAR;
1128	/** Pointer to a variable or argument. */
1129	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1130	/** Pointer to a const variable or argument. */
1131	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1132
1133	/** What is being kept in a host register. */
1134	typedef enum IEMNATIVEWHAT : uint8_t
1135	{
1136	/** The traditional invalid zero value. */
1137	kIemNativeWhat_Invalid = 0,
1138	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1139	kIemNativeWhat_Var,
1140	/** Temporary register, this is typically freed when a MC completes. */
1141	kIemNativeWhat_Tmp,
1142	/** Call argument w/o a variable mapping. This is free (via
1143	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1144	kIemNativeWhat_Arg,
1145	/** Return status code.
1146	* @todo not sure if we need this... */
1147	kIemNativeWhat_rc,
1148	/** The fixed pVCpu (PVMCPUCC) register.
1149	* @todo consider offsetting this on amd64 to use negative offsets to access
1150	* more members using 8-byte disp. */
1151	kIemNativeWhat_pVCpuFixed,
1152	/** The fixed pCtx (PCPUMCTX) register.
1153	* @todo consider offsetting this on amd64 to use negative offsets to access
1154	* more members using 8-byte disp. */
1155	kIemNativeWhat_pCtxFixed,
1156	/** Fixed temporary register. */
1157	kIemNativeWhat_FixedTmp,
1158	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1159	/** Shadow RIP for the delayed RIP updating debugging. */
1160	kIemNativeWhat_PcShadow,
1161	#endif
1162	/** Register reserved by the CPU or OS architecture. */
1163	kIemNativeWhat_FixedReserved,
1164	/** End of valid values. */
1165	kIemNativeWhat_End
1166	} IEMNATIVEWHAT;
1167
1168	/**
1169	* Host general register entry.
1170	*
1171	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1172	*
1173	* @todo Track immediate values in host registers similarlly to how we track the
1174	* guest register shadow copies. For it to be real helpful, though,
1175	* we probably need to know which will be reused and put them into
1176	* non-volatile registers, otherwise it's going to be more or less
1177	* restricted to an instruction or two.
1178	*/
1179	typedef struct IEMNATIVEHSTREG
1180	{
1181	/** Set of guest registers this one shadows.
1182	*
1183	* Using a bitmap here so we can designate the same host register as a copy
1184	* for more than one guest register. This is expected to be useful in
1185	* situations where one value is copied to several registers in a sequence.
1186	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1187	* sequence we'd want to let this register follow to be a copy of and there
1188	* will always be places where we'd be picking the wrong one.
1189	*/
1190	uint64_t fGstRegShadows;
1191	/** What is being kept in this register. */
1192	IEMNATIVEWHAT enmWhat;
1193	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1194	uint8_t idxVar;
1195	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1196	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1197	* that scope. */
1198	uint8_t idxStackSlot;
1199	/** Alignment padding. */
1200	uint8_t abAlign[5];
1201	} IEMNATIVEHSTREG;
1202
1203
1204	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1205	/**
1206	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1207	* halves, on architectures where there is no 256-bit register available this entry will track
1208	* two adjacent 128-bit host registers.
1209	*
1210	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1211	*/
1212	typedef struct IEMNATIVEHSTSIMDREG
1213	{
1214	/** Set of guest registers this one shadows.
1215	*
1216	* Using a bitmap here so we can designate the same host register as a copy
1217	* for more than one guest register. This is expected to be useful in
1218	* situations where one value is copied to several registers in a sequence.
1219	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1220	* sequence we'd want to let this register follow to be a copy of and there
1221	* will always be places where we'd be picking the wrong one.
1222	*/
1223	uint64_t fGstRegShadows;
1224	/** What is being kept in this register. */
1225	IEMNATIVEWHAT enmWhat;
1226	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1227	uint8_t idxVar;
1228	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1229	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1230	/** Alignment padding. */
1231	uint8_t abAlign[5];
1232	} IEMNATIVEHSTSIMDREG;
1233	#endif
1234
1235
1236	/**
1237	* Core state for the native recompiler, that is, things that needs careful
1238	* handling when dealing with branches.
1239	*/
1240	typedef struct IEMNATIVECORESTATE
1241	{
1242	/** Allocation bitmap for aHstRegs. */
1243	uint32_t bmHstRegs;
1244
1245	/** Bitmap marking which host register contains guest register shadow copies.
1246	* This is used during register allocation to try preserve copies. */
1247	uint32_t bmHstRegsWithGstShadow;
1248	/** Bitmap marking valid entries in aidxGstRegShadows. */
1249	uint64_t bmGstRegShadows;
1250	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1251	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1252	uint64_t bmGstRegShadowDirty;
1253	#endif
1254
1255	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1256	/** The current instruction offset in bytes from when the guest program counter
1257	* was updated last. Used for delaying the write to the guest context program counter
1258	* as long as possible. */
1259	int64_t offPc;
1260	# if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1261	/** Statistics: The idxInstr+1 value at the last PC update. */
1262	uint8_t idxInstrPlusOneOfLastPcUpdate;
1263	# endif
1264	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1265	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1266	bool fDebugPcInitialized;
1267	# endif
1268	#endif
1269
1270	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1271	/** Allocation bitmap for aHstSimdRegs. */
1272	uint32_t bmHstSimdRegs;
1273
1274	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1275	* This is used during register allocation to try preserve copies. */
1276	uint32_t bmHstSimdRegsWithGstShadow;
1277	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1278	uint64_t bmGstSimdRegShadows;
1279	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1280	uint64_t bmGstSimdRegShadowDirtyLo128;
1281	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1282	uint64_t bmGstSimdRegShadowDirtyHi128;
1283	#endif
1284
1285	union
1286	{
1287	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1288	uint8_t aidxArgVars[8];
1289	/** For more efficient resetting. */
1290	uint64_t u64ArgVars;
1291	};
1292
1293	/** Allocation bitmap for the stack. */
1294	uint32_t bmStack;
1295	/** Allocation bitmap for aVars. */
1296	uint32_t bmVars;
1297
1298	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1299	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1300	* (A shadow copy of a guest register can only be held in a one host register,
1301	* there are no duplicate copies or ambiguities like that). */
1302	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1303	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1304	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1305	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1306	* (A shadow copy of a guest register can only be held in a one host register,
1307	* there are no duplicate copies or ambiguities like that). */
1308	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1309	#endif
1310
1311	/** Host register allocation tracking. */
1312	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1313	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1314	/** Host SIMD register allocation tracking. */
1315	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1316	#endif
1317
1318	/** Variables and arguments. */
1319	IEMNATIVEVAR aVars[9];
1320	} IEMNATIVECORESTATE;
1321	/** Pointer to core state. */
1322	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1323	/** Pointer to const core state. */
1324	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1325
1326	/** @def IEMNATIVE_VAR_IDX_UNPACK
1327	* @returns Index into IEMNATIVECORESTATE::aVars.
1328	* @param a_idxVar Variable index w/ magic (in strict builds).
1329	*/
1330	/** @def IEMNATIVE_VAR_IDX_PACK
1331	* @returns Variable index w/ magic (in strict builds).
1332	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1333	*/
1334	#ifdef VBOX_STRICT
1335	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1336	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1337	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1338	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1339	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1340	#else
1341	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1342	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1343	#endif
1344
1345
1346	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1347	/** Clear the dirty state of the given guest SIMD register. */
1348	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1349	do { \
1350	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1351	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1352	} while (0)
1353
1354	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1355	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1356	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1357	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1358	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1359	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1360	/** Returns whether the given guest SIMD register is dirty. */
1361	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1362	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1363
1364	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1365	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1366	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1367	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1368	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1369	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1370
1371	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1372	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1373	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1374	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1375	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1376	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1377	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1378	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1379	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1380	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1381	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1382	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1383	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1384	# endif
1385	#endif
1386
1387
1388	/**
1389	* Conditional stack entry.
1390	*/
1391	typedef struct IEMNATIVECOND
1392	{
1393	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1394	bool fInElse;
1395	union
1396	{
1397	struct
1398	{
1399	/** Set if the if-block unconditionally exited the TB. */
1400	bool fIfExitTb;
1401	/** Set if the else-block unconditionally exited the TB. */
1402	bool fElseExitTb;
1403	};
1404	/** Indexed by fInElse. */
1405	bool afExitTb[2];
1406	};
1407	bool afPadding[5];
1408	/** The label for the IEM_MC_ELSE. */
1409	uint32_t idxLabelElse;
1410	/** The label for the IEM_MC_ENDIF. */
1411	uint32_t idxLabelEndIf;
1412	/** The initial state snapshot as the if-block starts executing. */
1413	IEMNATIVECORESTATE InitialState;
1414	/** The state snapshot at the end of the if-block. */
1415	IEMNATIVECORESTATE IfFinalState;
1416	} IEMNATIVECOND;
1417	/** Pointer to a condition stack entry. */
1418	typedef IEMNATIVECOND *PIEMNATIVECOND;
1419
1420
1421	/**
1422	* Native recompiler state.
1423	*/
1424	typedef struct IEMRECOMPILERSTATE
1425	{
1426	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1427	* IEMNATIVEINSTR units. */
1428	uint32_t cInstrBufAlloc;
1429	#ifdef VBOX_STRICT
1430	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1431	uint32_t offInstrBufChecked;
1432	#else
1433	uint32_t uPadding1; /* We don't keep track of the size here... */
1434	#endif
1435	/** Fixed temporary code buffer for native recompilation. */
1436	PIEMNATIVEINSTR pInstrBuf;
1437
1438	/** Bitmaps with the label types used. */
1439	uint64_t bmLabelTypes;
1440	/** Actual number of labels in paLabels. */
1441	uint32_t cLabels;
1442	/** Max number of entries allowed in paLabels before reallocating it. */
1443	uint32_t cLabelsAlloc;
1444	/** Labels defined while recompiling (referenced by fixups). */
1445	PIEMNATIVELABEL paLabels;
1446	/** Array with indexes of unique labels (uData always 0). */
1447	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1448
1449	/** Actual number of fixups paFixups. */
1450	uint32_t cFixups;
1451	/** Max number of entries allowed in paFixups before reallocating it. */
1452	uint32_t cFixupsAlloc;
1453	/** Buffer used by the recompiler for recording fixups when generating code. */
1454	PIEMNATIVEFIXUP paFixups;
1455
1456	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1457	/** Actual number of fixups in paTbExitFixups. */
1458	uint32_t cTbExitFixups;
1459	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1460	uint32_t cTbExitFixupsAlloc;
1461	/** Buffer used by the recompiler for recording fixups when generating code. */
1462	PIEMNATIVEEXITFIXUP paTbExitFixups;
1463	#endif
1464
1465	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1466	/** Number of debug info entries allocated for pDbgInfo. */
1467	uint32_t cDbgInfoAlloc;
1468	uint32_t uPadding;
1469	/** Debug info. */
1470	PIEMTBDBG pDbgInfo;
1471	#endif
1472
1473	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1474	/** The current call index (liveness array and threaded calls in TB). */
1475	uint32_t idxCurCall;
1476	/** Number of liveness entries allocated. */
1477	uint32_t cLivenessEntriesAlloc;
1478	/** Liveness entries for all the calls in the TB begin recompiled.
1479	* The entry for idxCurCall contains the info for what the next call will
1480	* require wrt registers. (Which means the last entry is the initial liveness
1481	* state.) */
1482	PIEMLIVENESSENTRY paLivenessEntries;
1483	#endif
1484
1485	/** The translation block being recompiled. */
1486	PCIEMTB pTbOrg;
1487	/** The VMCPU structure of the EMT. */
1488	PVMCPUCC pVCpu;
1489
1490	/** Condition sequence number (for generating unique labels). */
1491	uint16_t uCondSeqNo;
1492	/** Check IRQ sequence number (for generating unique labels). */
1493	uint16_t uCheckIrqSeqNo;
1494	/** TLB load sequence number (for generating unique labels). */
1495	uint16_t uTlbSeqNo;
1496	/** The current condition stack depth (aCondStack). */
1497	uint8_t cCondDepth;
1498
1499	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1500	uint8_t cArgsX;
1501	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1502	uint32_t fCImpl;
1503	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1504	uint32_t fMc;
1505	/** The expected IEMCPU::fExec value for the current call/instruction. */
1506	uint32_t fExec;
1507	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1508	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1509	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1510	*
1511	* This is an optimization because these control registers can only be changed from
1512	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1513	* consisting of multiple SIMD instructions.
1514	*/
1515	uint32_t fSimdRaiseXcptChecksEmitted;
1516	#endif
1517	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1518	uint32_t idxLastCheckIrqCallNo;
1519
1520	/** Core state requiring care with branches. */
1521	IEMNATIVECORESTATE Core;
1522
1523	/** The condition nesting stack. */
1524	IEMNATIVECOND aCondStack[2];
1525
1526	#ifndef IEM_WITH_THROW_CATCH
1527	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1528	* for recompilation error handling. */
1529	jmp_buf JmpBuf;
1530	#endif
1531	} IEMRECOMPILERSTATE;
1532	/** Pointer to a native recompiler state. */
1533	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1534
1535
1536	/** @def IEMNATIVE_TRY_SETJMP
1537	* Wrapper around setjmp / try, hiding all the ugly differences.
1538	*
1539	* @note Use with extreme care as this is a fragile macro.
1540	* @param a_pReNative The native recompile state.
1541	* @param a_rcTarget The variable that should receive the status code in case
1542	* of a longjmp/throw.
1543	*/
1544	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1545	* Start wrapper for catch / setjmp-else.
1546	*
1547	* This will set up a scope.
1548	*
1549	* @note Use with extreme care as this is a fragile macro.
1550	* @param a_pReNative The native recompile state.
1551	* @param a_rcTarget The variable that should receive the status code in case
1552	* of a longjmp/throw.
1553	*/
1554	/** @def IEMNATIVE_CATCH_LONGJMP_END
1555	* End wrapper for catch / setjmp-else.
1556	*
1557	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1558	* up the state.
1559	*
1560	* @note Use with extreme care as this is a fragile macro.
1561	* @param a_pReNative The native recompile state.
1562	*/
1563	/** @def IEMNATIVE_DO_LONGJMP
1564	*
1565	* Wrapper around longjmp / throw.
1566	*
1567	* @param a_pReNative The native recompile state.
1568	* @param a_rc The status code jump back with / throw.
1569	*/
1570	#ifdef IEM_WITH_THROW_CATCH
1571	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1572	a_rcTarget = VINF_SUCCESS; \
1573	try
1574	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1575	catch (int rcThrown) \
1576	{ \
1577	a_rcTarget = rcThrown
1578	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1579	} \
1580	((void)0)
1581	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1582	#else /* !IEM_WITH_THROW_CATCH */
1583	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1584	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1585	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1586	else \
1587	{ \
1588	((void)0)
1589	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1590	}
1591	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1592	#endif /* !IEM_WITH_THROW_CATCH */
1593
1594
1595	/**
1596	* Native recompiler worker for a threaded function.
1597	*
1598	* @returns New code buffer offset; throws VBox status code in case of a failure.
1599	* @param pReNative The native recompiler state.
1600	* @param off The current code buffer offset.
1601	* @param pCallEntry The threaded call entry.
1602	*
1603	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1604	*/
1605	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1606	/** Pointer to a native recompiler worker for a threaded function. */
1607	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1608
1609	/** Defines a native recompiler worker for a threaded function.
1610	* @see FNIEMNATIVERECOMPFUNC */
1611	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1612	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1613
1614	/** Prototypes a native recompiler function for a threaded function.
1615	* @see FNIEMNATIVERECOMPFUNC */
1616	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1617
1618
1619	/**
1620	* Native recompiler liveness analysis worker for a threaded function.
1621	*
1622	* @param pCallEntry The threaded call entry.
1623	* @param pIncoming The incoming liveness state entry.
1624	* @param pOutgoing The outgoing liveness state entry.
1625	*/
1626	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1627	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1628	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1629	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1630
1631	/** Defines a native recompiler liveness analysis worker for a threaded function.
1632	* @see FNIEMNATIVELIVENESSFUNC */
1633	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1634	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1635
1636	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1637	* @see FNIEMNATIVELIVENESSFUNC */
1638	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1639
1640
1641	/** Define a native recompiler helper function, safe to call from the TB code. */
1642	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1643	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1644	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1645	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1646	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1647	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1648	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1649	a_RetType (VBOXCALL *a_Name) a_ArgList
1650
1651
1652	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1653	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1654	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1655	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1656	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1657	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1658	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1659	uint8_t idxHstSimdReg = UINT8_MAX,
1660	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1661	# endif
1662	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1663	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1664	uint8_t idxGstReg, uint8_t idxHstReg);
1665	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1666	uint64_t fGstReg);
1667	# endif
1668	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1669	uint64_t offPc, uint32_t cInstrSkipped);
1670	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1671
1672	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1673	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1674	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1675	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1676	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1677	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1678	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1679	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1680	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1681	#endif
1682	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1683
1684	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1685	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1686	bool fPreferVolatile = true);
1687	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1688	bool fPreferVolatile = true);
1689	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1690	IEMNATIVEGSTREG enmGstReg,
1691	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1692	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1693	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1694	IEMNATIVEGSTREG enmGstReg);
1695
1696	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1697	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1698	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1699	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1700	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1701	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1702	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1703	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1704	# endif
1705	#endif
1706	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1707	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1708	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1709	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1710	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1711	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1712	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1713	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1714	# endif
1715	#endif
1716	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1717	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1718	uint32_t fKeepVars = 0);
1719	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1720	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1721	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1722	uint32_t fHstRegsActiveShadows);
1723	#ifdef VBOX_STRICT
1724	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1725	#endif
1726	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1727	uint64_t fGstSimdShwExcept);
1728	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1729	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1730	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1731	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
1732	# endif
1733	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1734	#endif
1735	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1736	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1737	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1738	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
1739	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1740	uint64_t fFlushGstReg = UINT64_MAX);
1741	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
1742	uint32_t off, uint8_t idxHstReg);
1743	#endif
1744
1745
1746	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1747	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1748	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1749	bool fPreferVolatile = true);
1750	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1751	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1752	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1753	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1754	bool fNoVolatileRegs = false);
1755	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1756	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1757	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1758	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1759	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1760	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1761	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1762	#endif
1763
1764	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1765	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1766	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1767	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1768	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1769	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1770	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1771	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1772	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1773	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1774	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1775	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1776	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1777	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1778	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1779	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1780	#endif
1781	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1782	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1783	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1784	uint32_t fHstRegsNotToSave);
1785	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1786	uint32_t fHstRegsNotToSave);
1787	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1788	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1789
1790	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1791	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1792	#ifdef VBOX_STRICT
1793	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1794	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1795	IEMNATIVEGSTREG enmGstReg);
1796	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1797	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1798	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1799	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1800	# endif
1801	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1802	#endif
1803	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1804	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1805	#endif
1806	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1807	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1808	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1809	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1810	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1811	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1812	PCIEMTHRDEDCALLENTRY pCallEntry);
1813	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1814	uint8_t idxAddrReg, uint8_t idxInstr);
1815	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1816	uint8_t idxAddrReg, uint8_t idxInstr);
1817	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1818	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1819
1820
1821	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1822	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1823	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1824	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1825	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1826	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1827	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1828	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1829	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1830	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1831	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1832	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1833	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1834
1835	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1836	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1837	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1838	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1839	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1840	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1841	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1842	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1843	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1844	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1845	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1846	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1847	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1848	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1849	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1850	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1851	#endif
1852	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1853	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1854	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1855	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1856	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1857	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1858	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1859	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1860	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1861	#endif
1862	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1863	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1864	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1865	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1866	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1867	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1868	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1869
1870	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1871	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1872	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1873	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1874	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1875	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1876	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1877	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1878	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1879	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1880	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1881	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1882	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1883	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1884	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1885	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1886	#endif
1887	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1888	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1889	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1890	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1891	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1892	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1893	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1894	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1895	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1896	#endif
1897	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1898	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1899	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1900	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1901	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1902	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1903	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1904
1905	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1906	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1907	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1908	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1909	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1910	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1911	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1912	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1913	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1914	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1915	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1916	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1917	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1918	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1919	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1920	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1921	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1922	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1923	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1924	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1925	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1926	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1927
1928	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1929	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1930	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1931	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1932	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1933	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1934	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1935	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1936	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1937	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1938	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1939	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1940	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1941	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1942	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1943	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1944	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1945	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1946	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1947	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1948	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1949	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1950
1951	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1952	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1953	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1954	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1955
1956
1957	/**
1958	* Info about shadowed guest register values.
1959	* @see IEMNATIVEGSTREG
1960	*/
1961	typedef struct IEMANTIVEGSTREGINFO
1962	{
1963	/** Offset in VMCPU. */
1964	uint32_t off;
1965	/** The field size. */
1966	uint8_t cb;
1967	/** Name (for logging). */
1968	const char *pszName;
1969	} IEMANTIVEGSTREGINFO;
1970	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1971	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1972	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1973	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1974	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1975
1976
1977
1978	/**
1979	* Ensures that there is sufficient space in the instruction output buffer.
1980	*
1981	* This will reallocate the buffer if needed and allowed.
1982	*
1983	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1984	* allocation size.
1985	*
1986	* @returns Pointer to the instruction output buffer on success; throws VBox
1987	* status code on failure, so no need to check it.
1988	* @param pReNative The native recompile state.
1989	* @param off Current instruction offset. Works safely for UINT32_MAX
1990	* as well.
1991	* @param cInstrReq Number of instruction about to be added. It's okay to
1992	* overestimate this a bit.
1993	*/
1994	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1995	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1996	{
1997	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1998	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1999	{
2000	#ifdef VBOX_STRICT
2001	pReNative->offInstrBufChecked = offChecked;
2002	#endif
2003	return pReNative->pInstrBuf;
2004	}
2005	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2006	}
2007
2008	/**
2009	* Checks that we didn't exceed the space requested in the last
2010	* iemNativeInstrBufEnsure() call.
2011	*/
2012	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2013	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2014	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2015
2016	/**
2017	* Checks that a variable index is valid.
2018	*/
2019	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2020	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2021	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2022	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2023	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2024	("%s=%#x\n", #a_idxVar, a_idxVar))
2025	#else
2026	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2027	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2028	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2029	#endif
2030
2031	/**
2032	* Checks that a variable index is valid and that the variable is assigned the
2033	* correct argument number.
2034	* This also adds a RT_NOREF of a_idxVar.
2035	*/
2036	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2037	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2038	RT_NOREF_PV(a_idxVar); \
2039	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2040	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2041	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2042	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2043	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2044	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2045	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2046	a_uArgNo)); \
2047	} while (0)
2048	#else
2049	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2050	RT_NOREF_PV(a_idxVar); \
2051	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2052	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2053	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2054	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2055	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2056	} while (0)
2057	#endif
2058
2059
2060	/**
2061	* Checks that a variable has the expected size.
2062	*/
2063	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2064	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2065	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2066	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2067
2068
2069	/**
2070	* Calculates the stack address of a variable as a [r]BP displacement value.
2071	*/
2072	DECL_FORCE_INLINE(int32_t)
2073	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2074	{
2075	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2076	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2077	}
2078
2079
2080	/**
2081	* Releases the variable's register.
2082	*
2083	* The register must have been previously acquired calling
2084	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2085	* iemNativeVarRegisterSetAndAcquire().
2086	*/
2087	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2088	{
2089	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2090	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2091	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2092	}
2093
2094
2095	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2096	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2097	{
2098	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2099	iemNativeVarRegisterRelease(pReNative, idxVar);
2100	}
2101	#endif
2102
2103
2104	/**
2105	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2106	*
2107	* @returns The flush mask.
2108	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2109	* @param fGstShwFlush The starting flush mask.
2110	*/
2111	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2112	{
2113	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2114	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2115	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2116	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2117	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2118	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2119	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2120	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2121	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2122	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2123	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2124	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2125	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2126	return fGstShwFlush;
2127	}
2128
2129
2130	/** Number of hidden arguments for CIMPL calls.
2131	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2132	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2133	# define IEM_CIMPL_HIDDEN_ARGS 3
2134	#else
2135	# define IEM_CIMPL_HIDDEN_ARGS 2
2136	#endif
2137
2138
2139	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2140	/** Number of hidden arguments for SSE_AIMPL calls. */
2141	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2142	/** Number of hidden arguments for AVX_AIMPL calls. */
2143	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2144	#endif
2145
2146
2147	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2148
2149	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2150	/**
2151	* Helper for iemNativeLivenessGetStateByGstReg.
2152	*
2153	* @returns IEMLIVENESS_STATE_XXX
2154	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2155	* ORed together.
2156	*/
2157	DECL_FORCE_INLINE(uint32_t)
2158	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2159	{
2160	/* INPUT trumps anything else. */
2161	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2162	return IEMLIVENESS_STATE_INPUT;
2163
2164	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2165	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2166	{
2167	/* If not all sub-fields are clobbered they must be considered INPUT. */
2168	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2169	return IEMLIVENESS_STATE_INPUT;
2170	return IEMLIVENESS_STATE_CLOBBERED;
2171	}
2172
2173	/* XCPT_OR_CALL trumps UNUSED. */
2174	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2175	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2176
2177	return IEMLIVENESS_STATE_UNUSED;
2178	}
2179	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2180
2181
2182	DECL_FORCE_INLINE(uint32_t)
2183	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2184	{
2185	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2186	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2187	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2188	# else
2189	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2190	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2191	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2192	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2193	# endif
2194	}
2195
2196
2197	DECL_FORCE_INLINE(uint32_t)
2198	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2199	{
2200	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2201	if (enmGstReg == kIemNativeGstReg_EFlags)
2202	{
2203	/* Merge the eflags states to one. */
2204	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2205	uRet = RT_BIT_32(uRet);
2206	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2207	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2208	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2209	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2210	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2211	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2212	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2213	# else
2214	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2215	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2216	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2217	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2218	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2219	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2220	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2221	# endif
2222	}
2223	return uRet;
2224	}
2225
2226
2227	# ifdef VBOX_STRICT
2228	/** For assertions only, user checks that idxCurCall isn't zerow. */
2229	DECL_FORCE_INLINE(uint32_t)
2230	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2231	{
2232	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2233	}
2234	# endif /* VBOX_STRICT */
2235
2236	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2237
2238
2239	/**
2240	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2241	*/
2242	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2243	{
2244	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2245	return IEM_CIMPL_HIDDEN_ARGS;
2246	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2247	return 1;
2248	return 0;
2249	}
2250
2251
2252	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2253	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2254	{
2255	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2256
2257	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2258	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2259	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2260	return (uint8_t)idxReg;
2261	}
2262
2263
2264
2265	/*********************************************************************************************************************************
2266	* Register Allocator (GPR) *
2267	*********************************************************************************************************************************/
2268
2269	/**
2270	* Marks host register @a idxHstReg as containing a shadow copy of guest
2271	* register @a enmGstReg.
2272	*
2273	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2274	* host register before calling.
2275	*/
2276	DECL_FORCE_INLINE(void)
2277	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2278	{
2279	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2280	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2281	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2282
2283	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2284	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2285	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2286	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2287	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2288	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2289	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2290	#else
2291	RT_NOREF(off);
2292	#endif
2293	}
2294
2295
2296	/**
2297	* Clear any guest register shadow claims from @a idxHstReg.
2298	*
2299	* The register does not need to be shadowing any guest registers.
2300	*/
2301	DECL_FORCE_INLINE(void)
2302	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2303	{
2304	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2305	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2306	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2307	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2308	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2309	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2310	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2311	#endif
2312
2313	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2314	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2315	if (fGstRegs)
2316	{
2317	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2318	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2319	while (fGstRegs)
2320	{
2321	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2322	fGstRegs &= ~RT_BIT_64(iGstReg);
2323	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2324	}
2325	}
2326	#else
2327	RT_NOREF(off);
2328	#endif
2329
2330	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2331	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2332	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2333	}
2334
2335
2336	/**
2337	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2338	* and global overview flags.
2339	*/
2340	DECL_FORCE_INLINE(void)
2341	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2342	{
2343	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2344	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2345	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2346	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2347	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2348	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2349	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2350	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2351	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2352	#endif
2353
2354	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2355	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2356	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2357	#else
2358	RT_NOREF(off);
2359	#endif
2360
2361	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2362	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2363	if (!fGstRegShadowsNew)
2364	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2365	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2366	}
2367
2368
2369	#if 0 /* unused */
2370	/**
2371	* Clear any guest register shadow claim for @a enmGstReg.
2372	*/
2373	DECL_FORCE_INLINE(void)
2374	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2375	{
2376	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2377	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2378	{
2379	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2380	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2381	}
2382	}
2383	#endif
2384
2385
2386	/**
2387	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2388	* as the new shadow of it.
2389	*
2390	* Unlike the other guest reg shadow helpers, this does the logging for you.
2391	* However, it is the liveness state is not asserted here, the caller must do
2392	* that.
2393	*/
2394	DECL_FORCE_INLINE(void)
2395	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2396	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2397	{
2398	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2399	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2400	{
2401	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2402	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2403	if (idxHstRegOld == idxHstRegNew)
2404	return;
2405	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2406	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2407	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2408	}
2409	else
2410	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2411	g_aGstShadowInfo[enmGstReg].pszName));
2412	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2413	}
2414
2415
2416	/**
2417	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2418	* to @a idxRegTo.
2419	*/
2420	DECL_FORCE_INLINE(void)
2421	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2422	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2423	{
2424	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2425	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2426	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2427	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2428	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2429	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2430	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2431	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2432	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2433
2434	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2435	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2436	if (!fGstRegShadowsFrom)
2437	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2438	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2439	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2440	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2441	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2442	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2443	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2444	#else
2445	RT_NOREF(off);
2446	#endif
2447	}
2448
2449
2450	/**
2451	* Flushes any delayed guest register writes.
2452	*
2453	* This must be called prior to calling CImpl functions and any helpers that use
2454	* the guest state (like raising exceptions) and such.
2455	*
2456	* This optimization has not yet been implemented. The first target would be
2457	* RIP updates, since these are the most common ones.
2458	*
2459	* @note This function does not flush any shadowing information for guest
2460	* registers. This needs to be done by the caller if it wishes to do so.
2461	*/
2462	DECL_INLINE_THROW(uint32_t)
2463	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
2464	uint64_t fGstSimdShwExcept = 0)
2465	{
2466	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2467	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2468	#else
2469	uint64_t const bmGstRegShadowDirty = 0;
2470	#endif
2471	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2472	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
2473	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2474	& ~fGstSimdShwExcept;
2475	#else
2476	uint64_t const bmGstSimdRegShadowDirty = 0;
2477	#endif
2478	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2479	uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2480	#else
2481	uint64_t const fWritebackPc = 0;
2482	#endif
2483	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2484	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2485
2486	return off;
2487	}
2488
2489
2490
2491	/*********************************************************************************************************************************
2492	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2493	*********************************************************************************************************************************/
2494
2495	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2496
2497	DECL_FORCE_INLINE(uint8_t)
2498	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2499	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2500	{
2501	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2502
2503	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2504	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2505	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2506	return idxSimdReg;
2507	}
2508
2509
2510	/**
2511	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2512	* SIMD register @a enmGstSimdReg.
2513	*
2514	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2515	* host register before calling.
2516	*/
2517	DECL_FORCE_INLINE(void)
2518	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2519	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2520	{
2521	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2522	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2523	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2524
2525	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2526	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2527	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2528	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2529	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2530	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2531	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2532	#else
2533	RT_NOREF(off);
2534	#endif
2535	}
2536
2537
2538	/**
2539	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2540	* to @a idxSimdRegTo.
2541	*/
2542	DECL_FORCE_INLINE(void)
2543	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2544	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2545	{
2546	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2547	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2548	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2549	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2550	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2551	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2552	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2553	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2554	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2555	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2556	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2557
2558	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2559	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2560	if (!fGstRegShadowsFrom)
2561	{
2562	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2563	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2564	}
2565	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2566	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2567	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2568	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2569	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2570	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2571	#else
2572	RT_NOREF(off);
2573	#endif
2574	}
2575
2576
2577	/**
2578	* Clear any guest register shadow claims from @a idxHstSimdReg.
2579	*
2580	* The register does not need to be shadowing any guest registers.
2581	*/
2582	DECL_FORCE_INLINE(void)
2583	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2584	{
2585	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2586	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2587	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2588	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2589	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2590	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2591	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2592
2593	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2594	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2595	if (fGstRegs)
2596	{
2597	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2598	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2599	while (fGstRegs)
2600	{
2601	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2602	fGstRegs &= ~RT_BIT_64(iGstReg);
2603	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2604	}
2605	}
2606	#else
2607	RT_NOREF(off);
2608	#endif
2609
2610	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2611	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2612	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2613	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2614	}
2615
2616	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2617
2618
2619	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2620	/**
2621	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2622	*/
2623	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2624	{
2625	if (pReNative->Core.offPc)
2626	return iemNativeEmitPcWritebackSlow(pReNative, off);
2627	return off;
2628	}
2629	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2630
2631
2632	#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2633	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2634	* it saves us the trouble of a hidden parameter on MSC/amd64. */
2635	# ifdef RT_ARCH_AMD64
2636	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2637	# elif defined(RT_ARCH_ARM64)
2638	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2639	# endif
2640	#endif
2641
2642	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2643	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2644	#endif
2645
2646	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2647
2648	/** @} */
2649
2650	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2651

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 105855

Download in other formats: