IEMN8veRecompiler.h@ 106061

Last change on this file since 106061 was 106061, checked in by vboxsync, 5 months ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 131.4 KB

Line
1	/* $Id: IEMN8veRecompiler.h 106061 2024-09-16 14:03:52Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	/# define IEMLIVENESS_EXTENDED_LAYOUT/
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62	* Enables strict consistency checks around EFLAGS skipping.
63	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65	# ifdef VBOX_STRICT
66	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67	# endif
68	#elif defined(DOXYGEN_RUNNING)
69	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70	#endif
71
72	#ifdef VBOX_WITH_STATISTICS
73	/** Always count instructions for now. */
74	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75	#endif
76
77	/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78	* Enables having only a single prologue for native TBs. */
79	#if 1 \|\| defined(DOXYGEN_RUNNING)
80	# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81	#endif
82
83	/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84	* Enable this to use common epilogue and tail code for all TBs in a chunk. */
85	#if 1 \|\| defined(DOXYGEN_RUNNING)
86	# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87	#endif
88
89
90	/** @name Stack Frame Layout
91	*
92	* @{ */
93	/** The size of the area for stack variables and spills and stuff.
94	* @note This limit is duplicated in the python script(s). We add 0x40 for
95	* alignment padding. */
96	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101	#ifdef RT_ARCH_AMD64
102	/** An stack alignment adjustment (between non-volatile register pushes and
103	* the stack variable area, so the latter better aligned). */
104	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106	/** Number of stack arguments slots for calls made from the frame. */
107	# ifdef RT_OS_WINDOWS
108	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109	# else
110	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111	# endif
112	/** Number of any shadow arguments (spill area) for calls we make. */
113	# ifdef RT_OS_WINDOWS
114	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115	# else
116	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117	# endif
118
119	/** Frame pointer (RBP) relative offset of the last push. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122	# else
123	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124	# endif
125	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126	* address for it). */
127	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132	# ifdef RT_OS_WINDOWS
133	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137	# endif
138
139	# ifdef RT_OS_WINDOWS
140	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148	# endif
149
150	#elif RT_ARCH_ARM64
151	/** No alignment padding needed for arm64. */
152	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153	/** No stack argument slots, got 8 registers for arguments will suffice. */
154	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155	/** There are no argument spill area. */
156	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158	/** Number of saved registers at the top of our stack frame.
159	* This includes the return address and old frame pointer, so x19 thru x30. */
160	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164	/** Frame pointer (BP) relative offset of the last push. */
165	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168	* address for it). */
169	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171	#else
172	# error "port me"
173	#endif
174	/** @} */
175
176
177	/** @name Fixed Register Allocation(s)
178	* @{ */
179	/** @def IEMNATIVE_REG_FIXED_PVMCPU
180	* The number of the register holding the pVCpu pointer. */
181	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183	* @note This not available on AMD64, only ARM64. */
184	/** @def IEMNATIVE_REG_FIXED_TMP0
185	* Dedicated temporary register.
186	* @note This has extremely short lifetime, must be used with great care to make
187	* sure any calling code or code being called is making use of it.
188	* It will definitely not survive a call or anything of that nature.
189	* @todo replace this by a register allocator and content tracker. */
190	/** @def IEMNATIVE_REG_FIXED_MASK
191	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
192	* architecture. */
193	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
194	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
196	* architecture. */
197	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
198	* Dedicated temporary SIMD register. */
199	#endif
200	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
201	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
202	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
203	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
204	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
205	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
206	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
207	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
208	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
209	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
210	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
211	# else
212	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
213	# endif
214	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
215	\| RT_BIT_32(ARMV8_A64_REG_LR) \
216	\| RT_BIT_32(ARMV8_A64_REG_BP) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
218	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
219	\| RT_BIT_32(ARMV8_A64_REG_X18) \
220	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
221	\| IEMNATIVE_REG_FIXED_MASK_ADD)
222
223	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
224	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
225	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
227	# else
228	/** @note
229	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
230	* support emulating 256-bit registers we pair two real registers statically to
231	* one virtual for now, leaving us with only 16 256-bit registers. We always
232	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
233	* the register allocator assumes that it will be always free when the lower is
234	* picked.
235	*
236	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
237	* touch them in order to avoid having to save and restore them in the
238	* prologue/epilogue.
239	*/
240	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
252	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
253	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
255	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
256	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
257	\| RT_BIT_32(ARMV8_A64_REG_Q1))
258	# endif
259	# endif
260
261	#elif defined(RT_ARCH_AMD64)
262	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
263	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
264	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
265	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
266	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
267	\| RT_BIT_32(X86_GREG_xSP) \
268	\| RT_BIT_32(X86_GREG_xBP) )
269
270	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
271	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
272	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
273	# ifndef _MSC_VER
274	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275	# endif
276	# endif
277	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
278	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
279	# else
280	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
281	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
282	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
283	# endif
284	# endif
285
286	#else
287	# error "port me"
288	#endif
289	/** @} */
290
291	/** @name Call related registers.
292	* @{ */
293	/** @def IEMNATIVE_CALL_RET_GREG
294	* The return value register. */
295	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
296	* Number of arguments in registers. */
297	/** @def IEMNATIVE_CALL_ARG0_GREG
298	* The general purpose register carrying argument \#0. */
299	/** @def IEMNATIVE_CALL_ARG1_GREG
300	* The general purpose register carrying argument \#1. */
301	/** @def IEMNATIVE_CALL_ARG2_GREG
302	* The general purpose register carrying argument \#2. */
303	/** @def IEMNATIVE_CALL_ARG3_GREG
304	* The general purpose register carrying argument \#3. */
305	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
306	* Mask of registers the callee will not save and may trash. */
307	#ifdef RT_ARCH_AMD64
308	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
309
310	# ifdef RT_OS_WINDOWS
311	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
312	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
313	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
314	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
315	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
316	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
317	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
318	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
319	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
320	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
321	\| RT_BIT_32(X86_GREG_xCX) \
322	\| RT_BIT_32(X86_GREG_xDX) \
323	\| RT_BIT_32(X86_GREG_x8) \
324	\| RT_BIT_32(X86_GREG_x9) \
325	\| RT_BIT_32(X86_GREG_x10) \
326	\| RT_BIT_32(X86_GREG_x11) )
327	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
328	/* xmm0 - xmm5 are marked as volatile. */
329	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
330	# endif
331
332	# else /* !RT_OS_WINDOWS */
333	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
334	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
335	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
336	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
337	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
338	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
339	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
340	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
341	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
342	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
343	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
344	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
345	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
346	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
347	\| RT_BIT_32(X86_GREG_xCX) \
348	\| RT_BIT_32(X86_GREG_xDX) \
349	\| RT_BIT_32(X86_GREG_xDI) \
350	\| RT_BIT_32(X86_GREG_xSI) \
351	\| RT_BIT_32(X86_GREG_x8) \
352	\| RT_BIT_32(X86_GREG_x9) \
353	\| RT_BIT_32(X86_GREG_x10) \
354	\| RT_BIT_32(X86_GREG_x11) )
355	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
356	/* xmm0 - xmm15 are marked as volatile. */
357	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
358	# endif
359	# endif /* !RT_OS_WINDOWS */
360
361	#elif defined(RT_ARCH_ARM64)
362	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
363	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
364	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
365	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
366	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
367	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
368	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
369	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
370	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
371	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
372	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
373	\| RT_BIT_32(ARMV8_A64_REG_X1) \
374	\| RT_BIT_32(ARMV8_A64_REG_X2) \
375	\| RT_BIT_32(ARMV8_A64_REG_X3) \
376	\| RT_BIT_32(ARMV8_A64_REG_X4) \
377	\| RT_BIT_32(ARMV8_A64_REG_X5) \
378	\| RT_BIT_32(ARMV8_A64_REG_X6) \
379	\| RT_BIT_32(ARMV8_A64_REG_X7) )
380	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
381	\| RT_BIT_32(ARMV8_A64_REG_X1) \
382	\| RT_BIT_32(ARMV8_A64_REG_X2) \
383	\| RT_BIT_32(ARMV8_A64_REG_X3) \
384	\| RT_BIT_32(ARMV8_A64_REG_X4) \
385	\| RT_BIT_32(ARMV8_A64_REG_X5) \
386	\| RT_BIT_32(ARMV8_A64_REG_X6) \
387	\| RT_BIT_32(ARMV8_A64_REG_X7) \
388	\| RT_BIT_32(ARMV8_A64_REG_X8) \
389	\| RT_BIT_32(ARMV8_A64_REG_X9) \
390	\| RT_BIT_32(ARMV8_A64_REG_X10) \
391	\| RT_BIT_32(ARMV8_A64_REG_X11) \
392	\| RT_BIT_32(ARMV8_A64_REG_X12) \
393	\| RT_BIT_32(ARMV8_A64_REG_X13) \
394	\| RT_BIT_32(ARMV8_A64_REG_X14) \
395	\| RT_BIT_32(ARMV8_A64_REG_X15) \
396	\| RT_BIT_32(ARMV8_A64_REG_X16) \
397	\| RT_BIT_32(ARMV8_A64_REG_X17) )
398	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
399	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
400	* so to simplify our life a bit we just mark everything as volatile. */
401	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
402	# endif
403
404	#endif
405
406	/** This is the maximum argument count we'll ever be needing. */
407	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
408	#ifdef RT_OS_WINDOWS
409	# ifdef VBOXSTRICTRC_STRICT_ENABLED
410	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
411	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
412	# endif
413	#endif
414
415	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
416	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
417	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
418	#ifdef IEMNATIVE_REG_FIXED_TMP0
419	# ifdef IEMNATIVE_REG_FIXED_TMP1
420	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
421	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
422	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
423	# else
424	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
425	# endif
426	#else
427	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
428	#endif
429	/** @} */
430
431
432	/** @def IEMNATIVE_HST_GREG_COUNT
433	* Number of host general purpose registers we tracker. */
434	/** @def IEMNATIVE_HST_GREG_MASK
435	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
436	* inverted register masks and such to get down to a correct set of regs. */
437	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
438	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
439	* Number of host SIMD registers we track. */
440	/** @def IEMNATIVE_HST_SIMD_REG_MASK
441	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
442	* inverted register masks and such to get down to a correct set of regs. */
443	#endif
444	#ifdef RT_ARCH_AMD64
445	# define IEMNATIVE_HST_GREG_COUNT 16
446	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
447
448	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
449	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
450	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
451	# endif
452
453	#elif defined(RT_ARCH_ARM64)
454	# define IEMNATIVE_HST_GREG_COUNT 32
455	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
456
457	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
458	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
459	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
460	# endif
461
462	#else
463	# error "Port me!"
464	#endif
465
466
467	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
468
469
470	/** Native code generator label types. */
471	typedef enum
472	{
473	kIemNativeLabelType_Invalid = 0,
474	/*
475	* Labels w/o data, only once instance per TB - aka exit reasons.
476	*
477	* Note! Jumps to these requires instructions that are capable of spanning
478	* the max TB length.
479	*/
480	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
481	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
482	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
483	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
484	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
485	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
486	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
487	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
488	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
489	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
490	kIemNativeLabelType_ObsoleteTb,
491	kIemNativeLabelType_NeedCsLimChecking,
492	kIemNativeLabelType_CheckBranchMiss,
493	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
494	/* Manually defined labels. */
495	kIemNativeLabelType_ReturnBreak,
496	kIemNativeLabelType_ReturnBreakFF,
497	kIemNativeLabelType_ReturnBreakViaLookup,
498	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
499	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
500	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
501	kIemNativeLabelType_ReturnWithFlags,
502	kIemNativeLabelType_NonZeroRetOrPassUp,
503	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
504	kIemNativeLabelType_ReturnSuccess, /*< Sets eax/w0 to zero and returns. /
505	#else
506	kIemNativeLabelType_Return,
507	#endif
508	/** The last fixup for branches that can span almost the whole TB length.
509	* @note Whether kIemNativeLabelType_Return needs to be one of these is
510	* a bit questionable, since nobody jumps to it except other tail code. */
511	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
512	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
513	#else
514	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
515	#endif
516	/** The last fixup for branches that exits the TB. */
517	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
518	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_ReturnSuccess,
519	#else
520	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
521	#endif
522
523	/** Loop-jump target. */
524	kIemNativeLabelType_LoopJumpTarget,
525
526	/*
527	* Labels with data, potentially multiple instances per TB:
528	*
529	* These are localized labels, so no fixed jump type restrictions here.
530	*/
531	kIemNativeLabelType_FirstWithMultipleInstances,
532	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
533	kIemNativeLabelType_Else,
534	kIemNativeLabelType_Endif,
535	kIemNativeLabelType_CheckIrq,
536	kIemNativeLabelType_TlbLookup,
537	kIemNativeLabelType_TlbMiss,
538	kIemNativeLabelType_TlbDone,
539	kIemNativeLabelType_End
540	} IEMNATIVELABELTYPE;
541
542	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
543	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
544
545
546	/** Native code generator label definition. */
547	typedef struct IEMNATIVELABEL
548	{
549	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
550	* the epilog. */
551	uint32_t off;
552	/** The type of label (IEMNATIVELABELTYPE). */
553	uint16_t enmType;
554	/** Additional label data, type specific. */
555	uint16_t uData;
556	} IEMNATIVELABEL;
557	/** Pointer to a label. */
558	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
559
560
561	/** Native code generator fixup types. */
562	typedef enum
563	{
564	kIemNativeFixupType_Invalid = 0,
565	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
566	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
567	kIemNativeFixupType_Rel32,
568	#elif defined(RT_ARCH_ARM64)
569	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
570	kIemNativeFixupType_RelImm26At0,
571	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
572	kIemNativeFixupType_RelImm19At5,
573	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
574	kIemNativeFixupType_RelImm14At5,
575	#endif
576	kIemNativeFixupType_End
577	} IEMNATIVEFIXUPTYPE;
578
579	/** Native code generator fixup. */
580	typedef struct IEMNATIVEFIXUP
581	{
582	/** Code offset of the fixup location. */
583	uint32_t off;
584	/** The IEMNATIVELABEL this is a fixup for. */
585	uint16_t idxLabel;
586	/** The fixup type (IEMNATIVEFIXUPTYPE). */
587	uint8_t enmType;
588	/** Addend or other data. */
589	int8_t offAddend;
590	} IEMNATIVEFIXUP;
591	/** Pointer to a native code generator fixup. */
592	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
593
594	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
595
596	/** Native code generator fixup to per chunk TB tail code. */
597	typedef struct IEMNATIVEEXITFIXUP
598	{
599	/** Code offset of the fixup location. */
600	uint32_t off;
601	/** The exit reason. */
602	IEMNATIVELABELTYPE enmExitReason;
603	} IEMNATIVEEXITFIXUP;
604	/** Pointer to a native code generator TB exit fixup. */
605	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
606
607	/**
608	* Per executable memory chunk context with addresses for common code.
609	*/
610	typedef struct IEMNATIVEPERCHUNKCTX
611	{
612	/** Pointers to the exit labels */
613	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
614	} IEMNATIVEPERCHUNKCTX;
615	/** Pointer to per-chunk recompiler context. */
616	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
617	/** Pointer to const per-chunk recompiler context. */
618	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
619
620	#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
621
622
623	/**
624	* One bit of the state.
625	*
626	* Each register state takes up two bits. We keep the two bits in two separate
627	* 64-bit words to simplify applying them to the guest shadow register mask in
628	* the register allocator.
629	*/
630	typedef union IEMLIVENESSBIT
631	{
632	uint64_t bm64;
633	RT_GCC_EXTENSION struct
634	{ /* bit no */
635	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
636	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
637	uint64_t fCr0 : 1; /*< 0x11 / 17: /
638	uint64_t fFcw : 1; /*< 0x12 / 18: /
639	uint64_t fFsw : 1; /*< 0x13 / 19: /
640	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
641	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
642	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
643	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
644	uint64_t fCr4 : 1; /*< 0x2c / 44: /
645	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
646	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
647	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
648	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
649	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
650	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
651	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
652	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
653	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
654	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
655	};
656	} IEMLIVENESSBIT;
657	AssertCompileSize(IEMLIVENESSBIT, 8);
658
659	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
660	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
661	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
662	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
663	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
664	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
665	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
666
667
668	/**
669	* A liveness state entry.
670	*
671	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
672	* Once we add a SSE register shadowing, we'll add another 64-bit element for
673	* that.
674	*/
675	typedef union IEMLIVENESSENTRY
676	{
677	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
678	uint64_t bm64[16 / 8];
679	uint16_t bm32[16 / 4];
680	uint16_t bm16[16 / 2];
681	uint8_t bm8[ 16 / 1];
682	IEMLIVENESSBIT aBits[2];
683	#else
684	uint64_t bm64[32 / 8];
685	uint16_t bm32[32 / 4];
686	uint16_t bm16[32 / 2];
687	uint8_t bm8[ 32 / 1];
688	IEMLIVENESSBIT aBits[4];
689	#endif
690	RT_GCC_EXTENSION struct
691	{
692	/** Bit \#0 of the register states. */
693	IEMLIVENESSBIT Bit0;
694	/** Bit \#1 of the register states. */
695	IEMLIVENESSBIT Bit1;
696	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
697	/** Bit \#2 of the register states. */
698	IEMLIVENESSBIT Bit2;
699	/** Bit \#3 of the register states. */
700	IEMLIVENESSBIT Bit3;
701	#endif
702	};
703	} IEMLIVENESSENTRY;
704	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
705	AssertCompileSize(IEMLIVENESSENTRY, 16);
706	#else
707	AssertCompileSize(IEMLIVENESSENTRY, 32);
708	#endif
709	/** Pointer to a liveness state entry. */
710	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
711	/** Pointer to a const liveness state entry. */
712	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
713
714	/** @name 64-bit value masks for IEMLIVENESSENTRY.
715	* @{ / / 0xzzzzyyyyxxxxwwww */
716	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
717
718	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
719	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
720	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
721
722	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
723	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
724	#endif
725
726	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
727	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
728
729	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
730	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
731	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
732	#endif
733	/** @} */
734
735
736	/** @name The liveness state for a register.
737	*
738	* The state values have been picked to with state accumulation in mind (what
739	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
740	* performance critical work done with the values.
741	*
742	* This is a compressed state that only requires 2 bits per register.
743	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
744	* 1. the incoming state from the following call,
745	* 2. the outgoing state for this call,
746	* 3. mask of the entries set in the 2nd.
747	*
748	* The mask entry (3rd one above) will be used both when updating the outgoing
749	* state and when merging in incoming state for registers not touched by the
750	* current call.
751	*
752	* @{ */
753	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
754	/** The register will be clobbered and the current value thrown away.
755	*
756	* When this is applied to the state (2) we'll simply be AND'ing it with the
757	* (old) mask (3) and adding the register to the mask. This way we'll
758	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
759	* IEMLIVENESS_STATE_INPUT states. */
760	# define IEMLIVENESS_STATE_CLOBBERED 0
761	/** The register is unused in the remainder of the TB.
762	*
763	* This is an initial state and can not be set by any of the
764	* iemNativeLivenessFunc_xxxx callbacks. */
765	# define IEMLIVENESS_STATE_UNUSED 1
766	/** The register value is required in a potential call or exception.
767	*
768	* This means that the register value must be calculated and is best written to
769	* the state, but that any shadowing registers can be flushed thereafter as it's
770	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
771	*
772	* It is typically applied across the board, but we preserve incoming
773	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
774	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
775	* 1. r0 = old & ~mask;
776	* 2. r0 = t1 & (t1 >> 1)'
777	* 3. state \|= r0 \| 0b10;
778	* 4. mask = ~0;
779	*/
780	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
781	/** The register value is used as input.
782	*
783	* This means that the register value must be calculated and it is best to keep
784	* it in a register. It does not need to be writtent out as such. This is the
785	* highest priority state.
786	*
787	* Whether the call modifies the register or not isn't relevant to earlier
788	* calls, so that's not recorded.
789	*
790	* When applying this state we just or in the value in the outgoing state and
791	* mask. */
792	# define IEMLIVENESS_STATE_INPUT 3
793	/** Mask of the state bits. */
794	# define IEMLIVENESS_STATE_MASK 3
795	/** The number of bits per state. */
796	# define IEMLIVENESS_STATE_BIT_COUNT 2
797	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
798	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
799	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
800	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
801	/** Check if a register clobbering is expected given the (previous) liveness state.
802	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
803	* include INPUT if the register is used in more than one place. */
804	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
805
806	/** Check if all status flags are going to be clobbered and doesn't need
807	* calculating in the current step.
808	* @param a_pCurEntry The current liveness entry. */
809	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
810	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
811
812	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
813	/** The register is not used any more. */
814	# define IEMLIVENESS_STATE_UNUSED 0
815	/** Flag: The register is required in a potential exception or call. */
816	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
817	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
818	/** Flag: The register is read. */
819	# define IEMLIVENESS_STATE_READ 2
820	# define IEMLIVENESS_BIT_READ 1
821	/** Flag: The register is written. */
822	# define IEMLIVENESS_STATE_WRITE 4
823	# define IEMLIVENESS_BIT_WRITE 2
824	/** Flag: Unconditional call (not needed, can be redefined for research). */
825	# define IEMLIVENESS_STATE_CALL 8
826	# define IEMLIVENESS_BIT_CALL 3
827	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
828	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
829	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
830	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
831	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
832
833	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
834	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
835	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
836	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
837
838	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
839	/** @} */
840
841	/** @name Liveness helpers for builtin functions and similar.
842	*
843	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
844	* own set of manimulator macros for those.
845	*
846	* @{ */
847	/** Initializing the state as all unused. */
848	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
849	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
850	do { \
851	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
852	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
853	} while (0)
854	#else
855	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
856	do { \
857	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
858	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
859	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
860	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
861	} while (0)
862	#endif
863
864	/** Initializing the outgoing state with a potential xcpt or call state.
865	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
866	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
867	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
868	do { \
869	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
870	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
871	} while (0)
872	#else
873	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
874	do { \
875	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
876	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
877	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
878	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
879	} while (0)
880	#endif
881
882	/** Adds a segment base register as input to the outgoing state. */
883	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
884	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
885	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
886	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
887	} while (0)
888	#else
889	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
890	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
891	} while (0)
892	#endif
893
894	/** Adds a segment attribute register as input to the outgoing state. */
895	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
896	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
897	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
898	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
899	} while (0)
900	#else
901	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
902	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
903	} while (0)
904	#endif
905
906	/** Adds a segment limit register as input to the outgoing state. */
907	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
908	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
909	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
910	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
911	} while (0)
912	#else
913	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
914	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
915	} while (0)
916	#endif
917
918	/** Adds a segment limit register as input to the outgoing state. */
919	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
920	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
921	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
922	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
923	} while (0)
924	#else
925	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
926	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
927	} while (0)
928	#endif
929	/** @} */
930
931	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
932	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
933	* calculated and up to date. This is to double check that we haven't skipped
934	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
935	* @note has to be placed in
936	*/
937	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
938	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
939	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
940	#else
941	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
942	#endif
943
944
945	/**
946	* Guest registers that can be shadowed in GPRs.
947	*
948	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
949	* must be placed last, as the liveness state tracks it as 7 subcomponents and
950	* we don't want to waste space here.
951	*
952	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
953	* friends as well as IEMAllN8veLiveness.cpp.
954	*/
955	typedef enum IEMNATIVEGSTREG : uint8_t
956	{
957	kIemNativeGstReg_GprFirst = 0,
958	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
959	kIemNativeGstReg_Pc,
960	kIemNativeGstReg_Cr0,
961	kIemNativeGstReg_FpuFcw,
962	kIemNativeGstReg_FpuFsw,
963	kIemNativeGstReg_SegBaseFirst,
964	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
965	kIemNativeGstReg_SegAttribFirst,
966	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
967	kIemNativeGstReg_SegLimitFirst,
968	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
969	kIemNativeGstReg_SegSelFirst,
970	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
971	kIemNativeGstReg_Cr4,
972	kIemNativeGstReg_Xcr0,
973	kIemNativeGstReg_MxCsr,
974	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
975	kIemNativeGstReg_End
976	} IEMNATIVEGSTREG;
977	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
978	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
979
980	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
981	* @{ */
982	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
983	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
984	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
985	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
986	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
987	/** @} */
988
989	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
990
991	/**
992	* Guest registers that can be shadowed in host SIMD registers.
993	*
994	* @todo r=aeichner Liveness tracking
995	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
996	*/
997	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
998	{
999	kIemNativeGstSimdReg_SimdRegFirst = 0,
1000	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1001	kIemNativeGstSimdReg_End
1002	} IEMNATIVEGSTSIMDREG;
1003
1004	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1005	* @{ */
1006	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1007	/** @} */
1008
1009	/**
1010	* The Load/store size for a SIMD guest register.
1011	*/
1012	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1013	{
1014	/** Invalid size. */
1015	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1016	/** Loads the low 128-bit of a guest SIMD register. */
1017	kIemNativeGstSimdRegLdStSz_Low128,
1018	/** Loads the high 128-bit of a guest SIMD register. */
1019	kIemNativeGstSimdRegLdStSz_High128,
1020	/** Loads the whole 256-bits of a guest SIMD register. */
1021	kIemNativeGstSimdRegLdStSz_256,
1022	/** End value. */
1023	kIemNativeGstSimdRegLdStSz_End
1024	} IEMNATIVEGSTSIMDREGLDSTSZ;
1025
1026	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1027
1028	/**
1029	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1030	*/
1031	typedef enum IEMNATIVEGSTREGUSE
1032	{
1033	/** The usage is read-only, the register holding the guest register
1034	* shadow copy will not be modified by the caller. */
1035	kIemNativeGstRegUse_ReadOnly = 0,
1036	/** The caller will update the guest register (think: PC += cbInstr).
1037	* The guest shadow copy will follow the returned register. */
1038	kIemNativeGstRegUse_ForUpdate,
1039	/** The call will put an entirely new value in the guest register, so
1040	* if new register is allocate it will be returned uninitialized. */
1041	kIemNativeGstRegUse_ForFullWrite,
1042	/** The caller will use the guest register value as input in a calculation
1043	* and the host register will be modified.
1044	* This means that the returned host register will not be marked as a shadow
1045	* copy of the guest register. */
1046	kIemNativeGstRegUse_Calculation
1047	} IEMNATIVEGSTREGUSE;
1048
1049	/**
1050	* Guest registers (classes) that can be referenced.
1051	*/
1052	typedef enum IEMNATIVEGSTREGREF : uint8_t
1053	{
1054	kIemNativeGstRegRef_Invalid = 0,
1055	kIemNativeGstRegRef_Gpr,
1056	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1057	kIemNativeGstRegRef_EFlags,
1058	kIemNativeGstRegRef_MxCsr,
1059	kIemNativeGstRegRef_FpuReg,
1060	kIemNativeGstRegRef_MReg,
1061	kIemNativeGstRegRef_XReg,
1062	kIemNativeGstRegRef_X87,
1063	kIemNativeGstRegRef_XState,
1064	//kIemNativeGstRegRef_YReg, - doesn't work.
1065	kIemNativeGstRegRef_End
1066	} IEMNATIVEGSTREGREF;
1067
1068
1069	/** Variable kinds. */
1070	typedef enum IEMNATIVEVARKIND : uint8_t
1071	{
1072	/** Customary invalid zero value. */
1073	kIemNativeVarKind_Invalid = 0,
1074	/** This is either in a register or on the stack. */
1075	kIemNativeVarKind_Stack,
1076	/** Immediate value - loaded into register when needed, or can live on the
1077	* stack if referenced (in theory). */
1078	kIemNativeVarKind_Immediate,
1079	/** Variable reference - loaded into register when needed, never stack. */
1080	kIemNativeVarKind_VarRef,
1081	/** Guest register reference - loaded into register when needed, never stack. */
1082	kIemNativeVarKind_GstRegRef,
1083	/** End of valid values. */
1084	kIemNativeVarKind_End
1085	} IEMNATIVEVARKIND;
1086
1087
1088	/** Variable or argument. */
1089	typedef struct IEMNATIVEVAR
1090	{
1091	/** The kind of variable. */
1092	IEMNATIVEVARKIND enmKind;
1093	/** The variable size in bytes. */
1094	uint8_t cbVar;
1095	/** The first stack slot (uint64_t), except for immediate and references
1096	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1097	* has a stack slot it has been initialized and has a value. Unused variables
1098	* has neither a stack slot nor a host register assignment. */
1099	uint8_t idxStackSlot;
1100	/** The host register allocated for the variable, UINT8_MAX if not. */
1101	uint8_t idxReg;
1102	/** The argument number if argument, UINT8_MAX if regular variable. */
1103	uint8_t uArgNo;
1104	/** If referenced, the index (unpacked) of the variable referencing this one,
1105	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1106	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1107	uint8_t idxReferrerVar;
1108	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1109	* @todo not sure what this really is for... */
1110	IEMNATIVEGSTREG enmGstReg;
1111	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1112	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1113	* only valid when idxReg is not UINT8_MAX. */
1114	bool fSimdReg : 1;
1115	/** Set if the registered is currently used exclusively, false if the
1116	* variable is idle and the register can be grabbed. */
1117	bool fRegAcquired : 1;
1118	#else
1119	/** Set if the registered is currently used exclusively, false if the
1120	* variable is idle and the register can be grabbed. */
1121	bool fRegAcquired;
1122	#endif
1123
1124	union
1125	{
1126	/** kIemNativeVarKind_Immediate: The immediate value. */
1127	uint64_t uValue;
1128	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1129	uint8_t idxRefVar;
1130	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1131	struct
1132	{
1133	/** The class of register. */
1134	IEMNATIVEGSTREGREF enmClass;
1135	/** Index within the class. */
1136	uint8_t idx;
1137	} GstRegRef;
1138	} u;
1139	} IEMNATIVEVAR;
1140	/** Pointer to a variable or argument. */
1141	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1142	/** Pointer to a const variable or argument. */
1143	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1144
1145	/** What is being kept in a host register. */
1146	typedef enum IEMNATIVEWHAT : uint8_t
1147	{
1148	/** The traditional invalid zero value. */
1149	kIemNativeWhat_Invalid = 0,
1150	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1151	kIemNativeWhat_Var,
1152	/** Temporary register, this is typically freed when a MC completes. */
1153	kIemNativeWhat_Tmp,
1154	/** Call argument w/o a variable mapping. This is free (via
1155	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1156	kIemNativeWhat_Arg,
1157	/** Return status code.
1158	* @todo not sure if we need this... */
1159	kIemNativeWhat_rc,
1160	/** The fixed pVCpu (PVMCPUCC) register.
1161	* @todo consider offsetting this on amd64 to use negative offsets to access
1162	* more members using 8-byte disp. */
1163	kIemNativeWhat_pVCpuFixed,
1164	/** The fixed pCtx (PCPUMCTX) register.
1165	* @todo consider offsetting this on amd64 to use negative offsets to access
1166	* more members using 8-byte disp. */
1167	kIemNativeWhat_pCtxFixed,
1168	/** Fixed temporary register. */
1169	kIemNativeWhat_FixedTmp,
1170	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1171	/** Shadow RIP for the delayed RIP updating debugging. */
1172	kIemNativeWhat_PcShadow,
1173	#endif
1174	/** Register reserved by the CPU or OS architecture. */
1175	kIemNativeWhat_FixedReserved,
1176	/** End of valid values. */
1177	kIemNativeWhat_End
1178	} IEMNATIVEWHAT;
1179
1180	/**
1181	* Host general register entry.
1182	*
1183	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1184	*
1185	* @todo Track immediate values in host registers similarlly to how we track the
1186	* guest register shadow copies. For it to be real helpful, though,
1187	* we probably need to know which will be reused and put them into
1188	* non-volatile registers, otherwise it's going to be more or less
1189	* restricted to an instruction or two.
1190	*/
1191	typedef struct IEMNATIVEHSTREG
1192	{
1193	/** Set of guest registers this one shadows.
1194	*
1195	* Using a bitmap here so we can designate the same host register as a copy
1196	* for more than one guest register. This is expected to be useful in
1197	* situations where one value is copied to several registers in a sequence.
1198	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1199	* sequence we'd want to let this register follow to be a copy of and there
1200	* will always be places where we'd be picking the wrong one.
1201	*/
1202	uint64_t fGstRegShadows;
1203	/** What is being kept in this register. */
1204	IEMNATIVEWHAT enmWhat;
1205	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1206	uint8_t idxVar;
1207	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1208	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1209	* that scope. */
1210	uint8_t idxStackSlot;
1211	/** Alignment padding. */
1212	uint8_t abAlign[5];
1213	} IEMNATIVEHSTREG;
1214
1215
1216	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1217	/**
1218	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1219	* halves, on architectures where there is no 256-bit register available this entry will track
1220	* two adjacent 128-bit host registers.
1221	*
1222	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1223	*/
1224	typedef struct IEMNATIVEHSTSIMDREG
1225	{
1226	/** Set of guest registers this one shadows.
1227	*
1228	* Using a bitmap here so we can designate the same host register as a copy
1229	* for more than one guest register. This is expected to be useful in
1230	* situations where one value is copied to several registers in a sequence.
1231	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1232	* sequence we'd want to let this register follow to be a copy of and there
1233	* will always be places where we'd be picking the wrong one.
1234	*/
1235	uint64_t fGstRegShadows;
1236	/** What is being kept in this register. */
1237	IEMNATIVEWHAT enmWhat;
1238	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1239	uint8_t idxVar;
1240	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1241	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1242	/** Alignment padding. */
1243	uint8_t abAlign[5];
1244	} IEMNATIVEHSTSIMDREG;
1245	#endif
1246
1247
1248	/**
1249	* Core state for the native recompiler, that is, things that needs careful
1250	* handling when dealing with branches.
1251	*/
1252	typedef struct IEMNATIVECORESTATE
1253	{
1254	/** Allocation bitmap for aHstRegs. */
1255	uint32_t bmHstRegs;
1256
1257	/** Bitmap marking which host register contains guest register shadow copies.
1258	* This is used during register allocation to try preserve copies. */
1259	uint32_t bmHstRegsWithGstShadow;
1260	/** Bitmap marking valid entries in aidxGstRegShadows. */
1261	uint64_t bmGstRegShadows;
1262	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1263	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1264	uint64_t bmGstRegShadowDirty;
1265	#endif
1266
1267	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1268	/** The current instruction offset in bytes from when the guest program counter
1269	* was updated last. Used for delaying the write to the guest context program counter
1270	* as long as possible. */
1271	int64_t offPc;
1272	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1273	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1274	bool fDebugPcInitialized;
1275	# endif
1276	#endif
1277
1278	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1279	/** Allocation bitmap for aHstSimdRegs. */
1280	uint32_t bmHstSimdRegs;
1281
1282	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1283	* This is used during register allocation to try preserve copies. */
1284	uint32_t bmHstSimdRegsWithGstShadow;
1285	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1286	uint64_t bmGstSimdRegShadows;
1287	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1288	uint64_t bmGstSimdRegShadowDirtyLo128;
1289	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1290	uint64_t bmGstSimdRegShadowDirtyHi128;
1291	#endif
1292
1293	union
1294	{
1295	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1296	uint8_t aidxArgVars[8];
1297	/** For more efficient resetting. */
1298	uint64_t u64ArgVars;
1299	};
1300
1301	/** Allocation bitmap for the stack. */
1302	uint32_t bmStack;
1303	/** Allocation bitmap for aVars. */
1304	uint32_t bmVars;
1305
1306	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1307	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1308	* (A shadow copy of a guest register can only be held in a one host register,
1309	* there are no duplicate copies or ambiguities like that). */
1310	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1311	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1312	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1313	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1314	* (A shadow copy of a guest register can only be held in a one host register,
1315	* there are no duplicate copies or ambiguities like that). */
1316	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1317	#endif
1318
1319	/** Host register allocation tracking. */
1320	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1321	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1322	/** Host SIMD register allocation tracking. */
1323	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1324	#endif
1325
1326	/** Variables and arguments. */
1327	IEMNATIVEVAR aVars[9];
1328	} IEMNATIVECORESTATE;
1329	/** Pointer to core state. */
1330	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1331	/** Pointer to const core state. */
1332	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1333
1334	/** @def IEMNATIVE_VAR_IDX_UNPACK
1335	* @returns Index into IEMNATIVECORESTATE::aVars.
1336	* @param a_idxVar Variable index w/ magic (in strict builds).
1337	*/
1338	/** @def IEMNATIVE_VAR_IDX_PACK
1339	* @returns Variable index w/ magic (in strict builds).
1340	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1341	*/
1342	#ifdef VBOX_STRICT
1343	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1344	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1345	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1346	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1347	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1348	#else
1349	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1350	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1351	#endif
1352
1353
1354	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1355	/** Clear the dirty state of the given guest SIMD register. */
1356	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1357	do { \
1358	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1359	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1360	} while (0)
1361
1362	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1363	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1364	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1365	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1366	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1367	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1368	/** Returns whether the given guest SIMD register is dirty. */
1369	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1370	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1371
1372	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1373	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1374	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1375	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1376	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1377	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1378
1379	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1380	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1381	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1382	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1383	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1384	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1385	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1386	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1387	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1388	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1389	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1390	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1391	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1392	# endif
1393	#endif
1394
1395
1396	/**
1397	* Conditional stack entry.
1398	*/
1399	typedef struct IEMNATIVECOND
1400	{
1401	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1402	bool fInElse;
1403	union
1404	{
1405	RT_GCC_EXTENSION struct
1406	{
1407	/** Set if the if-block unconditionally exited the TB. */
1408	bool fIfExitTb;
1409	/** Set if the else-block unconditionally exited the TB. */
1410	bool fElseExitTb;
1411	};
1412	/** Indexed by fInElse. */
1413	bool afExitTb[2];
1414	};
1415	bool afPadding[5];
1416	/** The label for the IEM_MC_ELSE. */
1417	uint32_t idxLabelElse;
1418	/** The label for the IEM_MC_ENDIF. */
1419	uint32_t idxLabelEndIf;
1420	/** The initial state snapshot as the if-block starts executing. */
1421	IEMNATIVECORESTATE InitialState;
1422	/** The state snapshot at the end of the if-block. */
1423	IEMNATIVECORESTATE IfFinalState;
1424	} IEMNATIVECOND;
1425	/** Pointer to a condition stack entry. */
1426	typedef IEMNATIVECOND *PIEMNATIVECOND;
1427
1428
1429	/**
1430	* Native recompiler state.
1431	*/
1432	typedef struct IEMRECOMPILERSTATE
1433	{
1434	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1435	* IEMNATIVEINSTR units. */
1436	uint32_t cInstrBufAlloc;
1437	#ifdef VBOX_STRICT
1438	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1439	uint32_t offInstrBufChecked;
1440	#else
1441	uint32_t uPadding1; /* We don't keep track of the size here... */
1442	#endif
1443	/** Fixed temporary code buffer for native recompilation. */
1444	PIEMNATIVEINSTR pInstrBuf;
1445
1446	/** Bitmaps with the label types used. */
1447	uint64_t bmLabelTypes;
1448	/** Actual number of labels in paLabels. */
1449	uint32_t cLabels;
1450	/** Max number of entries allowed in paLabels before reallocating it. */
1451	uint32_t cLabelsAlloc;
1452	/** Labels defined while recompiling (referenced by fixups). */
1453	PIEMNATIVELABEL paLabels;
1454	/** Array with indexes of unique labels (uData always 0). */
1455	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1456
1457	/** Actual number of fixups paFixups. */
1458	uint32_t cFixups;
1459	/** Max number of entries allowed in paFixups before reallocating it. */
1460	uint32_t cFixupsAlloc;
1461	/** Buffer used by the recompiler for recording fixups when generating code. */
1462	PIEMNATIVEFIXUP paFixups;
1463
1464	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1465	/** Actual number of fixups in paTbExitFixups. */
1466	uint32_t cTbExitFixups;
1467	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1468	uint32_t cTbExitFixupsAlloc;
1469	/** Buffer used by the recompiler for recording fixups when generating code. */
1470	PIEMNATIVEEXITFIXUP paTbExitFixups;
1471	#endif
1472
1473	#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1474	/** Statistics: The idxInstr+1 value at the last PC update. */
1475	uint8_t idxInstrPlusOneOfLastPcUpdate;
1476	#endif
1477
1478	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1479	/** Number of debug info entries allocated for pDbgInfo. */
1480	uint32_t cDbgInfoAlloc;
1481	/** Debug info. */
1482	PIEMTBDBG pDbgInfo;
1483	#endif
1484
1485	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1486	/** The current call index (liveness array and threaded calls in TB). */
1487	uint32_t idxCurCall;
1488	/** Number of liveness entries allocated. */
1489	uint32_t cLivenessEntriesAlloc;
1490	/** Liveness entries for all the calls in the TB begin recompiled.
1491	* The entry for idxCurCall contains the info for what the next call will
1492	* require wrt registers. (Which means the last entry is the initial liveness
1493	* state.) */
1494	PIEMLIVENESSENTRY paLivenessEntries;
1495	#endif
1496
1497	/** The translation block being recompiled. */
1498	PCIEMTB pTbOrg;
1499	/** The VMCPU structure of the EMT. */
1500	PVMCPUCC pVCpu;
1501
1502	/** Condition sequence number (for generating unique labels). */
1503	uint16_t uCondSeqNo;
1504	/** Check IRQ sequence number (for generating unique labels). */
1505	uint16_t uCheckIrqSeqNo;
1506	/** TLB load sequence number (for generating unique labels). */
1507	uint16_t uTlbSeqNo;
1508	/** The current condition stack depth (aCondStack). */
1509	uint8_t cCondDepth;
1510
1511	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1512	uint8_t cArgsX;
1513	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1514	uint32_t fCImpl;
1515	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1516	uint32_t fMc;
1517	/** The expected IEMCPU::fExec value for the current call/instruction. */
1518	uint32_t fExec;
1519	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1520	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1521	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1522	*
1523	* This is an optimization because these control registers can only be changed from
1524	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1525	* consisting of multiple SIMD instructions.
1526	*/
1527	uint32_t fSimdRaiseXcptChecksEmitted;
1528	#endif
1529	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1530	uint32_t idxLastCheckIrqCallNo;
1531
1532	/** Core state requiring care with branches. */
1533	IEMNATIVECORESTATE Core;
1534
1535	/** The condition nesting stack. */
1536	IEMNATIVECOND aCondStack[2];
1537
1538	#ifndef IEM_WITH_THROW_CATCH
1539	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1540	* for recompilation error handling. */
1541	jmp_buf JmpBuf;
1542	#endif
1543	} IEMRECOMPILERSTATE;
1544	/** Pointer to a native recompiler state. */
1545	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1546
1547
1548	/** @def IEMNATIVE_TRY_SETJMP
1549	* Wrapper around setjmp / try, hiding all the ugly differences.
1550	*
1551	* @note Use with extreme care as this is a fragile macro.
1552	* @param a_pReNative The native recompile state.
1553	* @param a_rcTarget The variable that should receive the status code in case
1554	* of a longjmp/throw.
1555	*/
1556	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1557	* Start wrapper for catch / setjmp-else.
1558	*
1559	* This will set up a scope.
1560	*
1561	* @note Use with extreme care as this is a fragile macro.
1562	* @param a_pReNative The native recompile state.
1563	* @param a_rcTarget The variable that should receive the status code in case
1564	* of a longjmp/throw.
1565	*/
1566	/** @def IEMNATIVE_CATCH_LONGJMP_END
1567	* End wrapper for catch / setjmp-else.
1568	*
1569	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1570	* up the state.
1571	*
1572	* @note Use with extreme care as this is a fragile macro.
1573	* @param a_pReNative The native recompile state.
1574	*/
1575	/** @def IEMNATIVE_DO_LONGJMP
1576	*
1577	* Wrapper around longjmp / throw.
1578	*
1579	* @param a_pReNative The native recompile state.
1580	* @param a_rc The status code jump back with / throw.
1581	*/
1582	#ifdef IEM_WITH_THROW_CATCH
1583	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1584	a_rcTarget = VINF_SUCCESS; \
1585	try
1586	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1587	catch (int rcThrown) \
1588	{ \
1589	a_rcTarget = rcThrown
1590	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1591	} \
1592	((void)0)
1593	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1594	#else /* !IEM_WITH_THROW_CATCH */
1595	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1596	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1597	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1598	else \
1599	{ \
1600	((void)0)
1601	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1602	}
1603	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1604	#endif /* !IEM_WITH_THROW_CATCH */
1605
1606
1607	/**
1608	* Native recompiler worker for a threaded function.
1609	*
1610	* @returns New code buffer offset; throws VBox status code in case of a failure.
1611	* @param pReNative The native recompiler state.
1612	* @param off The current code buffer offset.
1613	* @param pCallEntry The threaded call entry.
1614	*
1615	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1616	*/
1617	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1618	/** Pointer to a native recompiler worker for a threaded function. */
1619	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1620
1621	/** Defines a native recompiler worker for a threaded function.
1622	* @see FNIEMNATIVERECOMPFUNC */
1623	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1624	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1625
1626	/** Prototypes a native recompiler function for a threaded function.
1627	* @see FNIEMNATIVERECOMPFUNC */
1628	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1629
1630
1631	/**
1632	* Native recompiler liveness analysis worker for a threaded function.
1633	*
1634	* @param pCallEntry The threaded call entry.
1635	* @param pIncoming The incoming liveness state entry.
1636	* @param pOutgoing The outgoing liveness state entry.
1637	*/
1638	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1639	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1640	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1641	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1642
1643	/** Defines a native recompiler liveness analysis worker for a threaded function.
1644	* @see FNIEMNATIVELIVENESSFUNC */
1645	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1646	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1647
1648	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1649	* @see FNIEMNATIVELIVENESSFUNC */
1650	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1651
1652
1653	/** Define a native recompiler helper function, safe to call from the TB code. */
1654	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1655	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1656	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1657	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1658	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1659	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1660	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1661	a_RetType (VBOXCALL *a_Name) a_ArgList
1662
1663
1664	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1665	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1666	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1667	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1668	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1669	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1670	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1671	uint8_t idxHstSimdReg = UINT8_MAX,
1672	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1673	# endif
1674	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1675	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1676	uint8_t idxGstReg, uint8_t idxHstReg);
1677	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1678	uint64_t fGstReg);
1679	# endif
1680	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1681	uint64_t offPc, uint32_t cInstrSkipped);
1682	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1683
1684	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1685	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1686	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1687	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1688	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1689	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1690	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1691	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1692	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1693	#endif
1694	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1695
1696	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1697	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1698	bool fPreferVolatile = true);
1699	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1700	bool fPreferVolatile = true);
1701	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1702	IEMNATIVEGSTREG enmGstReg,
1703	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1704	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1705	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1706	IEMNATIVEGSTREG enmGstReg);
1707
1708	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1709	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1710	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1711	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1712	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1713	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1714	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1715	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1716	# endif
1717	#endif
1718	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1719	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1720	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1721	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1722	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1723	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1724	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1725	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1726	# endif
1727	#endif
1728	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1729	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1730	uint32_t fKeepVars = 0);
1731	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1732	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1733	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1734	uint32_t fHstRegsActiveShadows);
1735	#ifdef VBOX_STRICT
1736	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1737	#endif
1738	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1739	uint64_t fGstSimdShwExcept);
1740	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1741	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1742	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1743	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
1744	# endif
1745	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1746	#endif
1747	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1748	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1749	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1750	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
1751	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1752	uint64_t fFlushGstReg = UINT64_MAX);
1753	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
1754	uint32_t off, uint8_t idxHstReg);
1755	#endif
1756
1757
1758	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1759	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1760	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1761	bool fPreferVolatile = true);
1762	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1763	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1764	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1765	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1766	bool fNoVolatileRegs = false);
1767	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1768	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1769	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1770	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1771	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1772	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1773	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1774	#endif
1775
1776	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1777	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1778	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1779	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1780	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1781	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1782	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1783	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1784	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1785	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1786	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1787	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1788	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1789	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1790	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1791	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1792	#endif
1793	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1794	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1795	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1796	uint32_t fHstRegsNotToSave);
1797	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1798	uint32_t fHstRegsNotToSave);
1799	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1800	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1801
1802	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1803	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1804	#ifdef VBOX_STRICT
1805	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1806	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1807	IEMNATIVEGSTREG enmGstReg);
1808	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1809	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1810	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1811	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1812	# endif
1813	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1814	#endif
1815	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1816	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1817	#endif
1818	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1819	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1820	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1821	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1822	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1823	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1824	PCIEMTHRDEDCALLENTRY pCallEntry);
1825	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1826	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1827
1828
1829	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1830	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1831	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1832	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1833	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1834	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1835	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1836	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1837	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1838	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1839	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1840	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1841	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1842
1843	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1844	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1845	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1846	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1847	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1848	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1849	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1850	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1851	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1852	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1853	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1854	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1855	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1856	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1857	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1858	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1859	#endif
1860	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1861	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1862	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1863	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1864	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1865	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1866	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1867	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1868	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1869	#endif
1870	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1871	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1872	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1873	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1874	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1875	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1876	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1877
1878	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1879	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1880	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1881	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1882	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1883	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1884	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1885	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1886	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1887	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1888	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1889	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1890	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1891	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1892	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1893	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1894	#endif
1895	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1896	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1897	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1898	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1899	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1900	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1901	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1902	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1903	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1904	#endif
1905	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1906	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1907	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1908	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1909	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1910	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1911	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1912
1913	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1914	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1915	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1916	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1917	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1918	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1919	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1920	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1921	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1922	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1923	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1924	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1925	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1926	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1927	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1928	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1929	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1930	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1931	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1932	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1933	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1934	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1935
1936	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1937	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1938	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1939	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1940	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1941	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1942	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1943	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1944	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1945	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1946	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1947	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1948	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1949	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1950	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1951	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1952	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1953	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1954	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1955	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1956	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1957	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1958
1959	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1960	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1961	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1962	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1963
1964
1965	/**
1966	* Info about shadowed guest register values.
1967	* @see IEMNATIVEGSTREG
1968	*/
1969	typedef struct IEMANTIVEGSTREGINFO
1970	{
1971	/** Offset in VMCPU. */
1972	uint32_t off;
1973	/** The field size. */
1974	uint8_t cb;
1975	/** Name (for logging). */
1976	const char *pszName;
1977	} IEMANTIVEGSTREGINFO;
1978	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1979	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1980	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1981	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1982	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1983
1984
1985
1986	/**
1987	* Ensures that there is sufficient space in the instruction output buffer.
1988	*
1989	* This will reallocate the buffer if needed and allowed.
1990	*
1991	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1992	* allocation size.
1993	*
1994	* @returns Pointer to the instruction output buffer on success; throws VBox
1995	* status code on failure, so no need to check it.
1996	* @param pReNative The native recompile state.
1997	* @param off Current instruction offset. Works safely for UINT32_MAX
1998	* as well.
1999	* @param cInstrReq Number of instruction about to be added. It's okay to
2000	* overestimate this a bit.
2001	*/
2002	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2003	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2004	{
2005	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2006	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2007	{
2008	#ifdef VBOX_STRICT
2009	pReNative->offInstrBufChecked = offChecked;
2010	#endif
2011	return pReNative->pInstrBuf;
2012	}
2013	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2014	}
2015
2016	/**
2017	* Checks that we didn't exceed the space requested in the last
2018	* iemNativeInstrBufEnsure() call.
2019	*/
2020	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2021	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2022	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2023
2024	/**
2025	* Checks that a variable index is valid.
2026	*/
2027	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2028	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2029	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2030	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2031	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2032	("%s=%#x\n", #a_idxVar, a_idxVar))
2033	#else
2034	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2035	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2036	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2037	#endif
2038
2039	/**
2040	* Checks that a variable index is valid and that the variable is assigned the
2041	* correct argument number.
2042	* This also adds a RT_NOREF of a_idxVar.
2043	*/
2044	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2045	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2046	RT_NOREF_PV(a_idxVar); \
2047	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2048	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2049	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2050	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2051	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2052	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2053	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2054	a_uArgNo)); \
2055	} while (0)
2056	#else
2057	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2058	RT_NOREF_PV(a_idxVar); \
2059	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2060	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2061	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2062	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2063	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2064	} while (0)
2065	#endif
2066
2067
2068	/**
2069	* Checks that a variable has the expected size.
2070	*/
2071	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2072	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2073	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2074	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2075
2076
2077	/**
2078	* Calculates the stack address of a variable as a [r]BP displacement value.
2079	*/
2080	DECL_FORCE_INLINE(int32_t)
2081	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2082	{
2083	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2084	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2085	}
2086
2087
2088	/**
2089	* Releases the variable's register.
2090	*
2091	* The register must have been previously acquired calling
2092	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2093	* iemNativeVarRegisterSetAndAcquire().
2094	*/
2095	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2096	{
2097	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2098	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2099	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2100	}
2101
2102
2103	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2104	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2105	{
2106	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2107	iemNativeVarRegisterRelease(pReNative, idxVar);
2108	}
2109	#endif
2110
2111
2112	/**
2113	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2114	*
2115	* @returns The flush mask.
2116	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2117	* @param fGstShwFlush The starting flush mask.
2118	*/
2119	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2120	{
2121	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2122	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2123	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2124	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2125	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2126	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2127	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2128	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2129	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2130	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2131	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2132	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2133	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2134	return fGstShwFlush;
2135	}
2136
2137
2138	/** Number of hidden arguments for CIMPL calls.
2139	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2140	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2141	# define IEM_CIMPL_HIDDEN_ARGS 3
2142	#else
2143	# define IEM_CIMPL_HIDDEN_ARGS 2
2144	#endif
2145
2146
2147	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2148	/** Number of hidden arguments for SSE_AIMPL calls. */
2149	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2150	/** Number of hidden arguments for AVX_AIMPL calls. */
2151	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2152	#endif
2153
2154
2155	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2156
2157	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2158	/**
2159	* Helper for iemNativeLivenessGetStateByGstReg.
2160	*
2161	* @returns IEMLIVENESS_STATE_XXX
2162	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2163	* ORed together.
2164	*/
2165	DECL_FORCE_INLINE(uint32_t)
2166	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2167	{
2168	/* INPUT trumps anything else. */
2169	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2170	return IEMLIVENESS_STATE_INPUT;
2171
2172	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2173	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2174	{
2175	/* If not all sub-fields are clobbered they must be considered INPUT. */
2176	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2177	return IEMLIVENESS_STATE_INPUT;
2178	return IEMLIVENESS_STATE_CLOBBERED;
2179	}
2180
2181	/* XCPT_OR_CALL trumps UNUSED. */
2182	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2183	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2184
2185	return IEMLIVENESS_STATE_UNUSED;
2186	}
2187	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2188
2189
2190	DECL_FORCE_INLINE(uint32_t)
2191	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2192	{
2193	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2194	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2195	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2196	# else
2197	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2198	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2199	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2200	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2201	# endif
2202	}
2203
2204
2205	DECL_FORCE_INLINE(uint32_t)
2206	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2207	{
2208	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2209	if (enmGstReg == kIemNativeGstReg_EFlags)
2210	{
2211	/* Merge the eflags states to one. */
2212	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2213	uRet = RT_BIT_32(uRet);
2214	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2215	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2216	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2217	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2218	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2219	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2220	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2221	# else
2222	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2223	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2224	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2225	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2226	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2227	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2228	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2229	# endif
2230	}
2231	return uRet;
2232	}
2233
2234
2235	# ifdef VBOX_STRICT
2236	/** For assertions only, user checks that idxCurCall isn't zerow. */
2237	DECL_FORCE_INLINE(uint32_t)
2238	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2239	{
2240	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2241	}
2242	# endif /* VBOX_STRICT */
2243
2244	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2245
2246
2247	/**
2248	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2249	*/
2250	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2251	{
2252	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2253	return IEM_CIMPL_HIDDEN_ARGS;
2254	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2255	return 1;
2256	return 0;
2257	}
2258
2259
2260	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2261	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2262	{
2263	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2264
2265	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2266	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2267	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2268	return (uint8_t)idxReg;
2269	}
2270
2271
2272
2273	/*********************************************************************************************************************************
2274	* Register Allocator (GPR) *
2275	*********************************************************************************************************************************/
2276
2277	/**
2278	* Marks host register @a idxHstReg as containing a shadow copy of guest
2279	* register @a enmGstReg.
2280	*
2281	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2282	* host register before calling.
2283	*/
2284	DECL_FORCE_INLINE(void)
2285	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2286	{
2287	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2288	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2289	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2290
2291	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2292	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2293	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2294	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2295	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2296	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2297	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2298	#else
2299	RT_NOREF(off);
2300	#endif
2301	}
2302
2303
2304	/**
2305	* Clear any guest register shadow claims from @a idxHstReg.
2306	*
2307	* The register does not need to be shadowing any guest registers.
2308	*/
2309	DECL_FORCE_INLINE(void)
2310	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2311	{
2312	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2313	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2314	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2315	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2316	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2317	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2318	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2319	#endif
2320
2321	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2322	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2323	if (fGstRegs)
2324	{
2325	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2326	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2327	while (fGstRegs)
2328	{
2329	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2330	fGstRegs &= ~RT_BIT_64(iGstReg);
2331	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2332	}
2333	}
2334	#else
2335	RT_NOREF(off);
2336	#endif
2337
2338	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2339	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2340	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2341	}
2342
2343
2344	/**
2345	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2346	* and global overview flags.
2347	*/
2348	DECL_FORCE_INLINE(void)
2349	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2350	{
2351	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2352	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2353	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2354	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2355	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2356	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2357	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2358	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2359	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2360	#endif
2361
2362	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2363	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2364	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2365	#else
2366	RT_NOREF(off);
2367	#endif
2368
2369	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2370	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2371	if (!fGstRegShadowsNew)
2372	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2373	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2374	}
2375
2376
2377	#if 0 /* unused */
2378	/**
2379	* Clear any guest register shadow claim for @a enmGstReg.
2380	*/
2381	DECL_FORCE_INLINE(void)
2382	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2383	{
2384	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2385	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2386	{
2387	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2388	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2389	}
2390	}
2391	#endif
2392
2393
2394	/**
2395	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2396	* as the new shadow of it.
2397	*
2398	* Unlike the other guest reg shadow helpers, this does the logging for you.
2399	* However, it is the liveness state is not asserted here, the caller must do
2400	* that.
2401	*/
2402	DECL_FORCE_INLINE(void)
2403	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2404	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2405	{
2406	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2407	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2408	{
2409	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2410	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2411	if (idxHstRegOld == idxHstRegNew)
2412	return;
2413	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2414	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2415	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2416	}
2417	else
2418	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2419	g_aGstShadowInfo[enmGstReg].pszName));
2420	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2421	}
2422
2423
2424	/**
2425	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2426	* to @a idxRegTo.
2427	*/
2428	DECL_FORCE_INLINE(void)
2429	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2430	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2431	{
2432	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2433	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2434	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2435	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2436	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2437	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2438	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2439	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2440	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2441
2442	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2443	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2444	if (!fGstRegShadowsFrom)
2445	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2446	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2447	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2448	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2449	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2450	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2451	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2452	#else
2453	RT_NOREF(off);
2454	#endif
2455	}
2456
2457
2458	/**
2459	* Flushes any delayed guest register writes.
2460	*
2461	* This must be called prior to calling CImpl functions and any helpers that use
2462	* the guest state (like raising exceptions) and such.
2463	*
2464	* This optimization has not yet been implemented. The first target would be
2465	* RIP updates, since these are the most common ones.
2466	*
2467	* @note This function does not flush any shadowing information for guest
2468	* registers. This needs to be done by the caller if it wishes to do so.
2469	*/
2470	DECL_INLINE_THROW(uint32_t)
2471	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
2472	uint64_t fGstSimdShwExcept = 0)
2473	{
2474	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2475	uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
2476	#else
2477	uint64_t const fWritebackPc = 0;
2478	#endif
2479	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2480	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2481	#else
2482	uint64_t const bmGstRegShadowDirty = 0;
2483	#endif
2484	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2485	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
2486	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2487	& ~fGstSimdShwExcept;
2488	#else
2489	uint64_t const bmGstSimdRegShadowDirty = 0;
2490	#endif
2491	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2492	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2493
2494	return off;
2495	}
2496
2497
2498
2499	/*********************************************************************************************************************************
2500	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2501	*********************************************************************************************************************************/
2502
2503	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2504
2505	DECL_FORCE_INLINE(uint8_t)
2506	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2507	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2508	{
2509	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2510
2511	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2512	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2513	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2514	return idxSimdReg;
2515	}
2516
2517
2518	/**
2519	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2520	* SIMD register @a enmGstSimdReg.
2521	*
2522	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2523	* host register before calling.
2524	*/
2525	DECL_FORCE_INLINE(void)
2526	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2527	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2528	{
2529	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2530	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2531	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2532
2533	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2534	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2535	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2536	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2537	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2538	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2539	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2540	#else
2541	RT_NOREF(off);
2542	#endif
2543	}
2544
2545
2546	/**
2547	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2548	* to @a idxSimdRegTo.
2549	*/
2550	DECL_FORCE_INLINE(void)
2551	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2552	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2553	{
2554	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2555	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2556	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2557	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2558	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2559	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2560	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2561	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2562	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2563	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2564	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2565
2566	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2567	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2568	if (!fGstRegShadowsFrom)
2569	{
2570	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2571	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2572	}
2573	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2574	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2575	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2576	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2577	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2578	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2579	#else
2580	RT_NOREF(off);
2581	#endif
2582	}
2583
2584
2585	/**
2586	* Clear any guest register shadow claims from @a idxHstSimdReg.
2587	*
2588	* The register does not need to be shadowing any guest registers.
2589	*/
2590	DECL_FORCE_INLINE(void)
2591	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2592	{
2593	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2594	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2595	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2596	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2597	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2598	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2599	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2600
2601	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2602	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2603	if (fGstRegs)
2604	{
2605	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2606	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2607	while (fGstRegs)
2608	{
2609	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2610	fGstRegs &= ~RT_BIT_64(iGstReg);
2611	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2612	}
2613	}
2614	#else
2615	RT_NOREF(off);
2616	#endif
2617
2618	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2619	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2620	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2621	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2622	}
2623
2624	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2625
2626
2627	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2628	/**
2629	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2630	*/
2631	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2632	{
2633	if (pReNative->Core.offPc)
2634	return iemNativeEmitPcWritebackSlow(pReNative, off);
2635	return off;
2636	}
2637	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2638
2639
2640	#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2641	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2642	* it saves us the trouble of a hidden parameter on MSC/amd64. */
2643	# ifdef RT_ARCH_AMD64
2644	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2645	# elif defined(RT_ARCH_ARM64)
2646	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2647	# endif
2648	#endif
2649
2650	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2651	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2652	#endif
2653
2654	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2655
2656	/** @} */
2657
2658	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2659

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106061

Download in other formats: