IEMN8veRecompiler.h@ 103838

Last change on this file since 103838 was 103838, checked in by vboxsync, 13 months ago
VMM/IEM: Emit the IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE()/IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT()/IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks only once per TB or after a helper was called which could modify cr0/cr4/xcr0. Most of the time the check can be omitted for SIMD code, bugref:10614
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 113.5 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103838 2024-03-13 20:06:55Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0)
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
227	\| RT_BIT_32(ARMV8_A64_REG_Q30))
228	# endif
229	# endif
230
231	#else
232	# error "port me"
233	#endif
234	/** @} */
235
236	/** @name Call related registers.
237	* @{ */
238	/** @def IEMNATIVE_CALL_RET_GREG
239	* The return value register. */
240	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
241	* Number of arguments in registers. */
242	/** @def IEMNATIVE_CALL_ARG0_GREG
243	* The general purpose register carrying argument \#0. */
244	/** @def IEMNATIVE_CALL_ARG1_GREG
245	* The general purpose register carrying argument \#1. */
246	/** @def IEMNATIVE_CALL_ARG2_GREG
247	* The general purpose register carrying argument \#2. */
248	/** @def IEMNATIVE_CALL_ARG3_GREG
249	* The general purpose register carrying argument \#3. */
250	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
251	* Mask of registers the callee will not save and may trash. */
252	#ifdef RT_ARCH_AMD64
253	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
254
255	# ifdef RT_OS_WINDOWS
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
257	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
258	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
259	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
260	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
261	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
262	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
263	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
264	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
265	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
266	\| RT_BIT_32(X86_GREG_xCX) \
267	\| RT_BIT_32(X86_GREG_xDX) \
268	\| RT_BIT_32(X86_GREG_x8) \
269	\| RT_BIT_32(X86_GREG_x9) \
270	\| RT_BIT_32(X86_GREG_x10) \
271	\| RT_BIT_32(X86_GREG_x11) )
272	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
273	/* xmm0 - xmm5 are marked as volatile. */
274	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
275	# endif
276
277	# else
278	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
279	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
280	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
281	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
282	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
283	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
289	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
290	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
291	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
292	\| RT_BIT_32(X86_GREG_xCX) \
293	\| RT_BIT_32(X86_GREG_xDX) \
294	\| RT_BIT_32(X86_GREG_xDI) \
295	\| RT_BIT_32(X86_GREG_xSI) \
296	\| RT_BIT_32(X86_GREG_x8) \
297	\| RT_BIT_32(X86_GREG_x9) \
298	\| RT_BIT_32(X86_GREG_x10) \
299	\| RT_BIT_32(X86_GREG_x11) )
300	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
301	/* xmm0 - xmm15 are marked as volatile. */
302	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
303	# endif
304	# endif
305
306	#elif defined(RT_ARCH_ARM64)
307	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
308	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
309	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
310	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
311	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
312	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
313	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
314	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
315	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
316	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
317	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
318	\| RT_BIT_32(ARMV8_A64_REG_X1) \
319	\| RT_BIT_32(ARMV8_A64_REG_X2) \
320	\| RT_BIT_32(ARMV8_A64_REG_X3) \
321	\| RT_BIT_32(ARMV8_A64_REG_X4) \
322	\| RT_BIT_32(ARMV8_A64_REG_X5) \
323	\| RT_BIT_32(ARMV8_A64_REG_X6) \
324	\| RT_BIT_32(ARMV8_A64_REG_X7) )
325	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
326	\| RT_BIT_32(ARMV8_A64_REG_X1) \
327	\| RT_BIT_32(ARMV8_A64_REG_X2) \
328	\| RT_BIT_32(ARMV8_A64_REG_X3) \
329	\| RT_BIT_32(ARMV8_A64_REG_X4) \
330	\| RT_BIT_32(ARMV8_A64_REG_X5) \
331	\| RT_BIT_32(ARMV8_A64_REG_X6) \
332	\| RT_BIT_32(ARMV8_A64_REG_X7) \
333	\| RT_BIT_32(ARMV8_A64_REG_X8) \
334	\| RT_BIT_32(ARMV8_A64_REG_X9) \
335	\| RT_BIT_32(ARMV8_A64_REG_X10) \
336	\| RT_BIT_32(ARMV8_A64_REG_X11) \
337	\| RT_BIT_32(ARMV8_A64_REG_X12) \
338	\| RT_BIT_32(ARMV8_A64_REG_X13) \
339	\| RT_BIT_32(ARMV8_A64_REG_X14) \
340	\| RT_BIT_32(ARMV8_A64_REG_X15) \
341	\| RT_BIT_32(ARMV8_A64_REG_X16) \
342	\| RT_BIT_32(ARMV8_A64_REG_X17) )
343	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
344	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
345	* so to simplify our life a bit we just mark everything as volatile. */
346	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
347	# endif
348
349	#endif
350
351	/** This is the maximum argument count we'll ever be needing. */
352	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
353	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
354	#else
355	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
356	#endif
357	/** @} */
358
359
360	/** @def IEMNATIVE_HST_GREG_COUNT
361	* Number of host general purpose registers we tracker. */
362	/** @def IEMNATIVE_HST_GREG_MASK
363	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
364	* inverted register masks and such to get down to a correct set of regs. */
365	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
366	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
367	* Number of host SIMD registers we track. */
368	/** @def IEMNATIVE_HST_SIMD_REG_MASK
369	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
370	* inverted register masks and such to get down to a correct set of regs. */
371	#endif
372	#ifdef RT_ARCH_AMD64
373	# define IEMNATIVE_HST_GREG_COUNT 16
374	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
375
376	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
377	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
378	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
379	# endif
380
381	#elif defined(RT_ARCH_ARM64)
382	# define IEMNATIVE_HST_GREG_COUNT 32
383	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
384
385	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
386	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
387	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
388	# endif
389
390	#else
391	# error "Port me!"
392	#endif
393
394
395	/** Native code generator label types. */
396	typedef enum
397	{
398	kIemNativeLabelType_Invalid = 0,
399	/* Labels w/o data, only once instance per TB: */
400	kIemNativeLabelType_Return,
401	kIemNativeLabelType_ReturnBreak,
402	kIemNativeLabelType_ReturnWithFlags,
403	kIemNativeLabelType_NonZeroRetOrPassUp,
404	kIemNativeLabelType_RaiseGp0,
405	kIemNativeLabelType_RaiseNm,
406	kIemNativeLabelType_RaiseUd,
407	kIemNativeLabelType_RaiseMf,
408	kIemNativeLabelType_RaiseXf,
409	kIemNativeLabelType_ObsoleteTb,
410	kIemNativeLabelType_NeedCsLimChecking,
411	kIemNativeLabelType_CheckBranchMiss,
412	/* Labels with data, potentially multiple instances per TB: */
413	kIemNativeLabelType_FirstWithMultipleInstances,
414	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
415	kIemNativeLabelType_Else,
416	kIemNativeLabelType_Endif,
417	kIemNativeLabelType_CheckIrq,
418	kIemNativeLabelType_TlbLookup,
419	kIemNativeLabelType_TlbMiss,
420	kIemNativeLabelType_TlbDone,
421	kIemNativeLabelType_End
422	} IEMNATIVELABELTYPE;
423
424	/** Native code generator label definition. */
425	typedef struct IEMNATIVELABEL
426	{
427	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
428	* the epilog. */
429	uint32_t off;
430	/** The type of label (IEMNATIVELABELTYPE). */
431	uint16_t enmType;
432	/** Additional label data, type specific. */
433	uint16_t uData;
434	} IEMNATIVELABEL;
435	/** Pointer to a label. */
436	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
437
438
439	/** Native code generator fixup types. */
440	typedef enum
441	{
442	kIemNativeFixupType_Invalid = 0,
443	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
444	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
445	kIemNativeFixupType_Rel32,
446	#elif defined(RT_ARCH_ARM64)
447	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
448	kIemNativeFixupType_RelImm26At0,
449	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
450	kIemNativeFixupType_RelImm19At5,
451	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
452	kIemNativeFixupType_RelImm14At5,
453	#endif
454	kIemNativeFixupType_End
455	} IEMNATIVEFIXUPTYPE;
456
457	/** Native code generator fixup. */
458	typedef struct IEMNATIVEFIXUP
459	{
460	/** Code offset of the fixup location. */
461	uint32_t off;
462	/** The IEMNATIVELABEL this is a fixup for. */
463	uint16_t idxLabel;
464	/** The fixup type (IEMNATIVEFIXUPTYPE). */
465	uint8_t enmType;
466	/** Addend or other data. */
467	int8_t offAddend;
468	} IEMNATIVEFIXUP;
469	/** Pointer to a native code generator fixup. */
470	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
471
472
473	/**
474	* One bit of the state.
475	*
476	* Each register state takes up two bits. We keep the two bits in two separate
477	* 64-bit words to simplify applying them to the guest shadow register mask in
478	* the register allocator.
479	*/
480	typedef union IEMLIVENESSBIT
481	{
482	uint64_t bm64;
483	RT_GCC_EXTENSION struct
484	{ /* bit no */
485	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
486	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
487	uint64_t fCr0 : 1; /*< 0x11 / 17: /
488	uint64_t fFcw : 1; /*< 0x12 / 18: /
489	uint64_t fFsw : 1; /*< 0x13 / 19: /
490	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
491	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
492	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
493	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
494	uint64_t fCr4 : 1; /*< 0x2c / 44: /
495	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
496	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
497	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
498	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
499	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
500	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
501	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
502	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
503	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
504	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
505	};
506	} IEMLIVENESSBIT;
507	AssertCompileSize(IEMLIVENESSBIT, 8);
508
509	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
510	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
511	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
512	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
513	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
514	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
515	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
516
517
518	/**
519	* A liveness state entry.
520	*
521	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
522	* Once we add a SSE register shadowing, we'll add another 64-bit element for
523	* that.
524	*/
525	typedef union IEMLIVENESSENTRY
526	{
527	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
528	uint64_t bm64[16 / 8];
529	uint16_t bm32[16 / 4];
530	uint16_t bm16[16 / 2];
531	uint8_t bm8[ 16 / 1];
532	IEMLIVENESSBIT aBits[2];
533	#else
534	uint64_t bm64[32 / 8];
535	uint16_t bm32[32 / 4];
536	uint16_t bm16[32 / 2];
537	uint8_t bm8[ 32 / 1];
538	IEMLIVENESSBIT aBits[4];
539	#endif
540	RT_GCC_EXTENSION struct
541	{
542	/** Bit \#0 of the register states. */
543	IEMLIVENESSBIT Bit0;
544	/** Bit \#1 of the register states. */
545	IEMLIVENESSBIT Bit1;
546	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
547	/** Bit \#2 of the register states. */
548	IEMLIVENESSBIT Bit2;
549	/** Bit \#3 of the register states. */
550	IEMLIVENESSBIT Bit3;
551	#endif
552	};
553	} IEMLIVENESSENTRY;
554	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
555	AssertCompileSize(IEMLIVENESSENTRY, 16);
556	#else
557	AssertCompileSize(IEMLIVENESSENTRY, 32);
558	#endif
559	/** Pointer to a liveness state entry. */
560	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
561	/** Pointer to a const liveness state entry. */
562	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
563
564	/** @name 64-bit value masks for IEMLIVENESSENTRY.
565	* @{ / / 0xzzzzyyyyxxxxwwww */
566	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
567
568	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
569	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
570	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
571
572	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
573	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
574	#endif
575
576	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
577	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
578
579	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
580	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
581	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
582	#endif
583	/** @} */
584
585
586	/** @name The liveness state for a register.
587	*
588	* The state values have been picked to with state accumulation in mind (what
589	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
590	* performance critical work done with the values.
591	*
592	* This is a compressed state that only requires 2 bits per register.
593	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
594	* 1. the incoming state from the following call,
595	* 2. the outgoing state for this call,
596	* 3. mask of the entries set in the 2nd.
597	*
598	* The mask entry (3rd one above) will be used both when updating the outgoing
599	* state and when merging in incoming state for registers not touched by the
600	* current call.
601	*
602	* @{ */
603	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
604	/** The register will be clobbered and the current value thrown away.
605	*
606	* When this is applied to the state (2) we'll simply be AND'ing it with the
607	* (old) mask (3) and adding the register to the mask. This way we'll
608	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
609	* IEMLIVENESS_STATE_INPUT states. */
610	# define IEMLIVENESS_STATE_CLOBBERED 0
611	/** The register is unused in the remainder of the TB.
612	*
613	* This is an initial state and can not be set by any of the
614	* iemNativeLivenessFunc_xxxx callbacks. */
615	# define IEMLIVENESS_STATE_UNUSED 1
616	/** The register value is required in a potential call or exception.
617	*
618	* This means that the register value must be calculated and is best written to
619	* the state, but that any shadowing registers can be flushed thereafter as it's
620	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
621	*
622	* It is typically applied across the board, but we preserve incoming
623	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
624	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
625	* 1. r0 = old & ~mask;
626	* 2. r0 = t1 & (t1 >> 1)'
627	* 3. state \|= r0 \| 0b10;
628	* 4. mask = ~0;
629	*/
630	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
631	/** The register value is used as input.
632	*
633	* This means that the register value must be calculated and it is best to keep
634	* it in a register. It does not need to be writtent out as such. This is the
635	* highest priority state.
636	*
637	* Whether the call modifies the register or not isn't relevant to earlier
638	* calls, so that's not recorded.
639	*
640	* When applying this state we just or in the value in the outgoing state and
641	* mask. */
642	# define IEMLIVENESS_STATE_INPUT 3
643	/** Mask of the state bits. */
644	# define IEMLIVENESS_STATE_MASK 3
645	/** The number of bits per state. */
646	# define IEMLIVENESS_STATE_BIT_COUNT 2
647	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
648	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
649	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
650	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
651	/** Check if a register clobbering is expected given the (previous) liveness state.
652	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
653	* include INPUT if the register is used in more than one place. */
654	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
655
656	/** Check if all status flags are going to be clobbered and doesn't need
657	* calculating in the current step.
658	* @param a_pEntry The current liveness entry. */
659	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
660	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
661
662	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
663	/** The register is not used any more. */
664	# define IEMLIVENESS_STATE_UNUSED 0
665	/** Flag: The register is required in a potential exception or call. */
666	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
667	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
668	/** Flag: The register is read. */
669	# define IEMLIVENESS_STATE_READ 2
670	# define IEMLIVENESS_BIT_READ 1
671	/** Flag: The register is written. */
672	# define IEMLIVENESS_STATE_WRITE 4
673	# define IEMLIVENESS_BIT_WRITE 2
674	/** Flag: Unconditional call (not needed, can be redefined for research). */
675	# define IEMLIVENESS_STATE_CALL 8
676	# define IEMLIVENESS_BIT_CALL 3
677	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
678	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
679	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
680	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
681	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
682
683	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
684	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
685	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
686	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
687
688	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
689	/** @} */
690
691	/** @name Liveness helpers for builtin functions and similar.
692	*
693	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
694	* own set of manimulator macros for those.
695	*
696	* @{ */
697	/** Initializing the state as all unused. */
698	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
699	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
700	do { \
701	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
702	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
703	} while (0)
704	#else
705	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
706	do { \
707	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
708	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
709	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
710	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
711	} while (0)
712	#endif
713
714	/** Initializing the outgoing state with a potential xcpt or call state.
715	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
716	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
717	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
718	do { \
719	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
720	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
721	} while (0)
722	#else
723	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
724	do { \
725	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
726	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
727	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
728	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
729	} while (0)
730	#endif
731
732	/** Adds a segment base register as input to the outgoing state. */
733	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
734	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
735	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
736	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
737	} while (0)
738	#else
739	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
740	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
741	} while (0)
742	#endif
743
744	/** Adds a segment attribute register as input to the outgoing state. */
745	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
746	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
747	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
748	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
749	} while (0)
750	#else
751	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
752	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
753	} while (0)
754	#endif
755
756	/** Adds a segment limit register as input to the outgoing state. */
757	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
758	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
759	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
760	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
761	} while (0)
762	#else
763	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
764	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
765	} while (0)
766	#endif
767
768	/** Adds a segment limit register as input to the outgoing state. */
769	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
770	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
771	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
772	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
773	} while (0)
774	#else
775	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
776	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
777	} while (0)
778	#endif
779	/** @} */
780
781	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
782	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
783	* calculated and up to date. This is to double check that we haven't skipped
784	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
785	* @note has to be placed in
786	*/
787	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
788	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
789	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
790	#else
791	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
792	#endif
793
794
795	/**
796	* Guest registers that can be shadowed in GPRs.
797	*
798	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
799	* must be placed last, as the liveness state tracks it as 7 subcomponents and
800	* we don't want to waste space here.
801	*
802	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
803	* friends as well as IEMAllN8veLiveness.cpp.
804	*/
805	typedef enum IEMNATIVEGSTREG : uint8_t
806	{
807	kIemNativeGstReg_GprFirst = 0,
808	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
809	kIemNativeGstReg_Pc,
810	kIemNativeGstReg_Cr0,
811	kIemNativeGstReg_FpuFcw,
812	kIemNativeGstReg_FpuFsw,
813	kIemNativeGstReg_SegBaseFirst,
814	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
815	kIemNativeGstReg_SegAttribFirst,
816	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
817	kIemNativeGstReg_SegLimitFirst,
818	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
819	kIemNativeGstReg_SegSelFirst,
820	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
821	kIemNativeGstReg_Cr4,
822	kIemNativeGstReg_Xcr0,
823	kIemNativeGstReg_MxCsr,
824	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
825	kIemNativeGstReg_End
826	} IEMNATIVEGSTREG;
827	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
828	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
829
830	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
831	* @{ */
832	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
833	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
834	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
835	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
836	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
837	/** @} */
838
839	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
840
841	/**
842	* Guest registers that can be shadowed in host SIMD registers.
843	*
844	* @todo r=aeichner Liveness tracking
845	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
846	*/
847	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
848	{
849	kIemNativeGstSimdReg_SimdRegFirst = 0,
850	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
851	kIemNativeGstSimdReg_End
852	} IEMNATIVEGSTSIMDREG;
853
854	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
855	* @{ */
856	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
857	/** @} */
858
859	/**
860	* The Load/store size for a SIMD guest register.
861	*/
862	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
863	{
864	/** Invalid size. */
865	kIemNativeGstSimdRegLdStSz_Invalid = 0,
866	/** Loads the low 128-bit of a guest SIMD register. */
867	kIemNativeGstSimdRegLdStSz_Low128,
868	/** Loads the high 128-bit of a guest SIMD register. */
869	kIemNativeGstSimdRegLdStSz_High128,
870	/** Loads the whole 256-bits of a guest SIMD register. */
871	kIemNativeGstSimdRegLdStSz_256,
872	/** End value. */
873	kIemNativeGstSimdRegLdStSz_End
874	} IEMNATIVEGSTSIMDREGLDSTSZ;
875
876	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
877
878	/**
879	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
880	*/
881	typedef enum IEMNATIVEGSTREGUSE
882	{
883	/** The usage is read-only, the register holding the guest register
884	* shadow copy will not be modified by the caller. */
885	kIemNativeGstRegUse_ReadOnly = 0,
886	/** The caller will update the guest register (think: PC += cbInstr).
887	* The guest shadow copy will follow the returned register. */
888	kIemNativeGstRegUse_ForUpdate,
889	/** The call will put an entirely new value in the guest register, so
890	* if new register is allocate it will be returned uninitialized. */
891	kIemNativeGstRegUse_ForFullWrite,
892	/** The caller will use the guest register value as input in a calculation
893	* and the host register will be modified.
894	* This means that the returned host register will not be marked as a shadow
895	* copy of the guest register. */
896	kIemNativeGstRegUse_Calculation
897	} IEMNATIVEGSTREGUSE;
898
899	/**
900	* Guest registers (classes) that can be referenced.
901	*/
902	typedef enum IEMNATIVEGSTREGREF : uint8_t
903	{
904	kIemNativeGstRegRef_Invalid = 0,
905	kIemNativeGstRegRef_Gpr,
906	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
907	kIemNativeGstRegRef_EFlags,
908	kIemNativeGstRegRef_MxCsr,
909	kIemNativeGstRegRef_FpuReg,
910	kIemNativeGstRegRef_MReg,
911	kIemNativeGstRegRef_XReg,
912	//kIemNativeGstRegRef_YReg, - doesn't work.
913	kIemNativeGstRegRef_End
914	} IEMNATIVEGSTREGREF;
915
916
917	/** Variable kinds. */
918	typedef enum IEMNATIVEVARKIND : uint8_t
919	{
920	/** Customary invalid zero value. */
921	kIemNativeVarKind_Invalid = 0,
922	/** This is either in a register or on the stack. */
923	kIemNativeVarKind_Stack,
924	/** Immediate value - loaded into register when needed, or can live on the
925	* stack if referenced (in theory). */
926	kIemNativeVarKind_Immediate,
927	/** Variable reference - loaded into register when needed, never stack. */
928	kIemNativeVarKind_VarRef,
929	/** Guest register reference - loaded into register when needed, never stack. */
930	kIemNativeVarKind_GstRegRef,
931	/** End of valid values. */
932	kIemNativeVarKind_End
933	} IEMNATIVEVARKIND;
934
935
936	/** Variable or argument. */
937	typedef struct IEMNATIVEVAR
938	{
939	/** The kind of variable. */
940	IEMNATIVEVARKIND enmKind;
941	/** The variable size in bytes. */
942	uint8_t cbVar;
943	/** The first stack slot (uint64_t), except for immediate and references
944	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
945	* has a stack slot it has been initialized and has a value. Unused variables
946	* has neither a stack slot nor a host register assignment. */
947	uint8_t idxStackSlot;
948	/** The host register allocated for the variable, UINT8_MAX if not. */
949	uint8_t idxReg;
950	/** The argument number if argument, UINT8_MAX if regular variable. */
951	uint8_t uArgNo;
952	/** If referenced, the index (unpacked) of the variable referencing this one,
953	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
954	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
955	uint8_t idxReferrerVar;
956	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
957	* @todo not sure what this really is for... */
958	IEMNATIVEGSTREG enmGstReg;
959	/** Set if the registered is currently used exclusively, false if the
960	* variable is idle and the register can be grabbed. */
961	bool fRegAcquired;
962
963	union
964	{
965	/** kIemNativeVarKind_Immediate: The immediate value. */
966	uint64_t uValue;
967	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
968	uint8_t idxRefVar;
969	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
970	struct
971	{
972	/** The class of register. */
973	IEMNATIVEGSTREGREF enmClass;
974	/** Index within the class. */
975	uint8_t idx;
976	} GstRegRef;
977	} u;
978	} IEMNATIVEVAR;
979	/** Pointer to a variable or argument. */
980	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
981	/** Pointer to a const variable or argument. */
982	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
983
984	/** What is being kept in a host register. */
985	typedef enum IEMNATIVEWHAT : uint8_t
986	{
987	/** The traditional invalid zero value. */
988	kIemNativeWhat_Invalid = 0,
989	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
990	kIemNativeWhat_Var,
991	/** Temporary register, this is typically freed when a MC completes. */
992	kIemNativeWhat_Tmp,
993	/** Call argument w/o a variable mapping. This is free (via
994	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
995	kIemNativeWhat_Arg,
996	/** Return status code.
997	* @todo not sure if we need this... */
998	kIemNativeWhat_rc,
999	/** The fixed pVCpu (PVMCPUCC) register.
1000	* @todo consider offsetting this on amd64 to use negative offsets to access
1001	* more members using 8-byte disp. */
1002	kIemNativeWhat_pVCpuFixed,
1003	/** The fixed pCtx (PCPUMCTX) register.
1004	* @todo consider offsetting this on amd64 to use negative offsets to access
1005	* more members using 8-byte disp. */
1006	kIemNativeWhat_pCtxFixed,
1007	/** Fixed temporary register. */
1008	kIemNativeWhat_FixedTmp,
1009	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1010	/** Shadow RIP for the delayed RIP updating debugging. */
1011	kIemNativeWhat_PcShadow,
1012	#endif
1013	/** Register reserved by the CPU or OS architecture. */
1014	kIemNativeWhat_FixedReserved,
1015	/** End of valid values. */
1016	kIemNativeWhat_End
1017	} IEMNATIVEWHAT;
1018
1019	/**
1020	* Host general register entry.
1021	*
1022	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1023	*
1024	* @todo Track immediate values in host registers similarlly to how we track the
1025	* guest register shadow copies. For it to be real helpful, though,
1026	* we probably need to know which will be reused and put them into
1027	* non-volatile registers, otherwise it's going to be more or less
1028	* restricted to an instruction or two.
1029	*/
1030	typedef struct IEMNATIVEHSTREG
1031	{
1032	/** Set of guest registers this one shadows.
1033	*
1034	* Using a bitmap here so we can designate the same host register as a copy
1035	* for more than one guest register. This is expected to be useful in
1036	* situations where one value is copied to several registers in a sequence.
1037	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1038	* sequence we'd want to let this register follow to be a copy of and there
1039	* will always be places where we'd be picking the wrong one.
1040	*/
1041	uint64_t fGstRegShadows;
1042	/** What is being kept in this register. */
1043	IEMNATIVEWHAT enmWhat;
1044	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1045	uint8_t idxVar;
1046	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1047	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1048	* that scope. */
1049	uint8_t idxStackSlot;
1050	/** Alignment padding. */
1051	uint8_t abAlign[5];
1052	} IEMNATIVEHSTREG;
1053
1054
1055	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1056	/**
1057	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1058	* halves, on architectures where there is no 256-bit register available this entry will track
1059	* two adjacent 128-bit host registers.
1060	*
1061	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1062	*/
1063	typedef struct IEMNATIVEHSTSIMDREG
1064	{
1065	/** Set of guest registers this one shadows.
1066	*
1067	* Using a bitmap here so we can designate the same host register as a copy
1068	* for more than one guest register. This is expected to be useful in
1069	* situations where one value is copied to several registers in a sequence.
1070	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1071	* sequence we'd want to let this register follow to be a copy of and there
1072	* will always be places where we'd be picking the wrong one.
1073	*/
1074	uint64_t fGstRegShadows;
1075	/** What is being kept in this register. */
1076	IEMNATIVEWHAT enmWhat;
1077	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1078	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1079	/** Alignment padding. */
1080	uint8_t abAlign[6];
1081	} IEMNATIVEHSTSIMDREG;
1082	#endif
1083
1084
1085	/**
1086	* Core state for the native recompiler, that is, things that needs careful
1087	* handling when dealing with branches.
1088	*/
1089	typedef struct IEMNATIVECORESTATE
1090	{
1091	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1092	/** The current instruction offset in bytes from when the guest program counter
1093	* was updated last. Used for delaying the write to the guest context program counter
1094	* as long as possible. */
1095	uint32_t offPc;
1096	/** Number of instructions where we could skip the updating. */
1097	uint32_t cInstrPcUpdateSkipped;
1098	#endif
1099	/** Allocation bitmap for aHstRegs. */
1100	uint32_t bmHstRegs;
1101
1102	/** Bitmap marking which host register contains guest register shadow copies.
1103	* This is used during register allocation to try preserve copies. */
1104	uint32_t bmHstRegsWithGstShadow;
1105	/** Bitmap marking valid entries in aidxGstRegShadows. */
1106	uint64_t bmGstRegShadows;
1107
1108	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1109	/** Allocation bitmap for aHstSimdRegs. */
1110	uint32_t bmHstSimdRegs;
1111
1112	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1113	* This is used during register allocation to try preserve copies. */
1114	uint32_t bmHstSimdRegsWithGstShadow;
1115	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1116	uint64_t bmGstSimdRegShadows;
1117	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1118	uint64_t bmGstSimdRegShadowDirtyLo128;
1119	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1120	uint64_t bmGstSimdRegShadowDirtyHi128;
1121	#endif
1122
1123	union
1124	{
1125	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1126	uint8_t aidxArgVars[8];
1127	/** For more efficient resetting. */
1128	uint64_t u64ArgVars;
1129	};
1130
1131	/** Allocation bitmap for the stack. */
1132	uint32_t bmStack;
1133	/** Allocation bitmap for aVars. */
1134	uint32_t bmVars;
1135
1136	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1137	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1138	* (A shadow copy of a guest register can only be held in a one host register,
1139	* there are no duplicate copies or ambiguities like that). */
1140	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1141	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1142	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1143	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1144	* (A shadow copy of a guest register can only be held in a one host register,
1145	* there are no duplicate copies or ambiguities like that). */
1146	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1147	#endif
1148
1149	/** Host register allocation tracking. */
1150	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1151	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1152	/** Host SIMD register allocation tracking. */
1153	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1154	#endif
1155
1156	/** Variables and arguments. */
1157	IEMNATIVEVAR aVars[9];
1158	} IEMNATIVECORESTATE;
1159	/** Pointer to core state. */
1160	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1161	/** Pointer to const core state. */
1162	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1163
1164	/** @def IEMNATIVE_VAR_IDX_UNPACK
1165	* @returns Index into IEMNATIVECORESTATE::aVars.
1166	* @param a_idxVar Variable index w/ magic (in strict builds).
1167	*/
1168	/** @def IEMNATIVE_VAR_IDX_PACK
1169	* @returns Variable index w/ magic (in strict builds).
1170	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1171	*/
1172	#ifdef VBOX_STRICT
1173	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1174	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1175	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1176	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1177	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1178	#else
1179	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1180	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1181	#endif
1182
1183
1184	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1185	/** Clear the dirty state of the given guest SIMD register. */
1186	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1187	do { \
1188	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1189	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1190	} while (0)
1191
1192	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1193	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1194	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1195	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1196	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1197	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1198	/** Returns whether the given guest SIMD register is dirty. */
1199	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1200	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1201
1202	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1203	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1204	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1205	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1206	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1207	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1208
1209	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1210	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1211	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1212	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1213	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1214	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1215	#endif
1216
1217
1218	/**
1219	* Conditional stack entry.
1220	*/
1221	typedef struct IEMNATIVECOND
1222	{
1223	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1224	bool fInElse;
1225	/** The label for the IEM_MC_ELSE. */
1226	uint32_t idxLabelElse;
1227	/** The label for the IEM_MC_ENDIF. */
1228	uint32_t idxLabelEndIf;
1229	/** The initial state snapshot as the if-block starts executing. */
1230	IEMNATIVECORESTATE InitialState;
1231	/** The state snapshot at the end of the if-block. */
1232	IEMNATIVECORESTATE IfFinalState;
1233	} IEMNATIVECOND;
1234	/** Pointer to a condition stack entry. */
1235	typedef IEMNATIVECOND *PIEMNATIVECOND;
1236
1237
1238	/**
1239	* Native recompiler state.
1240	*/
1241	typedef struct IEMRECOMPILERSTATE
1242	{
1243	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1244	* IEMNATIVEINSTR units. */
1245	uint32_t cInstrBufAlloc;
1246	#ifdef VBOX_STRICT
1247	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1248	uint32_t offInstrBufChecked;
1249	#else
1250	uint32_t uPadding1; /* We don't keep track of the size here... */
1251	#endif
1252	/** Fixed temporary code buffer for native recompilation. */
1253	PIEMNATIVEINSTR pInstrBuf;
1254
1255	/** Bitmaps with the label types used. */
1256	uint64_t bmLabelTypes;
1257	/** Actual number of labels in paLabels. */
1258	uint32_t cLabels;
1259	/** Max number of entries allowed in paLabels before reallocating it. */
1260	uint32_t cLabelsAlloc;
1261	/** Labels defined while recompiling (referenced by fixups). */
1262	PIEMNATIVELABEL paLabels;
1263	/** Array with indexes of unique labels (uData always 0). */
1264	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1265
1266	/** Actual number of fixups paFixups. */
1267	uint32_t cFixups;
1268	/** Max number of entries allowed in paFixups before reallocating it. */
1269	uint32_t cFixupsAlloc;
1270	/** Buffer used by the recompiler for recording fixups when generating code. */
1271	PIEMNATIVEFIXUP paFixups;
1272
1273	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1274	/** Number of debug info entries allocated for pDbgInfo. */
1275	uint32_t cDbgInfoAlloc;
1276	uint32_t uPadding;
1277	/** Debug info. */
1278	PIEMTBDBG pDbgInfo;
1279	#endif
1280
1281	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1282	/** The current call index (liveness array and threaded calls in TB). */
1283	uint32_t idxCurCall;
1284	/** Number of liveness entries allocated. */
1285	uint32_t cLivenessEntriesAlloc;
1286	/** Liveness entries for all the calls in the TB begin recompiled.
1287	* The entry for idxCurCall contains the info for what the next call will
1288	* require wrt registers. (Which means the last entry is the initial liveness
1289	* state.) */
1290	PIEMLIVENESSENTRY paLivenessEntries;
1291	#endif
1292
1293	/** The translation block being recompiled. */
1294	PCIEMTB pTbOrg;
1295	/** The VMCPU structure of the EMT. */
1296	PVMCPUCC pVCpu;
1297
1298	/** Condition sequence number (for generating unique labels). */
1299	uint16_t uCondSeqNo;
1300	/** Check IRQ seqeunce number (for generating unique labels). */
1301	uint16_t uCheckIrqSeqNo;
1302	/** TLB load sequence number (for generating unique labels). */
1303	uint16_t uTlbSeqNo;
1304	/** The current condition stack depth (aCondStack). */
1305	uint8_t cCondDepth;
1306
1307	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1308	uint8_t cArgs;
1309	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1310	uint32_t fCImpl;
1311	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1312	uint32_t fMc;
1313	/** The expected IEMCPU::fExec value for the current call/instruction. */
1314	uint32_t fExec;
1315	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1316	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1317	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1318	*
1319	* This is an optimization because these control registers can only be changed from
1320	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1321	* consisting of multiple SIMD instructions.
1322	*/
1323	uint32_t fSimdRaiseXcptChecksEmitted;
1324	#endif
1325
1326	/** Core state requiring care with branches. */
1327	IEMNATIVECORESTATE Core;
1328
1329	/** The condition nesting stack. */
1330	IEMNATIVECOND aCondStack[2];
1331
1332	#ifndef IEM_WITH_THROW_CATCH
1333	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1334	* for recompilation error handling. */
1335	jmp_buf JmpBuf;
1336	#endif
1337	} IEMRECOMPILERSTATE;
1338	/** Pointer to a native recompiler state. */
1339	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1340
1341
1342	/** @def IEMNATIVE_TRY_SETJMP
1343	* Wrapper around setjmp / try, hiding all the ugly differences.
1344	*
1345	* @note Use with extreme care as this is a fragile macro.
1346	* @param a_pReNative The native recompile state.
1347	* @param a_rcTarget The variable that should receive the status code in case
1348	* of a longjmp/throw.
1349	*/
1350	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1351	* Start wrapper for catch / setjmp-else.
1352	*
1353	* This will set up a scope.
1354	*
1355	* @note Use with extreme care as this is a fragile macro.
1356	* @param a_pReNative The native recompile state.
1357	* @param a_rcTarget The variable that should receive the status code in case
1358	* of a longjmp/throw.
1359	*/
1360	/** @def IEMNATIVE_CATCH_LONGJMP_END
1361	* End wrapper for catch / setjmp-else.
1362	*
1363	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1364	* up the state.
1365	*
1366	* @note Use with extreme care as this is a fragile macro.
1367	* @param a_pReNative The native recompile state.
1368	*/
1369	/** @def IEMNATIVE_DO_LONGJMP
1370	*
1371	* Wrapper around longjmp / throw.
1372	*
1373	* @param a_pReNative The native recompile state.
1374	* @param a_rc The status code jump back with / throw.
1375	*/
1376	#ifdef IEM_WITH_THROW_CATCH
1377	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1378	a_rcTarget = VINF_SUCCESS; \
1379	try
1380	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1381	catch (int rcThrown) \
1382	{ \
1383	a_rcTarget = rcThrown
1384	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1385	} \
1386	((void)0)
1387	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1388	#else /* !IEM_WITH_THROW_CATCH */
1389	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1390	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1391	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1392	else \
1393	{ \
1394	((void)0)
1395	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1396	}
1397	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1398	#endif /* !IEM_WITH_THROW_CATCH */
1399
1400
1401	/**
1402	* Native recompiler worker for a threaded function.
1403	*
1404	* @returns New code buffer offset; throws VBox status code in case of a failure.
1405	* @param pReNative The native recompiler state.
1406	* @param off The current code buffer offset.
1407	* @param pCallEntry The threaded call entry.
1408	*
1409	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1410	*/
1411	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1412	/** Pointer to a native recompiler worker for a threaded function. */
1413	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1414
1415	/** Defines a native recompiler worker for a threaded function.
1416	* @see FNIEMNATIVERECOMPFUNC */
1417	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1418	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1419
1420	/** Prototypes a native recompiler function for a threaded function.
1421	* @see FNIEMNATIVERECOMPFUNC */
1422	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1423
1424
1425	/**
1426	* Native recompiler liveness analysis worker for a threaded function.
1427	*
1428	* @param pCallEntry The threaded call entry.
1429	* @param pIncoming The incoming liveness state entry.
1430	* @param pOutgoing The outgoing liveness state entry.
1431	*/
1432	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1433	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1434	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1435	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1436
1437	/** Defines a native recompiler liveness analysis worker for a threaded function.
1438	* @see FNIEMNATIVELIVENESSFUNC */
1439	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1440	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1441
1442	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1443	* @see FNIEMNATIVELIVENESSFUNC */
1444	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1445
1446
1447	/** Define a native recompiler helper function, safe to call from the TB code. */
1448	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1449	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1450	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1451	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1452	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1453
1454
1455	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1456	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1457	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1458	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1459	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1460	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1461	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1462	uint8_t idxHstSimdReg = UINT8_MAX,
1463	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1464	# endif
1465	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1466	uint32_t offPc, uint32_t cInstrSkipped);
1467	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1468
1469	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1470	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1471	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1472	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1473	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1474	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1475
1476	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1477	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1478	bool fPreferVolatile = true);
1479	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1480	bool fPreferVolatile = true);
1481	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1482	IEMNATIVEGSTREG enmGstReg,
1483	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1484	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1485	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1486	IEMNATIVEGSTREG enmGstReg);
1487
1488	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1489	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1490	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1491	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1492	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1493	#endif
1494	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1495	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1496	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1497	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1498	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1499	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1500	uint32_t fKeepVars = 0);
1501	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1502	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1503	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1504	uint32_t fHstRegsActiveShadows);
1505	#ifdef VBOX_STRICT
1506	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1507	#endif
1508	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1509	uint64_t fGstShwExcept, bool fFlushShadows);
1510	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1511	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1512	#endif
1513
1514
1515	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1516	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1517	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1518	bool fPreferVolatile = true);
1519	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1520	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1521	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1522	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1523	bool fNoVolatileRegs = false);
1524	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1525	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1526	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1527	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1528	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1529	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1530	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1531	#endif
1532
1533	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1534	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1535	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1536	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1537	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1538	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1539	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1540	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1541	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1542	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1543	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1544	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1545	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1546	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1547	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1548	uint32_t fHstRegsNotToSave);
1549	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1550	uint32_t fHstRegsNotToSave);
1551	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1552	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1553
1554	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1555	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1556	#ifdef VBOX_STRICT
1557	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1558	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1559	IEMNATIVEGSTREG enmGstReg);
1560	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1561	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1562	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1563	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1564	# endif
1565	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1566	#endif
1567	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1568	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1569	#endif
1570	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1571	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1572	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1573	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1574	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1575	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1576	PCIEMTHRDEDCALLENTRY pCallEntry);
1577	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1578	uint8_t idxAddrReg, uint8_t idxInstr);
1579	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1580	uint8_t idxAddrReg, uint8_t idxInstr);
1581
1582
1583	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1584	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1585	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1586	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1587	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1588	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1589	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1590	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1591	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1592
1593	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1594	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1595	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1596	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1597	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1598	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1599	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1600	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1601	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1602	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1603	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1604	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1605	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1606	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1607	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1608	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1609	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1610	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1611	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1612	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1613	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1614
1615	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1616	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1617	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1618	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1619	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1620	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1621	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1622	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1623	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1624	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1625	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1626	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1627	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1628	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1629	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1630	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1631	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1632	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1633	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1634	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1635	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1636
1637	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1638	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1639	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1640	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1641	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1642	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1643	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1644	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1645	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1646	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1647	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1648	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1649	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1650	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1651	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1652	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1653	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1654	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1655	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1656	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1657	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1658	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1659
1660	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1661	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1662	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1663	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1664	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1665	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1666	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1667	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1668	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1669	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1670	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1671	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1672	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1673	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1674	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1675	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1676	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1677	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1678	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1679	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1680	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1681	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1682
1683	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1684	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1685	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1686	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1687
1688
1689	/**
1690	* Info about shadowed guest register values.
1691	* @see IEMNATIVEGSTREG
1692	*/
1693	typedef struct IEMANTIVEGSTREGINFO
1694	{
1695	/** Offset in VMCPU. */
1696	uint32_t off;
1697	/** The field size. */
1698	uint8_t cb;
1699	/** Name (for logging). */
1700	const char *pszName;
1701	} IEMANTIVEGSTREGINFO;
1702	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1703	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1704	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1705	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1706	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1707
1708
1709
1710	/**
1711	* Ensures that there is sufficient space in the instruction output buffer.
1712	*
1713	* This will reallocate the buffer if needed and allowed.
1714	*
1715	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1716	* allocation size.
1717	*
1718	* @returns Pointer to the instruction output buffer on success; throws VBox
1719	* status code on failure, so no need to check it.
1720	* @param pReNative The native recompile state.
1721	* @param off Current instruction offset. Works safely for UINT32_MAX
1722	* as well.
1723	* @param cInstrReq Number of instruction about to be added. It's okay to
1724	* overestimate this a bit.
1725	*/
1726	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1727	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1728	{
1729	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1730	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1731	{
1732	#ifdef VBOX_STRICT
1733	pReNative->offInstrBufChecked = offChecked;
1734	#endif
1735	return pReNative->pInstrBuf;
1736	}
1737	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1738	}
1739
1740	/**
1741	* Checks that we didn't exceed the space requested in the last
1742	* iemNativeInstrBufEnsure() call.
1743	*/
1744	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1745	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1746	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1747
1748	/**
1749	* Checks that a variable index is valid.
1750	*/
1751	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1752	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1753	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1754	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1755	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1756	("%s=%#x\n", #a_idxVar, a_idxVar))
1757	#else
1758	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1759	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1760	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1761	#endif
1762
1763	/**
1764	* Checks that a variable index is valid and that the variable is assigned the
1765	* correct argument number.
1766	* This also adds a RT_NOREF of a_idxVar.
1767	*/
1768	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1769	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1770	RT_NOREF_PV(a_idxVar); \
1771	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1772	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1773	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1774	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1775	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1776	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1777	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1778	a_uArgNo)); \
1779	} while (0)
1780	#else
1781	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1782	RT_NOREF_PV(a_idxVar); \
1783	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1784	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1785	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1786	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1787	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1788	} while (0)
1789	#endif
1790
1791
1792	/**
1793	* Checks that a variable has the expected size.
1794	*/
1795	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1796	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1797	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1798	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1799
1800
1801	/**
1802	* Calculates the stack address of a variable as a [r]BP displacement value.
1803	*/
1804	DECL_FORCE_INLINE(int32_t)
1805	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1806	{
1807	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1808	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1809	}
1810
1811
1812	/**
1813	* Releases the variable's register.
1814	*
1815	* The register must have been previously acquired calling
1816	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1817	* iemNativeVarRegisterSetAndAcquire().
1818	*/
1819	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1820	{
1821	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1822	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1823	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1824	}
1825
1826
1827	/**
1828	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1829	*
1830	* @returns The flush mask.
1831	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1832	* @param fGstShwFlush The starting flush mask.
1833	*/
1834	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1835	{
1836	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1837	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1838	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1839	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1840	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1841	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1842	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1843	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1844	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1845	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1846	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1847	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1848	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1849	return fGstShwFlush;
1850	}
1851
1852
1853	/** Number of hidden arguments for CIMPL calls.
1854	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1855	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1856	# define IEM_CIMPL_HIDDEN_ARGS 3
1857	#else
1858	# define IEM_CIMPL_HIDDEN_ARGS 2
1859	#endif
1860
1861
1862	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1863
1864	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1865	/**
1866	* Helper for iemNativeLivenessGetStateByGstReg.
1867	*
1868	* @returns IEMLIVENESS_STATE_XXX
1869	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1870	* ORed together.
1871	*/
1872	DECL_FORCE_INLINE(uint32_t)
1873	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1874	{
1875	/* INPUT trumps anything else. */
1876	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1877	return IEMLIVENESS_STATE_INPUT;
1878
1879	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1880	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1881	{
1882	/* If not all sub-fields are clobbered they must be considered INPUT. */
1883	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1884	return IEMLIVENESS_STATE_INPUT;
1885	return IEMLIVENESS_STATE_CLOBBERED;
1886	}
1887
1888	/* XCPT_OR_CALL trumps UNUSED. */
1889	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1890	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1891
1892	return IEMLIVENESS_STATE_UNUSED;
1893	}
1894	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1895
1896
1897	DECL_FORCE_INLINE(uint32_t)
1898	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1899	{
1900	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1901	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1902	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1903	# else
1904	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1905	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1906	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1907	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1908	# endif
1909	}
1910
1911
1912	DECL_FORCE_INLINE(uint32_t)
1913	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
1914	{
1915	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
1916	if (enmGstReg == kIemNativeGstReg_EFlags)
1917	{
1918	/* Merge the eflags states to one. */
1919	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1920	uRet = RT_BIT_32(uRet);
1921	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
1922	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
1923	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
1924	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
1925	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
1926	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
1927	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
1928	# else
1929	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
1930	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
1931	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
1932	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
1933	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
1934	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
1935	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
1936	# endif
1937	}
1938	return uRet;
1939	}
1940
1941
1942	# ifdef VBOX_STRICT
1943	/** For assertions only, user checks that idxCurCall isn't zerow. */
1944	DECL_FORCE_INLINE(uint32_t)
1945	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
1946	{
1947	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
1948	}
1949	# endif /* VBOX_STRICT */
1950
1951	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
1952
1953
1954	/**
1955	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
1956	*/
1957	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
1958	{
1959	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
1960	return IEM_CIMPL_HIDDEN_ARGS;
1961	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
1962	return 1;
1963	return 0;
1964	}
1965
1966
1967	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
1968	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
1969	{
1970	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
1971
1972	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
1973	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
1974	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
1975	return (uint8_t)idxReg;
1976	}
1977
1978
1979
1980	/*********************************************************************************************************************************
1981	* Register Allocator (GPR) *
1982	*********************************************************************************************************************************/
1983
1984	/**
1985	* Marks host register @a idxHstReg as containing a shadow copy of guest
1986	* register @a enmGstReg.
1987	*
1988	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
1989	* host register before calling.
1990	*/
1991	DECL_FORCE_INLINE(void)
1992	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
1993	{
1994	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
1995	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
1996	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
1997
1998	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
1999	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2000	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2001	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2002	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2003	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2004	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2005	#else
2006	RT_NOREF(off);
2007	#endif
2008	}
2009
2010
2011	/**
2012	* Clear any guest register shadow claims from @a idxHstReg.
2013	*
2014	* The register does not need to be shadowing any guest registers.
2015	*/
2016	DECL_FORCE_INLINE(void)
2017	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2018	{
2019	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2020	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2021	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2022	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2023	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2024
2025	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2026	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2027	if (fGstRegs)
2028	{
2029	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2030	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2031	while (fGstRegs)
2032	{
2033	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2034	fGstRegs &= ~RT_BIT_64(iGstReg);
2035	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2036	}
2037	}
2038	#else
2039	RT_NOREF(off);
2040	#endif
2041
2042	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2043	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2044	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2045	}
2046
2047
2048	/**
2049	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2050	* and global overview flags.
2051	*/
2052	DECL_FORCE_INLINE(void)
2053	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2054	{
2055	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2056	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2057	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2058	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2059	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2060	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2061	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2062
2063	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2064	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2065	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2066	#else
2067	RT_NOREF(off);
2068	#endif
2069
2070	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2071	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2072	if (!fGstRegShadowsNew)
2073	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2074	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2075	}
2076
2077
2078	#if 0 /* unused */
2079	/**
2080	* Clear any guest register shadow claim for @a enmGstReg.
2081	*/
2082	DECL_FORCE_INLINE(void)
2083	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2084	{
2085	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2086	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2087	{
2088	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2089	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2090	}
2091	}
2092	#endif
2093
2094
2095	/**
2096	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2097	* as the new shadow of it.
2098	*
2099	* Unlike the other guest reg shadow helpers, this does the logging for you.
2100	* However, it is the liveness state is not asserted here, the caller must do
2101	* that.
2102	*/
2103	DECL_FORCE_INLINE(void)
2104	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2105	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2106	{
2107	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2108	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2109	{
2110	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2111	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2112	if (idxHstRegOld == idxHstRegNew)
2113	return;
2114	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2115	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2116	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2117	}
2118	else
2119	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2120	g_aGstShadowInfo[enmGstReg].pszName));
2121	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2122	}
2123
2124
2125	/**
2126	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2127	* to @a idxRegTo.
2128	*/
2129	DECL_FORCE_INLINE(void)
2130	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2131	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2132	{
2133	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2134	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2135	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2136	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2137	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2138	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2139	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2140	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2141	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2142
2143	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2144	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2145	if (!fGstRegShadowsFrom)
2146	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2147	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2148	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2149	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2150	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2151	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2152	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2153	#else
2154	RT_NOREF(off);
2155	#endif
2156	}
2157
2158
2159	/**
2160	* Flushes any delayed guest register writes.
2161	*
2162	* This must be called prior to calling CImpl functions and any helpers that use
2163	* the guest state (like raising exceptions) and such.
2164	*
2165	* This optimization has not yet been implemented. The first target would be
2166	* RIP updates, since these are the most common ones.
2167	*/
2168	DECL_INLINE_THROW(uint32_t)
2169	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2170	{
2171	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2172	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2173	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2174	#else
2175	RT_NOREF(pReNative, fGstShwExcept);
2176	#endif
2177
2178	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2179	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2180	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2181	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2182	#else
2183	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2184	return off;
2185	#endif
2186	}
2187
2188
2189
2190	/*********************************************************************************************************************************
2191	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2192	*********************************************************************************************************************************/
2193
2194	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2195
2196	DECL_FORCE_INLINE(uint8_t)
2197	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2198	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2199	{
2200	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2201
2202	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2203	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2204	RT_NOREF(idxVar);
2205	return idxSimdReg;
2206	}
2207
2208
2209	/**
2210	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2211	* SIMD register @a enmGstSimdReg.
2212	*
2213	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2214	* host register before calling.
2215	*/
2216	DECL_FORCE_INLINE(void)
2217	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2218	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2219	{
2220	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2221	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2222	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2223
2224	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2225	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2226	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2227	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2228	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2229	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2230	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2231	#else
2232	RT_NOREF(off);
2233	#endif
2234	}
2235
2236
2237	/**
2238	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2239	* to @a idxSimdRegTo.
2240	*/
2241	DECL_FORCE_INLINE(void)
2242	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2243	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2244	{
2245	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2246	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2247	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2248	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2249	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2250	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2251	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2252	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2253	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2254	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2255	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2256
2257	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2258	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2259	if (!fGstRegShadowsFrom)
2260	{
2261	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2262	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2263	}
2264	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2265	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2266	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2267	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2268	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2269	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2270	#else
2271	RT_NOREF(off);
2272	#endif
2273	}
2274
2275
2276	/**
2277	* Clear any guest register shadow claims from @a idxHstSimdReg.
2278	*
2279	* The register does not need to be shadowing any guest registers.
2280	*/
2281	DECL_FORCE_INLINE(void)
2282	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2283	{
2284	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2285	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2286	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2287	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2288	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2289	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2290	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2291
2292	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2293	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2294	if (fGstRegs)
2295	{
2296	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2297	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2298	while (fGstRegs)
2299	{
2300	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2301	fGstRegs &= ~RT_BIT_64(iGstReg);
2302	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2303	}
2304	}
2305	#else
2306	RT_NOREF(off);
2307	#endif
2308
2309	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2310	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2311	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2312	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2313	}
2314
2315	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2316
2317
2318	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2319	/**
2320	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2321	*/
2322	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2323	{
2324	if (pReNative->Core.offPc)
2325	return iemNativeEmitPcWritebackSlow(pReNative, off);
2326	return off;
2327	}
2328	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2329
2330
2331	/** @} */
2332
2333	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2334

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103838

Download in other formats: