IEMN8veRecompiler.h@ 103839

Last change on this file since 103839 was 103839, checked in by vboxsync, 13 months ago
VMM/IEM: Implement native emitters for IEM_MC_RAISE_DIVIDE_ERROR() and IEM_MC_IF_LOCAL_IS_Z(), bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 113.6 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103839 2024-03-14 09:05:23Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
54	* Enables skipping EFLAGS calculations/updating based on liveness info. */
55	#if (defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && 1) \|\| defined(DOXYGEN_RUNNING)
56	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
57	#endif
58
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#if (defined(VBOX_STRICT) && defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)) \|\| defined(DOXYGEN_RUNNING)
64	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
65	#endif
66
67	#ifdef VBOX_WITH_STATISTICS
68	/** Always count instructions for now. */
69	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
70	#endif
71
72
73	/** @name Stack Frame Layout
74	*
75	* @{ */
76	/** The size of the area for stack variables and spills and stuff.
77	* @note This limit is duplicated in the python script(s). We add 0x40 for
78	* alignment padding. */
79	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
80	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
81	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
82	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
83
84	#ifdef RT_ARCH_AMD64
85	/** An stack alignment adjustment (between non-volatile register pushes and
86	* the stack variable area, so the latter better aligned). */
87	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
88
89	/** Number of stack arguments slots for calls made from the frame. */
90	# ifdef RT_OS_WINDOWS
91	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
92	# else
93	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
94	# endif
95	/** Number of any shadow arguments (spill area) for calls we make. */
96	# ifdef RT_OS_WINDOWS
97	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
98	# else
99	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
100	# endif
101
102	/** Frame pointer (RBP) relative offset of the last push. */
103	# ifdef RT_OS_WINDOWS
104	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
105	# else
106	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
107	# endif
108	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
109	* address for it). */
110	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
111	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
112	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
113	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
114	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
115	# ifdef RT_OS_WINDOWS
116	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
117	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
118	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
119	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
120	# endif
121
122	# ifdef RT_OS_WINDOWS
123	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
124	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
125	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
126	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
127	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
128	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
129	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
130	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
131	# endif
132
133	#elif RT_ARCH_ARM64
134	/** No alignment padding needed for arm64. */
135	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
136	/** No stack argument slots, got 8 registers for arguments will suffice. */
137	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
138	/** There are no argument spill area. */
139	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
140
141	/** Number of saved registers at the top of our stack frame.
142	* This includes the return address and old frame pointer, so x19 thru x30. */
143	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
144	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
145	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
146
147	/** Frame pointer (BP) relative offset of the last push. */
148	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
149
150	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
151	* address for it). */
152	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
153
154	#else
155	# error "port me"
156	#endif
157	/** @} */
158
159
160	/** @name Fixed Register Allocation(s)
161	* @{ */
162	/** @def IEMNATIVE_REG_FIXED_PVMCPU
163	* The number of the register holding the pVCpu pointer. */
164	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
165	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
166	* @note This not available on AMD64, only ARM64. */
167	/** @def IEMNATIVE_REG_FIXED_TMP0
168	* Dedicated temporary register.
169	* @todo replace this by a register allocator and content tracker. */
170	/** @def IEMNATIVE_REG_FIXED_MASK
171	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
172	* architecture. */
173	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
174	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
175	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
176	* architecture. */
177	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
178	* Dedicated temporary SIMD register. */
179	#endif
180	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
181	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
182	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
183	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
184	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
185	\| RT_BIT_32(X86_GREG_xSP) \
186	\| RT_BIT_32(X86_GREG_xBP) )
187
188	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
189	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
190	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
191	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0)
192	# else
193	/** On Windows xmm6 through xmm15 are marked as callee saved. */
194	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
195	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
196	# endif
197	# endif
198
199	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
200	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
226	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
227	\| RT_BIT_32(ARMV8_A64_REG_Q30))
228	# endif
229	# endif
230
231	#else
232	# error "port me"
233	#endif
234	/** @} */
235
236	/** @name Call related registers.
237	* @{ */
238	/** @def IEMNATIVE_CALL_RET_GREG
239	* The return value register. */
240	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
241	* Number of arguments in registers. */
242	/** @def IEMNATIVE_CALL_ARG0_GREG
243	* The general purpose register carrying argument \#0. */
244	/** @def IEMNATIVE_CALL_ARG1_GREG
245	* The general purpose register carrying argument \#1. */
246	/** @def IEMNATIVE_CALL_ARG2_GREG
247	* The general purpose register carrying argument \#2. */
248	/** @def IEMNATIVE_CALL_ARG3_GREG
249	* The general purpose register carrying argument \#3. */
250	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
251	* Mask of registers the callee will not save and may trash. */
252	#ifdef RT_ARCH_AMD64
253	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
254
255	# ifdef RT_OS_WINDOWS
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
257	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
258	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
259	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
260	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
261	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
262	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
263	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
264	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
265	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
266	\| RT_BIT_32(X86_GREG_xCX) \
267	\| RT_BIT_32(X86_GREG_xDX) \
268	\| RT_BIT_32(X86_GREG_x8) \
269	\| RT_BIT_32(X86_GREG_x9) \
270	\| RT_BIT_32(X86_GREG_x10) \
271	\| RT_BIT_32(X86_GREG_x11) )
272	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
273	/* xmm0 - xmm5 are marked as volatile. */
274	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
275	# endif
276
277	# else
278	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
279	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
280	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
281	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
282	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
283	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
284	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
285	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
286	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
287	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
288	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
289	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
290	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
291	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
292	\| RT_BIT_32(X86_GREG_xCX) \
293	\| RT_BIT_32(X86_GREG_xDX) \
294	\| RT_BIT_32(X86_GREG_xDI) \
295	\| RT_BIT_32(X86_GREG_xSI) \
296	\| RT_BIT_32(X86_GREG_x8) \
297	\| RT_BIT_32(X86_GREG_x9) \
298	\| RT_BIT_32(X86_GREG_x10) \
299	\| RT_BIT_32(X86_GREG_x11) )
300	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
301	/* xmm0 - xmm15 are marked as volatile. */
302	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
303	# endif
304	# endif
305
306	#elif defined(RT_ARCH_ARM64)
307	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
308	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
309	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
310	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
311	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
312	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
313	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
314	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
315	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
316	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
317	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
318	\| RT_BIT_32(ARMV8_A64_REG_X1) \
319	\| RT_BIT_32(ARMV8_A64_REG_X2) \
320	\| RT_BIT_32(ARMV8_A64_REG_X3) \
321	\| RT_BIT_32(ARMV8_A64_REG_X4) \
322	\| RT_BIT_32(ARMV8_A64_REG_X5) \
323	\| RT_BIT_32(ARMV8_A64_REG_X6) \
324	\| RT_BIT_32(ARMV8_A64_REG_X7) )
325	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
326	\| RT_BIT_32(ARMV8_A64_REG_X1) \
327	\| RT_BIT_32(ARMV8_A64_REG_X2) \
328	\| RT_BIT_32(ARMV8_A64_REG_X3) \
329	\| RT_BIT_32(ARMV8_A64_REG_X4) \
330	\| RT_BIT_32(ARMV8_A64_REG_X5) \
331	\| RT_BIT_32(ARMV8_A64_REG_X6) \
332	\| RT_BIT_32(ARMV8_A64_REG_X7) \
333	\| RT_BIT_32(ARMV8_A64_REG_X8) \
334	\| RT_BIT_32(ARMV8_A64_REG_X9) \
335	\| RT_BIT_32(ARMV8_A64_REG_X10) \
336	\| RT_BIT_32(ARMV8_A64_REG_X11) \
337	\| RT_BIT_32(ARMV8_A64_REG_X12) \
338	\| RT_BIT_32(ARMV8_A64_REG_X13) \
339	\| RT_BIT_32(ARMV8_A64_REG_X14) \
340	\| RT_BIT_32(ARMV8_A64_REG_X15) \
341	\| RT_BIT_32(ARMV8_A64_REG_X16) \
342	\| RT_BIT_32(ARMV8_A64_REG_X17) )
343	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
344	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
345	* so to simplify our life a bit we just mark everything as volatile. */
346	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
347	# endif
348
349	#endif
350
351	/** This is the maximum argument count we'll ever be needing. */
352	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
353	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
354	#else
355	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
356	#endif
357	/** @} */
358
359
360	/** @def IEMNATIVE_HST_GREG_COUNT
361	* Number of host general purpose registers we tracker. */
362	/** @def IEMNATIVE_HST_GREG_MASK
363	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
364	* inverted register masks and such to get down to a correct set of regs. */
365	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
366	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
367	* Number of host SIMD registers we track. */
368	/** @def IEMNATIVE_HST_SIMD_REG_MASK
369	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
370	* inverted register masks and such to get down to a correct set of regs. */
371	#endif
372	#ifdef RT_ARCH_AMD64
373	# define IEMNATIVE_HST_GREG_COUNT 16
374	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
375
376	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
377	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
378	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
379	# endif
380
381	#elif defined(RT_ARCH_ARM64)
382	# define IEMNATIVE_HST_GREG_COUNT 32
383	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
384
385	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
386	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
387	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
388	# endif
389
390	#else
391	# error "Port me!"
392	#endif
393
394
395	/** Native code generator label types. */
396	typedef enum
397	{
398	kIemNativeLabelType_Invalid = 0,
399	/* Labels w/o data, only once instance per TB: */
400	kIemNativeLabelType_Return,
401	kIemNativeLabelType_ReturnBreak,
402	kIemNativeLabelType_ReturnWithFlags,
403	kIemNativeLabelType_NonZeroRetOrPassUp,
404	kIemNativeLabelType_RaiseGp0,
405	kIemNativeLabelType_RaiseNm,
406	kIemNativeLabelType_RaiseUd,
407	kIemNativeLabelType_RaiseMf,
408	kIemNativeLabelType_RaiseXf,
409	kIemNativeLabelType_RaiseDe,
410	kIemNativeLabelType_ObsoleteTb,
411	kIemNativeLabelType_NeedCsLimChecking,
412	kIemNativeLabelType_CheckBranchMiss,
413	/* Labels with data, potentially multiple instances per TB: */
414	kIemNativeLabelType_FirstWithMultipleInstances,
415	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
416	kIemNativeLabelType_Else,
417	kIemNativeLabelType_Endif,
418	kIemNativeLabelType_CheckIrq,
419	kIemNativeLabelType_TlbLookup,
420	kIemNativeLabelType_TlbMiss,
421	kIemNativeLabelType_TlbDone,
422	kIemNativeLabelType_End
423	} IEMNATIVELABELTYPE;
424
425	/** Native code generator label definition. */
426	typedef struct IEMNATIVELABEL
427	{
428	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
429	* the epilog. */
430	uint32_t off;
431	/** The type of label (IEMNATIVELABELTYPE). */
432	uint16_t enmType;
433	/** Additional label data, type specific. */
434	uint16_t uData;
435	} IEMNATIVELABEL;
436	/** Pointer to a label. */
437	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
438
439
440	/** Native code generator fixup types. */
441	typedef enum
442	{
443	kIemNativeFixupType_Invalid = 0,
444	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
445	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
446	kIemNativeFixupType_Rel32,
447	#elif defined(RT_ARCH_ARM64)
448	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
449	kIemNativeFixupType_RelImm26At0,
450	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
451	kIemNativeFixupType_RelImm19At5,
452	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
453	kIemNativeFixupType_RelImm14At5,
454	#endif
455	kIemNativeFixupType_End
456	} IEMNATIVEFIXUPTYPE;
457
458	/** Native code generator fixup. */
459	typedef struct IEMNATIVEFIXUP
460	{
461	/** Code offset of the fixup location. */
462	uint32_t off;
463	/** The IEMNATIVELABEL this is a fixup for. */
464	uint16_t idxLabel;
465	/** The fixup type (IEMNATIVEFIXUPTYPE). */
466	uint8_t enmType;
467	/** Addend or other data. */
468	int8_t offAddend;
469	} IEMNATIVEFIXUP;
470	/** Pointer to a native code generator fixup. */
471	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
472
473
474	/**
475	* One bit of the state.
476	*
477	* Each register state takes up two bits. We keep the two bits in two separate
478	* 64-bit words to simplify applying them to the guest shadow register mask in
479	* the register allocator.
480	*/
481	typedef union IEMLIVENESSBIT
482	{
483	uint64_t bm64;
484	RT_GCC_EXTENSION struct
485	{ /* bit no */
486	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
487	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
488	uint64_t fCr0 : 1; /*< 0x11 / 17: /
489	uint64_t fFcw : 1; /*< 0x12 / 18: /
490	uint64_t fFsw : 1; /*< 0x13 / 19: /
491	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
492	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
493	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
494	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
495	uint64_t fCr4 : 1; /*< 0x2c / 44: /
496	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
497	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
498	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
499	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
500	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
501	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
502	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
503	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
504	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
505	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
506	};
507	} IEMLIVENESSBIT;
508	AssertCompileSize(IEMLIVENESSBIT, 8);
509
510	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
511	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
512	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
513	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
514	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
515	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
516	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
517
518
519	/**
520	* A liveness state entry.
521	*
522	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
523	* Once we add a SSE register shadowing, we'll add another 64-bit element for
524	* that.
525	*/
526	typedef union IEMLIVENESSENTRY
527	{
528	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
529	uint64_t bm64[16 / 8];
530	uint16_t bm32[16 / 4];
531	uint16_t bm16[16 / 2];
532	uint8_t bm8[ 16 / 1];
533	IEMLIVENESSBIT aBits[2];
534	#else
535	uint64_t bm64[32 / 8];
536	uint16_t bm32[32 / 4];
537	uint16_t bm16[32 / 2];
538	uint8_t bm8[ 32 / 1];
539	IEMLIVENESSBIT aBits[4];
540	#endif
541	RT_GCC_EXTENSION struct
542	{
543	/** Bit \#0 of the register states. */
544	IEMLIVENESSBIT Bit0;
545	/** Bit \#1 of the register states. */
546	IEMLIVENESSBIT Bit1;
547	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
548	/** Bit \#2 of the register states. */
549	IEMLIVENESSBIT Bit2;
550	/** Bit \#3 of the register states. */
551	IEMLIVENESSBIT Bit3;
552	#endif
553	};
554	} IEMLIVENESSENTRY;
555	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
556	AssertCompileSize(IEMLIVENESSENTRY, 16);
557	#else
558	AssertCompileSize(IEMLIVENESSENTRY, 32);
559	#endif
560	/** Pointer to a liveness state entry. */
561	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
562	/** Pointer to a const liveness state entry. */
563	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
564
565	/** @name 64-bit value masks for IEMLIVENESSENTRY.
566	* @{ / / 0xzzzzyyyyxxxxwwww */
567	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
568
569	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
570	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
571	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
572
573	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
574	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
575	#endif
576
577	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
578	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
579
580	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
581	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
582	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
583	#endif
584	/** @} */
585
586
587	/** @name The liveness state for a register.
588	*
589	* The state values have been picked to with state accumulation in mind (what
590	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
591	* performance critical work done with the values.
592	*
593	* This is a compressed state that only requires 2 bits per register.
594	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
595	* 1. the incoming state from the following call,
596	* 2. the outgoing state for this call,
597	* 3. mask of the entries set in the 2nd.
598	*
599	* The mask entry (3rd one above) will be used both when updating the outgoing
600	* state and when merging in incoming state for registers not touched by the
601	* current call.
602	*
603	* @{ */
604	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
605	/** The register will be clobbered and the current value thrown away.
606	*
607	* When this is applied to the state (2) we'll simply be AND'ing it with the
608	* (old) mask (3) and adding the register to the mask. This way we'll
609	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
610	* IEMLIVENESS_STATE_INPUT states. */
611	# define IEMLIVENESS_STATE_CLOBBERED 0
612	/** The register is unused in the remainder of the TB.
613	*
614	* This is an initial state and can not be set by any of the
615	* iemNativeLivenessFunc_xxxx callbacks. */
616	# define IEMLIVENESS_STATE_UNUSED 1
617	/** The register value is required in a potential call or exception.
618	*
619	* This means that the register value must be calculated and is best written to
620	* the state, but that any shadowing registers can be flushed thereafter as it's
621	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
622	*
623	* It is typically applied across the board, but we preserve incoming
624	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
625	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
626	* 1. r0 = old & ~mask;
627	* 2. r0 = t1 & (t1 >> 1)'
628	* 3. state \|= r0 \| 0b10;
629	* 4. mask = ~0;
630	*/
631	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
632	/** The register value is used as input.
633	*
634	* This means that the register value must be calculated and it is best to keep
635	* it in a register. It does not need to be writtent out as such. This is the
636	* highest priority state.
637	*
638	* Whether the call modifies the register or not isn't relevant to earlier
639	* calls, so that's not recorded.
640	*
641	* When applying this state we just or in the value in the outgoing state and
642	* mask. */
643	# define IEMLIVENESS_STATE_INPUT 3
644	/** Mask of the state bits. */
645	# define IEMLIVENESS_STATE_MASK 3
646	/** The number of bits per state. */
647	# define IEMLIVENESS_STATE_BIT_COUNT 2
648	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
649	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
650	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
651	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
652	/** Check if a register clobbering is expected given the (previous) liveness state.
653	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
654	* include INPUT if the register is used in more than one place. */
655	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
656
657	/** Check if all status flags are going to be clobbered and doesn't need
658	* calculating in the current step.
659	* @param a_pEntry The current liveness entry. */
660	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
661	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
662
663	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
664	/** The register is not used any more. */
665	# define IEMLIVENESS_STATE_UNUSED 0
666	/** Flag: The register is required in a potential exception or call. */
667	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
668	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
669	/** Flag: The register is read. */
670	# define IEMLIVENESS_STATE_READ 2
671	# define IEMLIVENESS_BIT_READ 1
672	/** Flag: The register is written. */
673	# define IEMLIVENESS_STATE_WRITE 4
674	# define IEMLIVENESS_BIT_WRITE 2
675	/** Flag: Unconditional call (not needed, can be redefined for research). */
676	# define IEMLIVENESS_STATE_CALL 8
677	# define IEMLIVENESS_BIT_CALL 3
678	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
679	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
680	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
681	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
682	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
683
684	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
685	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
686	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
687	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
688
689	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
690	/** @} */
691
692	/** @name Liveness helpers for builtin functions and similar.
693	*
694	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
695	* own set of manimulator macros for those.
696	*
697	* @{ */
698	/** Initializing the state as all unused. */
699	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
700	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
701	do { \
702	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
703	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
704	} while (0)
705	#else
706	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
707	do { \
708	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
709	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
710	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
711	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
712	} while (0)
713	#endif
714
715	/** Initializing the outgoing state with a potential xcpt or call state.
716	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
717	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
718	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
719	do { \
720	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
721	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
722	} while (0)
723	#else
724	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
725	do { \
726	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
727	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
728	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
729	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
730	} while (0)
731	#endif
732
733	/** Adds a segment base register as input to the outgoing state. */
734	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
735	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
736	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
737	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
738	} while (0)
739	#else
740	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
741	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
742	} while (0)
743	#endif
744
745	/** Adds a segment attribute register as input to the outgoing state. */
746	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
747	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
748	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
749	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
750	} while (0)
751	#else
752	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
753	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
754	} while (0)
755	#endif
756
757	/** Adds a segment limit register as input to the outgoing state. */
758	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
759	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
760	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
761	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
762	} while (0)
763	#else
764	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
765	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
766	} while (0)
767	#endif
768
769	/** Adds a segment limit register as input to the outgoing state. */
770	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
771	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
772	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
773	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
774	} while (0)
775	#else
776	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
777	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
778	} while (0)
779	#endif
780	/** @} */
781
782	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
783	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
784	* calculated and up to date. This is to double check that we haven't skipped
785	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
786	* @note has to be placed in
787	*/
788	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
789	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
790	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
791	#else
792	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
793	#endif
794
795
796	/**
797	* Guest registers that can be shadowed in GPRs.
798	*
799	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
800	* must be placed last, as the liveness state tracks it as 7 subcomponents and
801	* we don't want to waste space here.
802	*
803	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
804	* friends as well as IEMAllN8veLiveness.cpp.
805	*/
806	typedef enum IEMNATIVEGSTREG : uint8_t
807	{
808	kIemNativeGstReg_GprFirst = 0,
809	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
810	kIemNativeGstReg_Pc,
811	kIemNativeGstReg_Cr0,
812	kIemNativeGstReg_FpuFcw,
813	kIemNativeGstReg_FpuFsw,
814	kIemNativeGstReg_SegBaseFirst,
815	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
816	kIemNativeGstReg_SegAttribFirst,
817	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
818	kIemNativeGstReg_SegLimitFirst,
819	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
820	kIemNativeGstReg_SegSelFirst,
821	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
822	kIemNativeGstReg_Cr4,
823	kIemNativeGstReg_Xcr0,
824	kIemNativeGstReg_MxCsr,
825	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
826	kIemNativeGstReg_End
827	} IEMNATIVEGSTREG;
828	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
829	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
830
831	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
832	* @{ */
833	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
834	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
835	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
836	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
837	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
838	/** @} */
839
840	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
841
842	/**
843	* Guest registers that can be shadowed in host SIMD registers.
844	*
845	* @todo r=aeichner Liveness tracking
846	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
847	*/
848	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
849	{
850	kIemNativeGstSimdReg_SimdRegFirst = 0,
851	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
852	kIemNativeGstSimdReg_End
853	} IEMNATIVEGSTSIMDREG;
854
855	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
856	* @{ */
857	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
858	/** @} */
859
860	/**
861	* The Load/store size for a SIMD guest register.
862	*/
863	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
864	{
865	/** Invalid size. */
866	kIemNativeGstSimdRegLdStSz_Invalid = 0,
867	/** Loads the low 128-bit of a guest SIMD register. */
868	kIemNativeGstSimdRegLdStSz_Low128,
869	/** Loads the high 128-bit of a guest SIMD register. */
870	kIemNativeGstSimdRegLdStSz_High128,
871	/** Loads the whole 256-bits of a guest SIMD register. */
872	kIemNativeGstSimdRegLdStSz_256,
873	/** End value. */
874	kIemNativeGstSimdRegLdStSz_End
875	} IEMNATIVEGSTSIMDREGLDSTSZ;
876
877	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
878
879	/**
880	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
881	*/
882	typedef enum IEMNATIVEGSTREGUSE
883	{
884	/** The usage is read-only, the register holding the guest register
885	* shadow copy will not be modified by the caller. */
886	kIemNativeGstRegUse_ReadOnly = 0,
887	/** The caller will update the guest register (think: PC += cbInstr).
888	* The guest shadow copy will follow the returned register. */
889	kIemNativeGstRegUse_ForUpdate,
890	/** The call will put an entirely new value in the guest register, so
891	* if new register is allocate it will be returned uninitialized. */
892	kIemNativeGstRegUse_ForFullWrite,
893	/** The caller will use the guest register value as input in a calculation
894	* and the host register will be modified.
895	* This means that the returned host register will not be marked as a shadow
896	* copy of the guest register. */
897	kIemNativeGstRegUse_Calculation
898	} IEMNATIVEGSTREGUSE;
899
900	/**
901	* Guest registers (classes) that can be referenced.
902	*/
903	typedef enum IEMNATIVEGSTREGREF : uint8_t
904	{
905	kIemNativeGstRegRef_Invalid = 0,
906	kIemNativeGstRegRef_Gpr,
907	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
908	kIemNativeGstRegRef_EFlags,
909	kIemNativeGstRegRef_MxCsr,
910	kIemNativeGstRegRef_FpuReg,
911	kIemNativeGstRegRef_MReg,
912	kIemNativeGstRegRef_XReg,
913	//kIemNativeGstRegRef_YReg, - doesn't work.
914	kIemNativeGstRegRef_End
915	} IEMNATIVEGSTREGREF;
916
917
918	/** Variable kinds. */
919	typedef enum IEMNATIVEVARKIND : uint8_t
920	{
921	/** Customary invalid zero value. */
922	kIemNativeVarKind_Invalid = 0,
923	/** This is either in a register or on the stack. */
924	kIemNativeVarKind_Stack,
925	/** Immediate value - loaded into register when needed, or can live on the
926	* stack if referenced (in theory). */
927	kIemNativeVarKind_Immediate,
928	/** Variable reference - loaded into register when needed, never stack. */
929	kIemNativeVarKind_VarRef,
930	/** Guest register reference - loaded into register when needed, never stack. */
931	kIemNativeVarKind_GstRegRef,
932	/** End of valid values. */
933	kIemNativeVarKind_End
934	} IEMNATIVEVARKIND;
935
936
937	/** Variable or argument. */
938	typedef struct IEMNATIVEVAR
939	{
940	/** The kind of variable. */
941	IEMNATIVEVARKIND enmKind;
942	/** The variable size in bytes. */
943	uint8_t cbVar;
944	/** The first stack slot (uint64_t), except for immediate and references
945	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
946	* has a stack slot it has been initialized and has a value. Unused variables
947	* has neither a stack slot nor a host register assignment. */
948	uint8_t idxStackSlot;
949	/** The host register allocated for the variable, UINT8_MAX if not. */
950	uint8_t idxReg;
951	/** The argument number if argument, UINT8_MAX if regular variable. */
952	uint8_t uArgNo;
953	/** If referenced, the index (unpacked) of the variable referencing this one,
954	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
955	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
956	uint8_t idxReferrerVar;
957	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
958	* @todo not sure what this really is for... */
959	IEMNATIVEGSTREG enmGstReg;
960	/** Set if the registered is currently used exclusively, false if the
961	* variable is idle and the register can be grabbed. */
962	bool fRegAcquired;
963
964	union
965	{
966	/** kIemNativeVarKind_Immediate: The immediate value. */
967	uint64_t uValue;
968	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
969	uint8_t idxRefVar;
970	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
971	struct
972	{
973	/** The class of register. */
974	IEMNATIVEGSTREGREF enmClass;
975	/** Index within the class. */
976	uint8_t idx;
977	} GstRegRef;
978	} u;
979	} IEMNATIVEVAR;
980	/** Pointer to a variable or argument. */
981	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
982	/** Pointer to a const variable or argument. */
983	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
984
985	/** What is being kept in a host register. */
986	typedef enum IEMNATIVEWHAT : uint8_t
987	{
988	/** The traditional invalid zero value. */
989	kIemNativeWhat_Invalid = 0,
990	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
991	kIemNativeWhat_Var,
992	/** Temporary register, this is typically freed when a MC completes. */
993	kIemNativeWhat_Tmp,
994	/** Call argument w/o a variable mapping. This is free (via
995	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
996	kIemNativeWhat_Arg,
997	/** Return status code.
998	* @todo not sure if we need this... */
999	kIemNativeWhat_rc,
1000	/** The fixed pVCpu (PVMCPUCC) register.
1001	* @todo consider offsetting this on amd64 to use negative offsets to access
1002	* more members using 8-byte disp. */
1003	kIemNativeWhat_pVCpuFixed,
1004	/** The fixed pCtx (PCPUMCTX) register.
1005	* @todo consider offsetting this on amd64 to use negative offsets to access
1006	* more members using 8-byte disp. */
1007	kIemNativeWhat_pCtxFixed,
1008	/** Fixed temporary register. */
1009	kIemNativeWhat_FixedTmp,
1010	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1011	/** Shadow RIP for the delayed RIP updating debugging. */
1012	kIemNativeWhat_PcShadow,
1013	#endif
1014	/** Register reserved by the CPU or OS architecture. */
1015	kIemNativeWhat_FixedReserved,
1016	/** End of valid values. */
1017	kIemNativeWhat_End
1018	} IEMNATIVEWHAT;
1019
1020	/**
1021	* Host general register entry.
1022	*
1023	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1024	*
1025	* @todo Track immediate values in host registers similarlly to how we track the
1026	* guest register shadow copies. For it to be real helpful, though,
1027	* we probably need to know which will be reused and put them into
1028	* non-volatile registers, otherwise it's going to be more or less
1029	* restricted to an instruction or two.
1030	*/
1031	typedef struct IEMNATIVEHSTREG
1032	{
1033	/** Set of guest registers this one shadows.
1034	*
1035	* Using a bitmap here so we can designate the same host register as a copy
1036	* for more than one guest register. This is expected to be useful in
1037	* situations where one value is copied to several registers in a sequence.
1038	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1039	* sequence we'd want to let this register follow to be a copy of and there
1040	* will always be places where we'd be picking the wrong one.
1041	*/
1042	uint64_t fGstRegShadows;
1043	/** What is being kept in this register. */
1044	IEMNATIVEWHAT enmWhat;
1045	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1046	uint8_t idxVar;
1047	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1048	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1049	* that scope. */
1050	uint8_t idxStackSlot;
1051	/** Alignment padding. */
1052	uint8_t abAlign[5];
1053	} IEMNATIVEHSTREG;
1054
1055
1056	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1057	/**
1058	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1059	* halves, on architectures where there is no 256-bit register available this entry will track
1060	* two adjacent 128-bit host registers.
1061	*
1062	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1063	*/
1064	typedef struct IEMNATIVEHSTSIMDREG
1065	{
1066	/** Set of guest registers this one shadows.
1067	*
1068	* Using a bitmap here so we can designate the same host register as a copy
1069	* for more than one guest register. This is expected to be useful in
1070	* situations where one value is copied to several registers in a sequence.
1071	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1072	* sequence we'd want to let this register follow to be a copy of and there
1073	* will always be places where we'd be picking the wrong one.
1074	*/
1075	uint64_t fGstRegShadows;
1076	/** What is being kept in this register. */
1077	IEMNATIVEWHAT enmWhat;
1078	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1079	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1080	/** Alignment padding. */
1081	uint8_t abAlign[6];
1082	} IEMNATIVEHSTSIMDREG;
1083	#endif
1084
1085
1086	/**
1087	* Core state for the native recompiler, that is, things that needs careful
1088	* handling when dealing with branches.
1089	*/
1090	typedef struct IEMNATIVECORESTATE
1091	{
1092	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1093	/** The current instruction offset in bytes from when the guest program counter
1094	* was updated last. Used for delaying the write to the guest context program counter
1095	* as long as possible. */
1096	uint32_t offPc;
1097	/** Number of instructions where we could skip the updating. */
1098	uint32_t cInstrPcUpdateSkipped;
1099	#endif
1100	/** Allocation bitmap for aHstRegs. */
1101	uint32_t bmHstRegs;
1102
1103	/** Bitmap marking which host register contains guest register shadow copies.
1104	* This is used during register allocation to try preserve copies. */
1105	uint32_t bmHstRegsWithGstShadow;
1106	/** Bitmap marking valid entries in aidxGstRegShadows. */
1107	uint64_t bmGstRegShadows;
1108
1109	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1110	/** Allocation bitmap for aHstSimdRegs. */
1111	uint32_t bmHstSimdRegs;
1112
1113	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1114	* This is used during register allocation to try preserve copies. */
1115	uint32_t bmHstSimdRegsWithGstShadow;
1116	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1117	uint64_t bmGstSimdRegShadows;
1118	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1119	uint64_t bmGstSimdRegShadowDirtyLo128;
1120	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1121	uint64_t bmGstSimdRegShadowDirtyHi128;
1122	#endif
1123
1124	union
1125	{
1126	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1127	uint8_t aidxArgVars[8];
1128	/** For more efficient resetting. */
1129	uint64_t u64ArgVars;
1130	};
1131
1132	/** Allocation bitmap for the stack. */
1133	uint32_t bmStack;
1134	/** Allocation bitmap for aVars. */
1135	uint32_t bmVars;
1136
1137	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1138	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1139	* (A shadow copy of a guest register can only be held in a one host register,
1140	* there are no duplicate copies or ambiguities like that). */
1141	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1142	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1143	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1144	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1145	* (A shadow copy of a guest register can only be held in a one host register,
1146	* there are no duplicate copies or ambiguities like that). */
1147	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1148	#endif
1149
1150	/** Host register allocation tracking. */
1151	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1152	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1153	/** Host SIMD register allocation tracking. */
1154	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1155	#endif
1156
1157	/** Variables and arguments. */
1158	IEMNATIVEVAR aVars[9];
1159	} IEMNATIVECORESTATE;
1160	/** Pointer to core state. */
1161	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1162	/** Pointer to const core state. */
1163	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1164
1165	/** @def IEMNATIVE_VAR_IDX_UNPACK
1166	* @returns Index into IEMNATIVECORESTATE::aVars.
1167	* @param a_idxVar Variable index w/ magic (in strict builds).
1168	*/
1169	/** @def IEMNATIVE_VAR_IDX_PACK
1170	* @returns Variable index w/ magic (in strict builds).
1171	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1172	*/
1173	#ifdef VBOX_STRICT
1174	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1175	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1176	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1177	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1178	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1179	#else
1180	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1181	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1182	#endif
1183
1184
1185	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1186	/** Clear the dirty state of the given guest SIMD register. */
1187	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1188	do { \
1189	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1190	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1191	} while (0)
1192
1193	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1194	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1195	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1196	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1197	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1198	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1199	/** Returns whether the given guest SIMD register is dirty. */
1200	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1201	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1202
1203	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1204	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1205	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1206	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1207	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1208	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1209
1210	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1211	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1212	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1213	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(1)
1214	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1215	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(2)
1216	#endif
1217
1218
1219	/**
1220	* Conditional stack entry.
1221	*/
1222	typedef struct IEMNATIVECOND
1223	{
1224	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1225	bool fInElse;
1226	/** The label for the IEM_MC_ELSE. */
1227	uint32_t idxLabelElse;
1228	/** The label for the IEM_MC_ENDIF. */
1229	uint32_t idxLabelEndIf;
1230	/** The initial state snapshot as the if-block starts executing. */
1231	IEMNATIVECORESTATE InitialState;
1232	/** The state snapshot at the end of the if-block. */
1233	IEMNATIVECORESTATE IfFinalState;
1234	} IEMNATIVECOND;
1235	/** Pointer to a condition stack entry. */
1236	typedef IEMNATIVECOND *PIEMNATIVECOND;
1237
1238
1239	/**
1240	* Native recompiler state.
1241	*/
1242	typedef struct IEMRECOMPILERSTATE
1243	{
1244	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1245	* IEMNATIVEINSTR units. */
1246	uint32_t cInstrBufAlloc;
1247	#ifdef VBOX_STRICT
1248	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1249	uint32_t offInstrBufChecked;
1250	#else
1251	uint32_t uPadding1; /* We don't keep track of the size here... */
1252	#endif
1253	/** Fixed temporary code buffer for native recompilation. */
1254	PIEMNATIVEINSTR pInstrBuf;
1255
1256	/** Bitmaps with the label types used. */
1257	uint64_t bmLabelTypes;
1258	/** Actual number of labels in paLabels. */
1259	uint32_t cLabels;
1260	/** Max number of entries allowed in paLabels before reallocating it. */
1261	uint32_t cLabelsAlloc;
1262	/** Labels defined while recompiling (referenced by fixups). */
1263	PIEMNATIVELABEL paLabels;
1264	/** Array with indexes of unique labels (uData always 0). */
1265	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1266
1267	/** Actual number of fixups paFixups. */
1268	uint32_t cFixups;
1269	/** Max number of entries allowed in paFixups before reallocating it. */
1270	uint32_t cFixupsAlloc;
1271	/** Buffer used by the recompiler for recording fixups when generating code. */
1272	PIEMNATIVEFIXUP paFixups;
1273
1274	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1275	/** Number of debug info entries allocated for pDbgInfo. */
1276	uint32_t cDbgInfoAlloc;
1277	uint32_t uPadding;
1278	/** Debug info. */
1279	PIEMTBDBG pDbgInfo;
1280	#endif
1281
1282	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1283	/** The current call index (liveness array and threaded calls in TB). */
1284	uint32_t idxCurCall;
1285	/** Number of liveness entries allocated. */
1286	uint32_t cLivenessEntriesAlloc;
1287	/** Liveness entries for all the calls in the TB begin recompiled.
1288	* The entry for idxCurCall contains the info for what the next call will
1289	* require wrt registers. (Which means the last entry is the initial liveness
1290	* state.) */
1291	PIEMLIVENESSENTRY paLivenessEntries;
1292	#endif
1293
1294	/** The translation block being recompiled. */
1295	PCIEMTB pTbOrg;
1296	/** The VMCPU structure of the EMT. */
1297	PVMCPUCC pVCpu;
1298
1299	/** Condition sequence number (for generating unique labels). */
1300	uint16_t uCondSeqNo;
1301	/** Check IRQ seqeunce number (for generating unique labels). */
1302	uint16_t uCheckIrqSeqNo;
1303	/** TLB load sequence number (for generating unique labels). */
1304	uint16_t uTlbSeqNo;
1305	/** The current condition stack depth (aCondStack). */
1306	uint8_t cCondDepth;
1307
1308	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1309	uint8_t cArgs;
1310	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1311	uint32_t fCImpl;
1312	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1313	uint32_t fMc;
1314	/** The expected IEMCPU::fExec value for the current call/instruction. */
1315	uint32_t fExec;
1316	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1317	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1318	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1319	*
1320	* This is an optimization because these control registers can only be changed from
1321	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1322	* consisting of multiple SIMD instructions.
1323	*/
1324	uint32_t fSimdRaiseXcptChecksEmitted;
1325	#endif
1326
1327	/** Core state requiring care with branches. */
1328	IEMNATIVECORESTATE Core;
1329
1330	/** The condition nesting stack. */
1331	IEMNATIVECOND aCondStack[2];
1332
1333	#ifndef IEM_WITH_THROW_CATCH
1334	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1335	* for recompilation error handling. */
1336	jmp_buf JmpBuf;
1337	#endif
1338	} IEMRECOMPILERSTATE;
1339	/** Pointer to a native recompiler state. */
1340	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1341
1342
1343	/** @def IEMNATIVE_TRY_SETJMP
1344	* Wrapper around setjmp / try, hiding all the ugly differences.
1345	*
1346	* @note Use with extreme care as this is a fragile macro.
1347	* @param a_pReNative The native recompile state.
1348	* @param a_rcTarget The variable that should receive the status code in case
1349	* of a longjmp/throw.
1350	*/
1351	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1352	* Start wrapper for catch / setjmp-else.
1353	*
1354	* This will set up a scope.
1355	*
1356	* @note Use with extreme care as this is a fragile macro.
1357	* @param a_pReNative The native recompile state.
1358	* @param a_rcTarget The variable that should receive the status code in case
1359	* of a longjmp/throw.
1360	*/
1361	/** @def IEMNATIVE_CATCH_LONGJMP_END
1362	* End wrapper for catch / setjmp-else.
1363	*
1364	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1365	* up the state.
1366	*
1367	* @note Use with extreme care as this is a fragile macro.
1368	* @param a_pReNative The native recompile state.
1369	*/
1370	/** @def IEMNATIVE_DO_LONGJMP
1371	*
1372	* Wrapper around longjmp / throw.
1373	*
1374	* @param a_pReNative The native recompile state.
1375	* @param a_rc The status code jump back with / throw.
1376	*/
1377	#ifdef IEM_WITH_THROW_CATCH
1378	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1379	a_rcTarget = VINF_SUCCESS; \
1380	try
1381	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1382	catch (int rcThrown) \
1383	{ \
1384	a_rcTarget = rcThrown
1385	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1386	} \
1387	((void)0)
1388	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1389	#else /* !IEM_WITH_THROW_CATCH */
1390	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1391	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1392	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1393	else \
1394	{ \
1395	((void)0)
1396	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1397	}
1398	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1399	#endif /* !IEM_WITH_THROW_CATCH */
1400
1401
1402	/**
1403	* Native recompiler worker for a threaded function.
1404	*
1405	* @returns New code buffer offset; throws VBox status code in case of a failure.
1406	* @param pReNative The native recompiler state.
1407	* @param off The current code buffer offset.
1408	* @param pCallEntry The threaded call entry.
1409	*
1410	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1411	*/
1412	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1413	/** Pointer to a native recompiler worker for a threaded function. */
1414	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1415
1416	/** Defines a native recompiler worker for a threaded function.
1417	* @see FNIEMNATIVERECOMPFUNC */
1418	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1419	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1420
1421	/** Prototypes a native recompiler function for a threaded function.
1422	* @see FNIEMNATIVERECOMPFUNC */
1423	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1424
1425
1426	/**
1427	* Native recompiler liveness analysis worker for a threaded function.
1428	*
1429	* @param pCallEntry The threaded call entry.
1430	* @param pIncoming The incoming liveness state entry.
1431	* @param pOutgoing The outgoing liveness state entry.
1432	*/
1433	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1434	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1435	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1436	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1437
1438	/** Defines a native recompiler liveness analysis worker for a threaded function.
1439	* @see FNIEMNATIVELIVENESSFUNC */
1440	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1441	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1442
1443	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1444	* @see FNIEMNATIVELIVENESSFUNC */
1445	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1446
1447
1448	/** Define a native recompiler helper function, safe to call from the TB code. */
1449	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1450	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1451	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1452	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1453	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1454
1455
1456	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1457	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1458	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1459	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1460	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1461	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1462	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1463	uint8_t idxHstSimdReg = UINT8_MAX,
1464	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1465	# endif
1466	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1467	uint32_t offPc, uint32_t cInstrSkipped);
1468	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1469
1470	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1471	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1472	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1473	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1474	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1475	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1476
1477	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1478	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1479	bool fPreferVolatile = true);
1480	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1481	bool fPreferVolatile = true);
1482	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1483	IEMNATIVEGSTREG enmGstReg,
1484	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1485	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1486	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1487	IEMNATIVEGSTREG enmGstReg);
1488
1489	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1490	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1491	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1492	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1493	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1494	#endif
1495	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1496	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1497	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1498	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1499	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1500	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1501	uint32_t fKeepVars = 0);
1502	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1503	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1504	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1505	uint32_t fHstRegsActiveShadows);
1506	#ifdef VBOX_STRICT
1507	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1508	#endif
1509	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1510	uint64_t fGstShwExcept, bool fFlushShadows);
1511	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1512	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1513	#endif
1514
1515
1516	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1517	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1518	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1519	bool fPreferVolatile = true);
1520	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1521	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1522	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1523	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1524	bool fNoVolatileRegs = false);
1525	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1526	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1527	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1528	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1529	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1530	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1531	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1532	#endif
1533
1534	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1535	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1536	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1537	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1538	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1539	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1540	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1541	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1542	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1543	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1544	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1545	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1546	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1547	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1548	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1549	uint32_t fHstRegsNotToSave);
1550	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1551	uint32_t fHstRegsNotToSave);
1552	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1553	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1554
1555	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1556	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1557	#ifdef VBOX_STRICT
1558	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1559	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1560	IEMNATIVEGSTREG enmGstReg);
1561	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1562	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1563	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1564	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1565	# endif
1566	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1567	#endif
1568	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1569	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1570	#endif
1571	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1572	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1573	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1574	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1575	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1576	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1577	PCIEMTHRDEDCALLENTRY pCallEntry);
1578	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1579	uint8_t idxAddrReg, uint8_t idxInstr);
1580	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1581	uint8_t idxAddrReg, uint8_t idxInstr);
1582
1583
1584	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1585	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1586	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1587	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1588	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1589	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1590	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1591	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1592	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1593	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1594
1595	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1596	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1597	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1598	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1599	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1600	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1601	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1602	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1603	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1604	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1605	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1606	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1607	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1608	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1609	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1610	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1611	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1612	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1613	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1614	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1615	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1616
1617	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1618	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1619	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1620	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1621	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1622	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1623	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1624	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1625	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1626	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1627	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1628	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1629	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1630	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1631	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1632	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1633	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1634	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1635	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1636	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1637	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1638
1639	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1640	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1641	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1642	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1643	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1644	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1645	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1646	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1647	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1648	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1649	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1650	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1651	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1652	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1653	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1654	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1655	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1656	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1657	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1658	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1659	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1660	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1661
1662	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1663	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1664	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1665	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1666	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1667	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1668	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1669	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1670	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1671	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1672	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1673	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1674	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1675	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1676	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1677	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1678	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1679	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1680	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1681	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1682	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1683	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1684
1685	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1686	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1687	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1688	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1689
1690
1691	/**
1692	* Info about shadowed guest register values.
1693	* @see IEMNATIVEGSTREG
1694	*/
1695	typedef struct IEMANTIVEGSTREGINFO
1696	{
1697	/** Offset in VMCPU. */
1698	uint32_t off;
1699	/** The field size. */
1700	uint8_t cb;
1701	/** Name (for logging). */
1702	const char *pszName;
1703	} IEMANTIVEGSTREGINFO;
1704	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1705	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1706	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1707	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1708	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1709
1710
1711
1712	/**
1713	* Ensures that there is sufficient space in the instruction output buffer.
1714	*
1715	* This will reallocate the buffer if needed and allowed.
1716	*
1717	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1718	* allocation size.
1719	*
1720	* @returns Pointer to the instruction output buffer on success; throws VBox
1721	* status code on failure, so no need to check it.
1722	* @param pReNative The native recompile state.
1723	* @param off Current instruction offset. Works safely for UINT32_MAX
1724	* as well.
1725	* @param cInstrReq Number of instruction about to be added. It's okay to
1726	* overestimate this a bit.
1727	*/
1728	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1729	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1730	{
1731	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1732	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1733	{
1734	#ifdef VBOX_STRICT
1735	pReNative->offInstrBufChecked = offChecked;
1736	#endif
1737	return pReNative->pInstrBuf;
1738	}
1739	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1740	}
1741
1742	/**
1743	* Checks that we didn't exceed the space requested in the last
1744	* iemNativeInstrBufEnsure() call.
1745	*/
1746	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1747	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1748	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1749
1750	/**
1751	* Checks that a variable index is valid.
1752	*/
1753	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1754	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1755	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1756	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1757	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1758	("%s=%#x\n", #a_idxVar, a_idxVar))
1759	#else
1760	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1761	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1762	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1763	#endif
1764
1765	/**
1766	* Checks that a variable index is valid and that the variable is assigned the
1767	* correct argument number.
1768	* This also adds a RT_NOREF of a_idxVar.
1769	*/
1770	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1771	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1772	RT_NOREF_PV(a_idxVar); \
1773	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1774	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1775	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1776	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1777	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1778	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1779	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1780	a_uArgNo)); \
1781	} while (0)
1782	#else
1783	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1784	RT_NOREF_PV(a_idxVar); \
1785	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1786	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1787	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1788	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1789	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1790	} while (0)
1791	#endif
1792
1793
1794	/**
1795	* Checks that a variable has the expected size.
1796	*/
1797	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1798	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1799	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1800	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1801
1802
1803	/**
1804	* Calculates the stack address of a variable as a [r]BP displacement value.
1805	*/
1806	DECL_FORCE_INLINE(int32_t)
1807	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1808	{
1809	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1810	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1811	}
1812
1813
1814	/**
1815	* Releases the variable's register.
1816	*
1817	* The register must have been previously acquired calling
1818	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1819	* iemNativeVarRegisterSetAndAcquire().
1820	*/
1821	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1822	{
1823	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1824	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1825	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1826	}
1827
1828
1829	/**
1830	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1831	*
1832	* @returns The flush mask.
1833	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1834	* @param fGstShwFlush The starting flush mask.
1835	*/
1836	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1837	{
1838	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1839	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1840	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1841	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1842	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1843	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1844	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1845	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1846	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1847	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1848	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1849	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1850	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1851	return fGstShwFlush;
1852	}
1853
1854
1855	/** Number of hidden arguments for CIMPL calls.
1856	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1857	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1858	# define IEM_CIMPL_HIDDEN_ARGS 3
1859	#else
1860	# define IEM_CIMPL_HIDDEN_ARGS 2
1861	#endif
1862
1863
1864	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1865
1866	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1867	/**
1868	* Helper for iemNativeLivenessGetStateByGstReg.
1869	*
1870	* @returns IEMLIVENESS_STATE_XXX
1871	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1872	* ORed together.
1873	*/
1874	DECL_FORCE_INLINE(uint32_t)
1875	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1876	{
1877	/* INPUT trumps anything else. */
1878	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1879	return IEMLIVENESS_STATE_INPUT;
1880
1881	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1882	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1883	{
1884	/* If not all sub-fields are clobbered they must be considered INPUT. */
1885	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1886	return IEMLIVENESS_STATE_INPUT;
1887	return IEMLIVENESS_STATE_CLOBBERED;
1888	}
1889
1890	/* XCPT_OR_CALL trumps UNUSED. */
1891	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1892	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1893
1894	return IEMLIVENESS_STATE_UNUSED;
1895	}
1896	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1897
1898
1899	DECL_FORCE_INLINE(uint32_t)
1900	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1901	{
1902	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1903	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1904	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1905	# else
1906	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1907	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1908	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1909	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1910	# endif
1911	}
1912
1913
1914	DECL_FORCE_INLINE(uint32_t)
1915	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
1916	{
1917	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
1918	if (enmGstReg == kIemNativeGstReg_EFlags)
1919	{
1920	/* Merge the eflags states to one. */
1921	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1922	uRet = RT_BIT_32(uRet);
1923	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
1924	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
1925	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
1926	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
1927	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
1928	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
1929	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
1930	# else
1931	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
1932	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
1933	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
1934	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
1935	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
1936	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
1937	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
1938	# endif
1939	}
1940	return uRet;
1941	}
1942
1943
1944	# ifdef VBOX_STRICT
1945	/** For assertions only, user checks that idxCurCall isn't zerow. */
1946	DECL_FORCE_INLINE(uint32_t)
1947	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
1948	{
1949	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
1950	}
1951	# endif /* VBOX_STRICT */
1952
1953	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
1954
1955
1956	/**
1957	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
1958	*/
1959	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
1960	{
1961	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
1962	return IEM_CIMPL_HIDDEN_ARGS;
1963	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
1964	return 1;
1965	return 0;
1966	}
1967
1968
1969	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
1970	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
1971	{
1972	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
1973
1974	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
1975	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
1976	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
1977	return (uint8_t)idxReg;
1978	}
1979
1980
1981
1982	/*********************************************************************************************************************************
1983	* Register Allocator (GPR) *
1984	*********************************************************************************************************************************/
1985
1986	/**
1987	* Marks host register @a idxHstReg as containing a shadow copy of guest
1988	* register @a enmGstReg.
1989	*
1990	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
1991	* host register before calling.
1992	*/
1993	DECL_FORCE_INLINE(void)
1994	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
1995	{
1996	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
1997	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
1998	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
1999
2000	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2001	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2002	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2003	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2004	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2005	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2006	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2007	#else
2008	RT_NOREF(off);
2009	#endif
2010	}
2011
2012
2013	/**
2014	* Clear any guest register shadow claims from @a idxHstReg.
2015	*
2016	* The register does not need to be shadowing any guest registers.
2017	*/
2018	DECL_FORCE_INLINE(void)
2019	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2020	{
2021	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2022	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2023	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2024	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2025	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2026
2027	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2028	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2029	if (fGstRegs)
2030	{
2031	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2032	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2033	while (fGstRegs)
2034	{
2035	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2036	fGstRegs &= ~RT_BIT_64(iGstReg);
2037	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2038	}
2039	}
2040	#else
2041	RT_NOREF(off);
2042	#endif
2043
2044	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2045	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2046	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2047	}
2048
2049
2050	/**
2051	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2052	* and global overview flags.
2053	*/
2054	DECL_FORCE_INLINE(void)
2055	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2056	{
2057	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2058	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2059	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2060	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2061	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2062	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2063	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2064
2065	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2066	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2067	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2068	#else
2069	RT_NOREF(off);
2070	#endif
2071
2072	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2073	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2074	if (!fGstRegShadowsNew)
2075	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2076	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2077	}
2078
2079
2080	#if 0 /* unused */
2081	/**
2082	* Clear any guest register shadow claim for @a enmGstReg.
2083	*/
2084	DECL_FORCE_INLINE(void)
2085	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2086	{
2087	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2088	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2089	{
2090	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2091	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2092	}
2093	}
2094	#endif
2095
2096
2097	/**
2098	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2099	* as the new shadow of it.
2100	*
2101	* Unlike the other guest reg shadow helpers, this does the logging for you.
2102	* However, it is the liveness state is not asserted here, the caller must do
2103	* that.
2104	*/
2105	DECL_FORCE_INLINE(void)
2106	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2107	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2108	{
2109	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2110	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2111	{
2112	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2113	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2114	if (idxHstRegOld == idxHstRegNew)
2115	return;
2116	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2117	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2118	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2119	}
2120	else
2121	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2122	g_aGstShadowInfo[enmGstReg].pszName));
2123	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2124	}
2125
2126
2127	/**
2128	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2129	* to @a idxRegTo.
2130	*/
2131	DECL_FORCE_INLINE(void)
2132	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2133	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2134	{
2135	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2136	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2137	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2138	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2139	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2140	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2141	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2142	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2143	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2144
2145	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2146	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2147	if (!fGstRegShadowsFrom)
2148	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2149	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2150	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2151	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2152	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2153	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2154	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2155	#else
2156	RT_NOREF(off);
2157	#endif
2158	}
2159
2160
2161	/**
2162	* Flushes any delayed guest register writes.
2163	*
2164	* This must be called prior to calling CImpl functions and any helpers that use
2165	* the guest state (like raising exceptions) and such.
2166	*
2167	* This optimization has not yet been implemented. The first target would be
2168	* RIP updates, since these are the most common ones.
2169	*/
2170	DECL_INLINE_THROW(uint32_t)
2171	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2172	{
2173	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2174	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2175	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2176	#else
2177	RT_NOREF(pReNative, fGstShwExcept);
2178	#endif
2179
2180	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2181	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2182	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2183	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2184	#else
2185	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2186	return off;
2187	#endif
2188	}
2189
2190
2191
2192	/*********************************************************************************************************************************
2193	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2194	*********************************************************************************************************************************/
2195
2196	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2197
2198	DECL_FORCE_INLINE(uint8_t)
2199	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2200	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2201	{
2202	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2203
2204	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2205	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2206	RT_NOREF(idxVar);
2207	return idxSimdReg;
2208	}
2209
2210
2211	/**
2212	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2213	* SIMD register @a enmGstSimdReg.
2214	*
2215	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2216	* host register before calling.
2217	*/
2218	DECL_FORCE_INLINE(void)
2219	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2220	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2221	{
2222	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2223	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2224	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2225
2226	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2227	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2228	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2229	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2230	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2231	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2232	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2233	#else
2234	RT_NOREF(off);
2235	#endif
2236	}
2237
2238
2239	/**
2240	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2241	* to @a idxSimdRegTo.
2242	*/
2243	DECL_FORCE_INLINE(void)
2244	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2245	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2246	{
2247	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2248	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2249	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2250	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2251	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2252	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2253	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2254	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2255	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2256	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2257	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2258
2259	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2260	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2261	if (!fGstRegShadowsFrom)
2262	{
2263	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2264	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2265	}
2266	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2267	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2268	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2269	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2270	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2271	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2272	#else
2273	RT_NOREF(off);
2274	#endif
2275	}
2276
2277
2278	/**
2279	* Clear any guest register shadow claims from @a idxHstSimdReg.
2280	*
2281	* The register does not need to be shadowing any guest registers.
2282	*/
2283	DECL_FORCE_INLINE(void)
2284	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2285	{
2286	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2287	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2288	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2289	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2290	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2291	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2292	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2293
2294	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2295	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2296	if (fGstRegs)
2297	{
2298	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2299	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2300	while (fGstRegs)
2301	{
2302	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2303	fGstRegs &= ~RT_BIT_64(iGstReg);
2304	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2305	}
2306	}
2307	#else
2308	RT_NOREF(off);
2309	#endif
2310
2311	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2312	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2313	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2314	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2315	}
2316
2317	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2318
2319
2320	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2321	/**
2322	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2323	*/
2324	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2325	{
2326	if (pReNative->Core.offPc)
2327	return iemNativeEmitPcWritebackSlow(pReNative, off);
2328	return off;
2329	}
2330	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2331
2332
2333	/** @} */
2334
2335	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2336

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103839

Download in other formats: