IEMN8veRecompiler.h@ 103809

Last change on this file since 103809 was 103809, checked in by vboxsync, 13 months ago
VMM/IEM: Split up the native recompiler functions (IEMNativeFunctions.cpp.h) into 4 files to speed up compilation and reduce compiler memory consumption. This involved splitting out half the content of IEMAllThrdRecompiler.cpp into IEMAllN8veRecompFuncs.h and IEMN8veRecompiler.h. [build fix] bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 110.3 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103809 2024-03-12 19:56:41Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	#ifdef VBOX_WITH_STATISTICS
54	/** Always count instructions for now. */
55	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
56	#endif
57
58
59	/** @name Stack Frame Layout
60	*
61	* @{ */
62	/** The size of the area for stack variables and spills and stuff.
63	* @note This limit is duplicated in the python script(s). We add 0x40 for
64	* alignment padding. */
65	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
66	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
67	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
68	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
69
70	#ifdef RT_ARCH_AMD64
71	/** An stack alignment adjustment (between non-volatile register pushes and
72	* the stack variable area, so the latter better aligned). */
73	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
74
75	/** Number of stack arguments slots for calls made from the frame. */
76	# ifdef RT_OS_WINDOWS
77	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
78	# else
79	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
80	# endif
81	/** Number of any shadow arguments (spill area) for calls we make. */
82	# ifdef RT_OS_WINDOWS
83	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
84	# else
85	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
86	# endif
87
88	/** Frame pointer (RBP) relative offset of the last push. */
89	# ifdef RT_OS_WINDOWS
90	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
91	# else
92	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
93	# endif
94	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
95	* address for it). */
96	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
97	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
98	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
99	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
100	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
101	# ifdef RT_OS_WINDOWS
102	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
103	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
104	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
105	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
106	# endif
107
108	# ifdef RT_OS_WINDOWS
109	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
110	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
111	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
112	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
113	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
114	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
115	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
116	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
117	# endif
118
119	#elif RT_ARCH_ARM64
120	/** No alignment padding needed for arm64. */
121	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
122	/** No stack argument slots, got 8 registers for arguments will suffice. */
123	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
124	/** There are no argument spill area. */
125	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
126
127	/** Number of saved registers at the top of our stack frame.
128	* This includes the return address and old frame pointer, so x19 thru x30. */
129	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
130	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
131	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
132
133	/** Frame pointer (BP) relative offset of the last push. */
134	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
135
136	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
137	* address for it). */
138	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
139
140	#else
141	# error "port me"
142	#endif
143	/** @} */
144
145
146	/** @name Fixed Register Allocation(s)
147	* @{ */
148	/** @def IEMNATIVE_REG_FIXED_PVMCPU
149	* The number of the register holding the pVCpu pointer. */
150	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
151	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
152	* @note This not available on AMD64, only ARM64. */
153	/** @def IEMNATIVE_REG_FIXED_TMP0
154	* Dedicated temporary register.
155	* @todo replace this by a register allocator and content tracker. */
156	/** @def IEMNATIVE_REG_FIXED_MASK
157	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
158	* architecture. */
159	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
160	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
161	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
162	* architecture. */
163	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
164	* Dedicated temporary SIMD register. */
165	#endif
166	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
167	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
168	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
169	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
170	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
171	\| RT_BIT_32(X86_GREG_xSP) \
172	\| RT_BIT_32(X86_GREG_xBP) )
173
174	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
175	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
176	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS) \|\| !defined(_MSC_VER)
177	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0)
178	# else
179	/** On Windows xmm6 through xmm15 are marked as callee saved. */
180	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0)
181	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
182	# endif
183	# endif
184
185	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
186	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
187	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
188	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
189	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
190	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
191	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
192	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
193	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
194	# else
195	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
196	# endif
197	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
198	\| RT_BIT_32(ARMV8_A64_REG_LR) \
199	\| RT_BIT_32(ARMV8_A64_REG_BP) \
200	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
201	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
202	\| RT_BIT_32(ARMV8_A64_REG_X18) \
203	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
204	\| IEMNATIVE_REG_FIXED_MASK_ADD)
205
206	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
207	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
208	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
209	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
210	# else
211	/** arm64 declares the low 64-bit of v8-v15 as callee saved. */
212	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
213	\| RT_BIT_32(ARMV8_A64_REG_Q30))
214	# endif
215	# endif
216
217	#else
218	# error "port me"
219	#endif
220	/** @} */
221
222	/** @name Call related registers.
223	* @{ */
224	/** @def IEMNATIVE_CALL_RET_GREG
225	* The return value register. */
226	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
227	* Number of arguments in registers. */
228	/** @def IEMNATIVE_CALL_ARG0_GREG
229	* The general purpose register carrying argument \#0. */
230	/** @def IEMNATIVE_CALL_ARG1_GREG
231	* The general purpose register carrying argument \#1. */
232	/** @def IEMNATIVE_CALL_ARG2_GREG
233	* The general purpose register carrying argument \#2. */
234	/** @def IEMNATIVE_CALL_ARG3_GREG
235	* The general purpose register carrying argument \#3. */
236	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
237	* Mask of registers the callee will not save and may trash. */
238	#ifdef RT_ARCH_AMD64
239	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
240
241	# ifdef RT_OS_WINDOWS
242	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
243	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
244	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
245	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
246	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
247	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
248	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
249	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
250	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
251	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
252	\| RT_BIT_32(X86_GREG_xCX) \
253	\| RT_BIT_32(X86_GREG_xDX) \
254	\| RT_BIT_32(X86_GREG_x8) \
255	\| RT_BIT_32(X86_GREG_x9) \
256	\| RT_BIT_32(X86_GREG_x10) \
257	\| RT_BIT_32(X86_GREG_x11) )
258	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
259	/* xmm0 - xmm5 are marked as volatile. */
260	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
261	# endif
262
263	# else
264	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
265	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
266	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
267	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
268	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
269	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
270	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
271	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
272	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
273	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
274	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
275	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
276	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
277	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
278	\| RT_BIT_32(X86_GREG_xCX) \
279	\| RT_BIT_32(X86_GREG_xDX) \
280	\| RT_BIT_32(X86_GREG_xDI) \
281	\| RT_BIT_32(X86_GREG_xSI) \
282	\| RT_BIT_32(X86_GREG_x8) \
283	\| RT_BIT_32(X86_GREG_x9) \
284	\| RT_BIT_32(X86_GREG_x10) \
285	\| RT_BIT_32(X86_GREG_x11) )
286	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
287	/* xmm0 - xmm15 are marked as volatile. */
288	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
289	# endif
290	# endif
291
292	#elif defined(RT_ARCH_ARM64)
293	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
294	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
295	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
296	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
297	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
298	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
299	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
300	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
301	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
302	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
303	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
304	\| RT_BIT_32(ARMV8_A64_REG_X1) \
305	\| RT_BIT_32(ARMV8_A64_REG_X2) \
306	\| RT_BIT_32(ARMV8_A64_REG_X3) \
307	\| RT_BIT_32(ARMV8_A64_REG_X4) \
308	\| RT_BIT_32(ARMV8_A64_REG_X5) \
309	\| RT_BIT_32(ARMV8_A64_REG_X6) \
310	\| RT_BIT_32(ARMV8_A64_REG_X7) )
311	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
312	\| RT_BIT_32(ARMV8_A64_REG_X1) \
313	\| RT_BIT_32(ARMV8_A64_REG_X2) \
314	\| RT_BIT_32(ARMV8_A64_REG_X3) \
315	\| RT_BIT_32(ARMV8_A64_REG_X4) \
316	\| RT_BIT_32(ARMV8_A64_REG_X5) \
317	\| RT_BIT_32(ARMV8_A64_REG_X6) \
318	\| RT_BIT_32(ARMV8_A64_REG_X7) \
319	\| RT_BIT_32(ARMV8_A64_REG_X8) \
320	\| RT_BIT_32(ARMV8_A64_REG_X9) \
321	\| RT_BIT_32(ARMV8_A64_REG_X10) \
322	\| RT_BIT_32(ARMV8_A64_REG_X11) \
323	\| RT_BIT_32(ARMV8_A64_REG_X12) \
324	\| RT_BIT_32(ARMV8_A64_REG_X13) \
325	\| RT_BIT_32(ARMV8_A64_REG_X14) \
326	\| RT_BIT_32(ARMV8_A64_REG_X15) \
327	\| RT_BIT_32(ARMV8_A64_REG_X16) \
328	\| RT_BIT_32(ARMV8_A64_REG_X17) )
329	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
330	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
331	* so to simplify our life a bit we just mark everything as volatile. */
332	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
333	# endif
334
335	#endif
336
337	/** This is the maximum argument count we'll ever be needing. */
338	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
339	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
340	#else
341	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
342	#endif
343	/** @} */
344
345
346	/** @def IEMNATIVE_HST_GREG_COUNT
347	* Number of host general purpose registers we tracker. */
348	/** @def IEMNATIVE_HST_GREG_MASK
349	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
350	* inverted register masks and such to get down to a correct set of regs. */
351	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
352	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
353	* Number of host SIMD registers we track. */
354	/** @def IEMNATIVE_HST_SIMD_REG_MASK
355	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
356	* inverted register masks and such to get down to a correct set of regs. */
357	#endif
358	#ifdef RT_ARCH_AMD64
359	# define IEMNATIVE_HST_GREG_COUNT 16
360	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
361
362	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
363	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
364	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
365	# endif
366
367	#elif defined(RT_ARCH_ARM64)
368	# define IEMNATIVE_HST_GREG_COUNT 32
369	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
370
371	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
372	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
373	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
374	# endif
375
376	#else
377	# error "Port me!"
378	#endif
379
380
381	/** Native code generator label types. */
382	typedef enum
383	{
384	kIemNativeLabelType_Invalid = 0,
385	/* Labels w/o data, only once instance per TB: */
386	kIemNativeLabelType_Return,
387	kIemNativeLabelType_ReturnBreak,
388	kIemNativeLabelType_ReturnWithFlags,
389	kIemNativeLabelType_NonZeroRetOrPassUp,
390	kIemNativeLabelType_RaiseGp0,
391	kIemNativeLabelType_RaiseNm,
392	kIemNativeLabelType_RaiseUd,
393	kIemNativeLabelType_RaiseMf,
394	kIemNativeLabelType_RaiseXf,
395	kIemNativeLabelType_ObsoleteTb,
396	kIemNativeLabelType_NeedCsLimChecking,
397	kIemNativeLabelType_CheckBranchMiss,
398	/* Labels with data, potentially multiple instances per TB: */
399	kIemNativeLabelType_FirstWithMultipleInstances,
400	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
401	kIemNativeLabelType_Else,
402	kIemNativeLabelType_Endif,
403	kIemNativeLabelType_CheckIrq,
404	kIemNativeLabelType_TlbLookup,
405	kIemNativeLabelType_TlbMiss,
406	kIemNativeLabelType_TlbDone,
407	kIemNativeLabelType_End
408	} IEMNATIVELABELTYPE;
409
410	/** Native code generator label definition. */
411	typedef struct IEMNATIVELABEL
412	{
413	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
414	* the epilog. */
415	uint32_t off;
416	/** The type of label (IEMNATIVELABELTYPE). */
417	uint16_t enmType;
418	/** Additional label data, type specific. */
419	uint16_t uData;
420	} IEMNATIVELABEL;
421	/** Pointer to a label. */
422	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
423
424
425	/** Native code generator fixup types. */
426	typedef enum
427	{
428	kIemNativeFixupType_Invalid = 0,
429	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
430	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
431	kIemNativeFixupType_Rel32,
432	#elif defined(RT_ARCH_ARM64)
433	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
434	kIemNativeFixupType_RelImm26At0,
435	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
436	kIemNativeFixupType_RelImm19At5,
437	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
438	kIemNativeFixupType_RelImm14At5,
439	#endif
440	kIemNativeFixupType_End
441	} IEMNATIVEFIXUPTYPE;
442
443	/** Native code generator fixup. */
444	typedef struct IEMNATIVEFIXUP
445	{
446	/** Code offset of the fixup location. */
447	uint32_t off;
448	/** The IEMNATIVELABEL this is a fixup for. */
449	uint16_t idxLabel;
450	/** The fixup type (IEMNATIVEFIXUPTYPE). */
451	uint8_t enmType;
452	/** Addend or other data. */
453	int8_t offAddend;
454	} IEMNATIVEFIXUP;
455	/** Pointer to a native code generator fixup. */
456	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
457
458
459	/**
460	* One bit of the state.
461	*
462	* Each register state takes up two bits. We keep the two bits in two separate
463	* 64-bit words to simplify applying them to the guest shadow register mask in
464	* the register allocator.
465	*/
466	typedef union IEMLIVENESSBIT
467	{
468	uint64_t bm64;
469	RT_GCC_EXTENSION struct
470	{ /* bit no */
471	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
472	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
473	uint64_t fCr0 : 1; /*< 0x11 / 17: /
474	uint64_t fFcw : 1; /*< 0x12 / 18: /
475	uint64_t fFsw : 1; /*< 0x13 / 19: /
476	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
477	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
478	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
479	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
480	uint64_t fCr4 : 1; /*< 0x2c / 44: /
481	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
482	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
483	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
484	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
485	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
486	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
487	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
488	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
489	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
490	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
491	};
492	} IEMLIVENESSBIT;
493	AssertCompileSize(IEMLIVENESSBIT, 8);
494
495	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
496	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
497	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
498	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
499	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
500	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
501	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
502
503
504	/**
505	* A liveness state entry.
506	*
507	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
508	* Once we add a SSE register shadowing, we'll add another 64-bit element for
509	* that.
510	*/
511	typedef union IEMLIVENESSENTRY
512	{
513	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
514	uint64_t bm64[16 / 8];
515	uint16_t bm32[16 / 4];
516	uint16_t bm16[16 / 2];
517	uint8_t bm8[ 16 / 1];
518	IEMLIVENESSBIT aBits[2];
519	#else
520	uint64_t bm64[32 / 8];
521	uint16_t bm32[32 / 4];
522	uint16_t bm16[32 / 2];
523	uint8_t bm8[ 32 / 1];
524	IEMLIVENESSBIT aBits[4];
525	#endif
526	RT_GCC_EXTENSION struct
527	{
528	/** Bit \#0 of the register states. */
529	IEMLIVENESSBIT Bit0;
530	/** Bit \#1 of the register states. */
531	IEMLIVENESSBIT Bit1;
532	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
533	/** Bit \#2 of the register states. */
534	IEMLIVENESSBIT Bit2;
535	/** Bit \#3 of the register states. */
536	IEMLIVENESSBIT Bit3;
537	#endif
538	};
539	} IEMLIVENESSENTRY;
540	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
541	AssertCompileSize(IEMLIVENESSENTRY, 16);
542	#else
543	AssertCompileSize(IEMLIVENESSENTRY, 32);
544	#endif
545	/** Pointer to a liveness state entry. */
546	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
547	/** Pointer to a const liveness state entry. */
548	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
549
550	/** @name 64-bit value masks for IEMLIVENESSENTRY.
551	* @{ / / 0xzzzzyyyyxxxxwwww */
552	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
553
554	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
555	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
556	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
557
558	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
559	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
560	#endif
561
562	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
563
564	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
565	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
566	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
567	#endif
568	/** @} */
569
570
571	/** @name The liveness state for a register.
572	*
573	* The state values have been picked to with state accumulation in mind (what
574	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
575	* performance critical work done with the values.
576	*
577	* This is a compressed state that only requires 2 bits per register.
578	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
579	* 1. the incoming state from the following call,
580	* 2. the outgoing state for this call,
581	* 3. mask of the entries set in the 2nd.
582	*
583	* The mask entry (3rd one above) will be used both when updating the outgoing
584	* state and when merging in incoming state for registers not touched by the
585	* current call.
586	*
587	* @{ */
588	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
589	/** The register will be clobbered and the current value thrown away.
590	*
591	* When this is applied to the state (2) we'll simply be AND'ing it with the
592	* (old) mask (3) and adding the register to the mask. This way we'll
593	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
594	* IEMLIVENESS_STATE_INPUT states. */
595	# define IEMLIVENESS_STATE_CLOBBERED 0
596	/** The register is unused in the remainder of the TB.
597	*
598	* This is an initial state and can not be set by any of the
599	* iemNativeLivenessFunc_xxxx callbacks. */
600	# define IEMLIVENESS_STATE_UNUSED 1
601	/** The register value is required in a potential call or exception.
602	*
603	* This means that the register value must be calculated and is best written to
604	* the state, but that any shadowing registers can be flushed thereafter as it's
605	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
606	*
607	* It is typically applied across the board, but we preserve incoming
608	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
609	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
610	* 1. r0 = old & ~mask;
611	* 2. r0 = t1 & (t1 >> 1)'
612	* 3. state \|= r0 \| 0b10;
613	* 4. mask = ~0;
614	*/
615	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
616	/** The register value is used as input.
617	*
618	* This means that the register value must be calculated and it is best to keep
619	* it in a register. It does not need to be writtent out as such. This is the
620	* highest priority state.
621	*
622	* Whether the call modifies the register or not isn't relevant to earlier
623	* calls, so that's not recorded.
624	*
625	* When applying this state we just or in the value in the outgoing state and
626	* mask. */
627	# define IEMLIVENESS_STATE_INPUT 3
628	/** Mask of the state bits. */
629	# define IEMLIVENESS_STATE_MASK 3
630	/** The number of bits per state. */
631	# define IEMLIVENESS_STATE_BIT_COUNT 2
632	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
633	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
634	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
635	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
636	/** Check if a register clobbering is expected given the (previous) liveness state.
637	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
638	* include INPUT if the register is used in more than one place. */
639	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
640
641	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
642	/** The register is not used any more. */
643	# define IEMLIVENESS_STATE_UNUSED 0
644	/** Flag: The register is required in a potential exception or call. */
645	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
646	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
647	/** Flag: The register is read. */
648	# define IEMLIVENESS_STATE_READ 2
649	# define IEMLIVENESS_BIT_READ 1
650	/** Flag: The register is written. */
651	# define IEMLIVENESS_STATE_WRITE 4
652	# define IEMLIVENESS_BIT_WRITE 2
653	/** Flag: Unconditional call (not needed, can be redefined for research). */
654	# define IEMLIVENESS_STATE_CALL 8
655	# define IEMLIVENESS_BIT_CALL 3
656	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
657	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
658	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
659	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
660	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
661	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
662	/** @} */
663
664	/** @name Liveness helpers for builtin functions and similar.
665	*
666	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
667	* own set of manimulator macros for those.
668	*
669	* @{ */
670	/** Initializing the state as all unused. */
671	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
672	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
673	do { \
674	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
675	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
676	} while (0)
677	#else
678	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
679	do { \
680	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
681	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
682	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
683	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
684	} while (0)
685	#endif
686
687	/** Initializing the outgoing state with a potential xcpt or call state.
688	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
689	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
690	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
691	do { \
692	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
693	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
694	} while (0)
695	#else
696	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
697	do { \
698	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
699	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
700	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
701	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
702	} while (0)
703	#endif
704
705	/** Adds a segment base register as input to the outgoing state. */
706	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
707	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
708	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
709	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
710	} while (0)
711	#else
712	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
713	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
714	} while (0)
715	#endif
716
717	/** Adds a segment attribute register as input to the outgoing state. */
718	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
719	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
720	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
721	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
722	} while (0)
723	#else
724	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
725	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
726	} while (0)
727	#endif
728
729	/** Adds a segment limit register as input to the outgoing state. */
730	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
731	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
732	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
733	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
734	} while (0)
735	#else
736	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
737	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
738	} while (0)
739	#endif
740
741	/** Adds a segment limit register as input to the outgoing state. */
742	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
743	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
744	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
745	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
746	} while (0)
747	#else
748	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
749	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
750	} while (0)
751	#endif
752	/** @} */
753
754	/**
755	* Guest registers that can be shadowed in GPRs.
756	*
757	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
758	* must be placed last, as the liveness state tracks it as 7 subcomponents and
759	* we don't want to waste space here.
760	*
761	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
762	* friends as well as IEMAllN8veLiveness.cpp.
763	*/
764	typedef enum IEMNATIVEGSTREG : uint8_t
765	{
766	kIemNativeGstReg_GprFirst = 0,
767	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
768	kIemNativeGstReg_Pc,
769	kIemNativeGstReg_Cr0,
770	kIemNativeGstReg_FpuFcw,
771	kIemNativeGstReg_FpuFsw,
772	kIemNativeGstReg_SegBaseFirst,
773	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
774	kIemNativeGstReg_SegAttribFirst,
775	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
776	kIemNativeGstReg_SegLimitFirst,
777	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
778	kIemNativeGstReg_SegSelFirst,
779	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
780	kIemNativeGstReg_Cr4,
781	kIemNativeGstReg_Xcr0,
782	kIemNativeGstReg_MxCsr,
783	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
784	kIemNativeGstReg_End
785	} IEMNATIVEGSTREG;
786	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
787	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
788
789	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
790	* @{ */
791	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
792	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
793	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
794	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
795	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
796	/** @} */
797
798	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
799
800	/**
801	* Guest registers that can be shadowed in host SIMD registers.
802	*
803	* @todo r=aeichner Liveness tracking
804	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
805	*/
806	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
807	{
808	kIemNativeGstSimdReg_SimdRegFirst = 0,
809	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
810	kIemNativeGstSimdReg_End
811	} IEMNATIVEGSTSIMDREG;
812
813	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
814	* @{ */
815	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
816	/** @} */
817
818	/**
819	* The Load/store size for a SIMD guest register.
820	*/
821	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
822	{
823	/** Invalid size. */
824	kIemNativeGstSimdRegLdStSz_Invalid = 0,
825	/** Loads the low 128-bit of a guest SIMD register. */
826	kIemNativeGstSimdRegLdStSz_Low128,
827	/** Loads the high 128-bit of a guest SIMD register. */
828	kIemNativeGstSimdRegLdStSz_High128,
829	/** Loads the whole 256-bits of a guest SIMD register. */
830	kIemNativeGstSimdRegLdStSz_256,
831	/** End value. */
832	kIemNativeGstSimdRegLdStSz_End
833	} IEMNATIVEGSTSIMDREGLDSTSZ;
834
835	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
836
837	/**
838	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
839	*/
840	typedef enum IEMNATIVEGSTREGUSE
841	{
842	/** The usage is read-only, the register holding the guest register
843	* shadow copy will not be modified by the caller. */
844	kIemNativeGstRegUse_ReadOnly = 0,
845	/** The caller will update the guest register (think: PC += cbInstr).
846	* The guest shadow copy will follow the returned register. */
847	kIemNativeGstRegUse_ForUpdate,
848	/** The call will put an entirely new value in the guest register, so
849	* if new register is allocate it will be returned uninitialized. */
850	kIemNativeGstRegUse_ForFullWrite,
851	/** The caller will use the guest register value as input in a calculation
852	* and the host register will be modified.
853	* This means that the returned host register will not be marked as a shadow
854	* copy of the guest register. */
855	kIemNativeGstRegUse_Calculation
856	} IEMNATIVEGSTREGUSE;
857
858	/**
859	* Guest registers (classes) that can be referenced.
860	*/
861	typedef enum IEMNATIVEGSTREGREF : uint8_t
862	{
863	kIemNativeGstRegRef_Invalid = 0,
864	kIemNativeGstRegRef_Gpr,
865	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
866	kIemNativeGstRegRef_EFlags,
867	kIemNativeGstRegRef_MxCsr,
868	kIemNativeGstRegRef_FpuReg,
869	kIemNativeGstRegRef_MReg,
870	kIemNativeGstRegRef_XReg,
871	//kIemNativeGstRegRef_YReg, - doesn't work.
872	kIemNativeGstRegRef_End
873	} IEMNATIVEGSTREGREF;
874
875
876	/** Variable kinds. */
877	typedef enum IEMNATIVEVARKIND : uint8_t
878	{
879	/** Customary invalid zero value. */
880	kIemNativeVarKind_Invalid = 0,
881	/** This is either in a register or on the stack. */
882	kIemNativeVarKind_Stack,
883	/** Immediate value - loaded into register when needed, or can live on the
884	* stack if referenced (in theory). */
885	kIemNativeVarKind_Immediate,
886	/** Variable reference - loaded into register when needed, never stack. */
887	kIemNativeVarKind_VarRef,
888	/** Guest register reference - loaded into register when needed, never stack. */
889	kIemNativeVarKind_GstRegRef,
890	/** End of valid values. */
891	kIemNativeVarKind_End
892	} IEMNATIVEVARKIND;
893
894
895	/** Variable or argument. */
896	typedef struct IEMNATIVEVAR
897	{
898	/** The kind of variable. */
899	IEMNATIVEVARKIND enmKind;
900	/** The variable size in bytes. */
901	uint8_t cbVar;
902	/** The first stack slot (uint64_t), except for immediate and references
903	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
904	* has a stack slot it has been initialized and has a value. Unused variables
905	* has neither a stack slot nor a host register assignment. */
906	uint8_t idxStackSlot;
907	/** The host register allocated for the variable, UINT8_MAX if not. */
908	uint8_t idxReg;
909	/** The argument number if argument, UINT8_MAX if regular variable. */
910	uint8_t uArgNo;
911	/** If referenced, the index (unpacked) of the variable referencing this one,
912	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
913	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
914	uint8_t idxReferrerVar;
915	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
916	* @todo not sure what this really is for... */
917	IEMNATIVEGSTREG enmGstReg;
918	/** Set if the registered is currently used exclusively, false if the
919	* variable is idle and the register can be grabbed. */
920	bool fRegAcquired;
921
922	union
923	{
924	/** kIemNativeVarKind_Immediate: The immediate value. */
925	uint64_t uValue;
926	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
927	uint8_t idxRefVar;
928	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
929	struct
930	{
931	/** The class of register. */
932	IEMNATIVEGSTREGREF enmClass;
933	/** Index within the class. */
934	uint8_t idx;
935	} GstRegRef;
936	} u;
937	} IEMNATIVEVAR;
938	/** Pointer to a variable or argument. */
939	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
940	/** Pointer to a const variable or argument. */
941	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
942
943	/** What is being kept in a host register. */
944	typedef enum IEMNATIVEWHAT : uint8_t
945	{
946	/** The traditional invalid zero value. */
947	kIemNativeWhat_Invalid = 0,
948	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
949	kIemNativeWhat_Var,
950	/** Temporary register, this is typically freed when a MC completes. */
951	kIemNativeWhat_Tmp,
952	/** Call argument w/o a variable mapping. This is free (via
953	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
954	kIemNativeWhat_Arg,
955	/** Return status code.
956	* @todo not sure if we need this... */
957	kIemNativeWhat_rc,
958	/** The fixed pVCpu (PVMCPUCC) register.
959	* @todo consider offsetting this on amd64 to use negative offsets to access
960	* more members using 8-byte disp. */
961	kIemNativeWhat_pVCpuFixed,
962	/** The fixed pCtx (PCPUMCTX) register.
963	* @todo consider offsetting this on amd64 to use negative offsets to access
964	* more members using 8-byte disp. */
965	kIemNativeWhat_pCtxFixed,
966	/** Fixed temporary register. */
967	kIemNativeWhat_FixedTmp,
968	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
969	/** Shadow RIP for the delayed RIP updating debugging. */
970	kIemNativeWhat_PcShadow,
971	#endif
972	/** Register reserved by the CPU or OS architecture. */
973	kIemNativeWhat_FixedReserved,
974	/** End of valid values. */
975	kIemNativeWhat_End
976	} IEMNATIVEWHAT;
977
978	/**
979	* Host general register entry.
980	*
981	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
982	*
983	* @todo Track immediate values in host registers similarlly to how we track the
984	* guest register shadow copies. For it to be real helpful, though,
985	* we probably need to know which will be reused and put them into
986	* non-volatile registers, otherwise it's going to be more or less
987	* restricted to an instruction or two.
988	*/
989	typedef struct IEMNATIVEHSTREG
990	{
991	/** Set of guest registers this one shadows.
992	*
993	* Using a bitmap here so we can designate the same host register as a copy
994	* for more than one guest register. This is expected to be useful in
995	* situations where one value is copied to several registers in a sequence.
996	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
997	* sequence we'd want to let this register follow to be a copy of and there
998	* will always be places where we'd be picking the wrong one.
999	*/
1000	uint64_t fGstRegShadows;
1001	/** What is being kept in this register. */
1002	IEMNATIVEWHAT enmWhat;
1003	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1004	uint8_t idxVar;
1005	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1006	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1007	* that scope. */
1008	uint8_t idxStackSlot;
1009	/** Alignment padding. */
1010	uint8_t abAlign[5];
1011	} IEMNATIVEHSTREG;
1012
1013
1014	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1015	/**
1016	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1017	* halves, on architectures where there is no 256-bit register available this entry will track
1018	* two adjacent 128-bit host registers.
1019	*
1020	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1021	*/
1022	typedef struct IEMNATIVEHSTSIMDREG
1023	{
1024	/** Set of guest registers this one shadows.
1025	*
1026	* Using a bitmap here so we can designate the same host register as a copy
1027	* for more than one guest register. This is expected to be useful in
1028	* situations where one value is copied to several registers in a sequence.
1029	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1030	* sequence we'd want to let this register follow to be a copy of and there
1031	* will always be places where we'd be picking the wrong one.
1032	*/
1033	uint64_t fGstRegShadows;
1034	/** What is being kept in this register. */
1035	IEMNATIVEWHAT enmWhat;
1036	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1037	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1038	/** Alignment padding. */
1039	uint8_t abAlign[6];
1040	} IEMNATIVEHSTSIMDREG;
1041	#endif
1042
1043
1044	/**
1045	* Core state for the native recompiler, that is, things that needs careful
1046	* handling when dealing with branches.
1047	*/
1048	typedef struct IEMNATIVECORESTATE
1049	{
1050	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1051	/** The current instruction offset in bytes from when the guest program counter
1052	* was updated last. Used for delaying the write to the guest context program counter
1053	* as long as possible. */
1054	uint32_t offPc;
1055	/** Number of instructions where we could skip the updating. */
1056	uint32_t cInstrPcUpdateSkipped;
1057	#endif
1058	/** Allocation bitmap for aHstRegs. */
1059	uint32_t bmHstRegs;
1060
1061	/** Bitmap marking which host register contains guest register shadow copies.
1062	* This is used during register allocation to try preserve copies. */
1063	uint32_t bmHstRegsWithGstShadow;
1064	/** Bitmap marking valid entries in aidxGstRegShadows. */
1065	uint64_t bmGstRegShadows;
1066
1067	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1068	/** Allocation bitmap for aHstSimdRegs. */
1069	uint32_t bmHstSimdRegs;
1070
1071	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1072	* This is used during register allocation to try preserve copies. */
1073	uint32_t bmHstSimdRegsWithGstShadow;
1074	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1075	uint64_t bmGstSimdRegShadows;
1076	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1077	uint64_t bmGstSimdRegShadowDirtyLo128;
1078	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1079	uint64_t bmGstSimdRegShadowDirtyHi128;
1080	#endif
1081
1082	union
1083	{
1084	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1085	uint8_t aidxArgVars[8];
1086	/** For more efficient resetting. */
1087	uint64_t u64ArgVars;
1088	};
1089
1090	/** Allocation bitmap for the stack. */
1091	uint32_t bmStack;
1092	/** Allocation bitmap for aVars. */
1093	uint32_t bmVars;
1094
1095	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1096	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1097	* (A shadow copy of a guest register can only be held in a one host register,
1098	* there are no duplicate copies or ambiguities like that). */
1099	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1100	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1101	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1102	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1103	* (A shadow copy of a guest register can only be held in a one host register,
1104	* there are no duplicate copies or ambiguities like that). */
1105	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1106	#endif
1107
1108	/** Host register allocation tracking. */
1109	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1110	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1111	/** Host SIMD register allocation tracking. */
1112	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1113	#endif
1114
1115	/** Variables and arguments. */
1116	IEMNATIVEVAR aVars[9];
1117	} IEMNATIVECORESTATE;
1118	/** Pointer to core state. */
1119	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1120	/** Pointer to const core state. */
1121	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1122
1123	/** @def IEMNATIVE_VAR_IDX_UNPACK
1124	* @returns Index into IEMNATIVECORESTATE::aVars.
1125	* @param a_idxVar Variable index w/ magic (in strict builds).
1126	*/
1127	/** @def IEMNATIVE_VAR_IDX_PACK
1128	* @returns Variable index w/ magic (in strict builds).
1129	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1130	*/
1131	#ifdef VBOX_STRICT
1132	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1133	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1134	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1135	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1136	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1137	#else
1138	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1139	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1140	#endif
1141
1142
1143	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1144	/** Clear the dirty state of the given guest SIMD register. */
1145	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1146	do { \
1147	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1148	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1149	} while (0)
1150
1151	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1152	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1153	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1154	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1155	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1156	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1157	/** Returns whether the given guest SIMD register is dirty. */
1158	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1159	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1160
1161	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1162	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1163	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1164	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1165	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1166	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1167	#endif
1168
1169
1170	/**
1171	* Conditional stack entry.
1172	*/
1173	typedef struct IEMNATIVECOND
1174	{
1175	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1176	bool fInElse;
1177	/** The label for the IEM_MC_ELSE. */
1178	uint32_t idxLabelElse;
1179	/** The label for the IEM_MC_ENDIF. */
1180	uint32_t idxLabelEndIf;
1181	/** The initial state snapshot as the if-block starts executing. */
1182	IEMNATIVECORESTATE InitialState;
1183	/** The state snapshot at the end of the if-block. */
1184	IEMNATIVECORESTATE IfFinalState;
1185	} IEMNATIVECOND;
1186	/** Pointer to a condition stack entry. */
1187	typedef IEMNATIVECOND *PIEMNATIVECOND;
1188
1189
1190	/**
1191	* Native recompiler state.
1192	*/
1193	typedef struct IEMRECOMPILERSTATE
1194	{
1195	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1196	* IEMNATIVEINSTR units. */
1197	uint32_t cInstrBufAlloc;
1198	#ifdef VBOX_STRICT
1199	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1200	uint32_t offInstrBufChecked;
1201	#else
1202	uint32_t uPadding1; /* We don't keep track of the size here... */
1203	#endif
1204	/** Fixed temporary code buffer for native recompilation. */
1205	PIEMNATIVEINSTR pInstrBuf;
1206
1207	/** Bitmaps with the label types used. */
1208	uint64_t bmLabelTypes;
1209	/** Actual number of labels in paLabels. */
1210	uint32_t cLabels;
1211	/** Max number of entries allowed in paLabels before reallocating it. */
1212	uint32_t cLabelsAlloc;
1213	/** Labels defined while recompiling (referenced by fixups). */
1214	PIEMNATIVELABEL paLabels;
1215	/** Array with indexes of unique labels (uData always 0). */
1216	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1217
1218	/** Actual number of fixups paFixups. */
1219	uint32_t cFixups;
1220	/** Max number of entries allowed in paFixups before reallocating it. */
1221	uint32_t cFixupsAlloc;
1222	/** Buffer used by the recompiler for recording fixups when generating code. */
1223	PIEMNATIVEFIXUP paFixups;
1224
1225	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1226	/** Number of debug info entries allocated for pDbgInfo. */
1227	uint32_t cDbgInfoAlloc;
1228	uint32_t uPadding;
1229	/** Debug info. */
1230	PIEMTBDBG pDbgInfo;
1231	#endif
1232
1233	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1234	/** The current call index (liveness array and threaded calls in TB). */
1235	uint32_t idxCurCall;
1236	/** Number of liveness entries allocated. */
1237	uint32_t cLivenessEntriesAlloc;
1238	/** Liveness entries for all the calls in the TB begin recompiled.
1239	* The entry for idxCurCall contains the info for what the next call will
1240	* require wrt registers. (Which means the last entry is the initial liveness
1241	* state.) */
1242	PIEMLIVENESSENTRY paLivenessEntries;
1243	#endif
1244
1245	/** The translation block being recompiled. */
1246	PCIEMTB pTbOrg;
1247	/** The VMCPU structure of the EMT. */
1248	PVMCPUCC pVCpu;
1249
1250	/** Condition sequence number (for generating unique labels). */
1251	uint16_t uCondSeqNo;
1252	/** Check IRQ seqeunce number (for generating unique labels). */
1253	uint16_t uCheckIrqSeqNo;
1254	/** TLB load sequence number (for generating unique labels). */
1255	uint16_t uTlbSeqNo;
1256	/** The current condition stack depth (aCondStack). */
1257	uint8_t cCondDepth;
1258
1259	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1260	uint8_t cArgs;
1261	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1262	uint32_t fCImpl;
1263	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1264	uint32_t fMc;
1265	/** The expected IEMCPU::fExec value for the current call/instruction. */
1266	uint32_t fExec;
1267
1268	/** Core state requiring care with branches. */
1269	IEMNATIVECORESTATE Core;
1270
1271	/** The condition nesting stack. */
1272	IEMNATIVECOND aCondStack[2];
1273
1274	#ifndef IEM_WITH_THROW_CATCH
1275	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1276	* for recompilation error handling. */
1277	jmp_buf JmpBuf;
1278	#endif
1279	} IEMRECOMPILERSTATE;
1280	/** Pointer to a native recompiler state. */
1281	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1282
1283
1284	/** @def IEMNATIVE_TRY_SETJMP
1285	* Wrapper around setjmp / try, hiding all the ugly differences.
1286	*
1287	* @note Use with extreme care as this is a fragile macro.
1288	* @param a_pReNative The native recompile state.
1289	* @param a_rcTarget The variable that should receive the status code in case
1290	* of a longjmp/throw.
1291	*/
1292	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1293	* Start wrapper for catch / setjmp-else.
1294	*
1295	* This will set up a scope.
1296	*
1297	* @note Use with extreme care as this is a fragile macro.
1298	* @param a_pReNative The native recompile state.
1299	* @param a_rcTarget The variable that should receive the status code in case
1300	* of a longjmp/throw.
1301	*/
1302	/** @def IEMNATIVE_CATCH_LONGJMP_END
1303	* End wrapper for catch / setjmp-else.
1304	*
1305	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1306	* up the state.
1307	*
1308	* @note Use with extreme care as this is a fragile macro.
1309	* @param a_pReNative The native recompile state.
1310	*/
1311	/** @def IEMNATIVE_DO_LONGJMP
1312	*
1313	* Wrapper around longjmp / throw.
1314	*
1315	* @param a_pReNative The native recompile state.
1316	* @param a_rc The status code jump back with / throw.
1317	*/
1318	#ifdef IEM_WITH_THROW_CATCH
1319	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1320	a_rcTarget = VINF_SUCCESS; \
1321	try
1322	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1323	catch (int rcThrown) \
1324	{ \
1325	a_rcTarget = rcThrown
1326	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1327	} \
1328	((void)0)
1329	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1330	#else /* !IEM_WITH_THROW_CATCH */
1331	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1332	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1333	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1334	else \
1335	{ \
1336	((void)0)
1337	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1338	}
1339	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1340	#endif /* !IEM_WITH_THROW_CATCH */
1341
1342
1343	/**
1344	* Native recompiler worker for a threaded function.
1345	*
1346	* @returns New code buffer offset; throws VBox status code in case of a failure.
1347	* @param pReNative The native recompiler state.
1348	* @param off The current code buffer offset.
1349	* @param pCallEntry The threaded call entry.
1350	*
1351	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1352	*/
1353	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1354	/** Pointer to a native recompiler worker for a threaded function. */
1355	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1356
1357	/** Defines a native recompiler worker for a threaded function.
1358	* @see FNIEMNATIVERECOMPFUNC */
1359	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1360	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1361
1362	/** Prototypes a native recompiler function for a threaded function.
1363	* @see FNIEMNATIVERECOMPFUNC */
1364	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1365
1366
1367	/**
1368	* Native recompiler liveness analysis worker for a threaded function.
1369	*
1370	* @param pCallEntry The threaded call entry.
1371	* @param pIncoming The incoming liveness state entry.
1372	* @param pOutgoing The outgoing liveness state entry.
1373	*/
1374	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1375	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1376	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1377	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1378
1379	/** Defines a native recompiler liveness analysis worker for a threaded function.
1380	* @see FNIEMNATIVELIVENESSFUNC */
1381	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1382	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1383
1384	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1385	* @see FNIEMNATIVELIVENESSFUNC */
1386	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1387
1388
1389	/** Define a native recompiler helper function, safe to call from the TB code. */
1390	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1391	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1392	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1393	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1394	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1395
1396
1397	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1398	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1399	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1400	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1401	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1402	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1403	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1404	uint8_t idxHstSimdReg = UINT8_MAX,
1405	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1406	# endif
1407	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1408	uint32_t offPc, uint32_t cInstrSkipped);
1409	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1410
1411	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1412	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1413	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1414	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1415	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1416	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1417
1418	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1419	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1420	bool fPreferVolatile = true);
1421	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1422	bool fPreferVolatile = true);
1423	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1424	IEMNATIVEGSTREG enmGstReg,
1425	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1426	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1427	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1428	IEMNATIVEGSTREG enmGstReg);
1429
1430	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1431	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1432	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_X86)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1433	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1434	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1435	#endif
1436	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1437	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1438	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1439	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1440	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1441	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1442	uint32_t fKeepVars = 0);
1443	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1444	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1445	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1446	uint32_t fHstRegsActiveShadows);
1447	#ifdef VBOX_STRICT
1448	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1449	#endif
1450	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1451	uint64_t fGstShwExcept, bool fFlushShadows);
1452	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1453	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1454	#endif
1455
1456
1457	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1458	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1459	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1460	bool fPreferVolatile = true);
1461	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1462	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1463	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1464	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1465	bool fNoVolatileRegs = false);
1466	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1467	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1468	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1469	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1470	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1471	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1472	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1473	#endif
1474
1475	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1476	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1477	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1478	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1479	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1480	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1481	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1482	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1483	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1484	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1485	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1486	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1487	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1488	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1489	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1490	uint32_t fHstRegsNotToSave);
1491	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1492	uint32_t fHstRegsNotToSave);
1493	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1494	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1495
1496	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1497	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1498	#ifdef VBOX_STRICT
1499	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1500	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1501	IEMNATIVEGSTREG enmGstReg);
1502	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1503	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1504	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1505	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1506	# endif
1507	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1508	#endif
1509	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1510	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs);
1511	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1512	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1513	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1514	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1515	PCIEMTHRDEDCALLENTRY pCallEntry);
1516	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1517	uint8_t idxAddrReg, uint8_t idxInstr);
1518	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1519	uint8_t idxAddrReg, uint8_t idxInstr);
1520
1521
1522	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1523	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1524	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1525	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1526	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1527	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1528	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1529	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1530	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1531
1532	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1533	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1534	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1535	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1536	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1537	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1538	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1539	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1540	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1541	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1542	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1543	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1544	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1545	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1546	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1547	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1548	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1549	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1550	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1551	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1552	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1553
1554	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1555	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1556	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1557	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1558	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1559	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1560	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1561	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1562	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1563	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1564	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1565	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1566	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1567	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1568	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1569	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1570	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1571	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1572	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1573	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1574	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1575
1576	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1577	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1578	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1579	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1580	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1581	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1582	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1583	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1584	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1585	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1586	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1587	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1588	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1589	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1590	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1591	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1592	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1593	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1594	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1595	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1596	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1597	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1598
1599	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1600	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1601	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1602	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1603	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1604	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1605	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1606	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1607	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1608	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1609	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1610	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1611	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1612	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1613	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1614	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1615	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1616	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1617	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1618	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1619	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1620	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1621
1622	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1623	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1624	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1625	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1626
1627
1628	/**
1629	* Info about shadowed guest register values.
1630	* @see IEMNATIVEGSTREG
1631	*/
1632	typedef struct IEMANTIVEGSTREGINFO
1633	{
1634	/** Offset in VMCPU. */
1635	uint32_t off;
1636	/** The field size. */
1637	uint8_t cb;
1638	/** Name (for logging). */
1639	const char *pszName;
1640	} IEMANTIVEGSTREGINFO;
1641	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1642	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1643	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1644	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1645	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1646
1647
1648
1649	/**
1650	* Ensures that there is sufficient space in the instruction output buffer.
1651	*
1652	* This will reallocate the buffer if needed and allowed.
1653	*
1654	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1655	* allocation size.
1656	*
1657	* @returns Pointer to the instruction output buffer on success; throws VBox
1658	* status code on failure, so no need to check it.
1659	* @param pReNative The native recompile state.
1660	* @param off Current instruction offset. Works safely for UINT32_MAX
1661	* as well.
1662	* @param cInstrReq Number of instruction about to be added. It's okay to
1663	* overestimate this a bit.
1664	*/
1665	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1666	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1667	{
1668	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1669	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1670	{
1671	#ifdef VBOX_STRICT
1672	pReNative->offInstrBufChecked = offChecked;
1673	#endif
1674	return pReNative->pInstrBuf;
1675	}
1676	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1677	}
1678
1679	/**
1680	* Checks that we didn't exceed the space requested in the last
1681	* iemNativeInstrBufEnsure() call.
1682	*/
1683	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1684	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1685	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1686
1687	/**
1688	* Checks that a variable index is valid.
1689	*/
1690	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1691	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1692	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1693	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1694	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1695	("%s=%#x\n", #a_idxVar, a_idxVar))
1696	#else
1697	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1698	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1699	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1700	#endif
1701
1702	/**
1703	* Checks that a variable index is valid and that the variable is assigned the
1704	* correct argument number.
1705	* This also adds a RT_NOREF of a_idxVar.
1706	*/
1707	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1708	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1709	RT_NOREF_PV(a_idxVar); \
1710	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1711	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1712	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1713	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1714	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1715	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1716	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1717	a_uArgNo)); \
1718	} while (0)
1719	#else
1720	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1721	RT_NOREF_PV(a_idxVar); \
1722	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1723	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1724	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1725	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1726	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1727	} while (0)
1728	#endif
1729
1730
1731	/**
1732	* Checks that a variable has the expected size.
1733	*/
1734	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1735	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1736	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1737	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1738
1739
1740	/**
1741	* Calculates the stack address of a variable as a [r]BP displacement value.
1742	*/
1743	DECL_FORCE_INLINE(int32_t)
1744	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1745	{
1746	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1747	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1748	}
1749
1750
1751	/**
1752	* Releases the variable's register.
1753	*
1754	* The register must have been previously acquired calling
1755	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1756	* iemNativeVarRegisterSetAndAcquire().
1757	*/
1758	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1759	{
1760	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1761	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1762	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1763	}
1764
1765
1766	/**
1767	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
1768	*
1769	* @returns The flush mask.
1770	* @param fCImpl The IEM_CIMPL_F_XXX flags.
1771	* @param fGstShwFlush The starting flush mask.
1772	*/
1773	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
1774	{
1775	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
1776	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
1777	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
1778	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
1779	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
1780	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
1781	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
1782	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
1783	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
1784	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
1785	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
1786	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
1787	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
1788	return fGstShwFlush;
1789	}
1790
1791
1792	/** Number of hidden arguments for CIMPL calls.
1793	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
1794	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
1795	# define IEM_CIMPL_HIDDEN_ARGS 3
1796	#else
1797	# define IEM_CIMPL_HIDDEN_ARGS 2
1798	#endif
1799
1800
1801	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1802
1803	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1804	/**
1805	* Helper for iemNativeLivenessGetStateByGstReg.
1806	*
1807	* @returns IEMLIVENESS_STATE_XXX
1808	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
1809	* ORed together.
1810	*/
1811	DECL_FORCE_INLINE(uint32_t)
1812	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
1813	{
1814	/* INPUT trumps anything else. */
1815	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
1816	return IEMLIVENESS_STATE_INPUT;
1817
1818	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
1819	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
1820	{
1821	/* If not all sub-fields are clobbered they must be considered INPUT. */
1822	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
1823	return IEMLIVENESS_STATE_INPUT;
1824	return IEMLIVENESS_STATE_CLOBBERED;
1825	}
1826
1827	/* XCPT_OR_CALL trumps UNUSED. */
1828	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
1829	return IEMLIVENESS_STATE_XCPT_OR_CALL;
1830
1831	return IEMLIVENESS_STATE_UNUSED;
1832	}
1833	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
1834
1835
1836	DECL_FORCE_INLINE(uint32_t)
1837	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
1838	{
1839	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1840	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1841	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
1842	# else
1843	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
1844	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
1845	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
1846	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
1847	# endif
1848	}
1849
1850
1851	DECL_FORCE_INLINE(uint32_t)
1852	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
1853	{
1854	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
1855	if (enmGstReg == kIemNativeGstReg_EFlags)
1856	{
1857	/* Merge the eflags states to one. */
1858	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
1859	uRet = RT_BIT_32(uRet);
1860	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
1861	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
1862	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
1863	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
1864	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
1865	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
1866	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
1867	# else
1868	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
1869	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
1870	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
1871	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
1872	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
1873	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
1874	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
1875	# endif
1876	}
1877	return uRet;
1878	}
1879
1880
1881	# ifdef VBOX_STRICT
1882	/** For assertions only, user checks that idxCurCall isn't zerow. */
1883	DECL_FORCE_INLINE(uint32_t)
1884	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
1885	{
1886	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
1887	}
1888	# endif /* VBOX_STRICT */
1889
1890	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
1891
1892
1893	/**
1894	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
1895	*/
1896	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
1897	{
1898	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
1899	return IEM_CIMPL_HIDDEN_ARGS;
1900	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
1901	return 1;
1902	return 0;
1903	}
1904
1905
1906	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
1907	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
1908	{
1909	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
1910
1911	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
1912	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
1913	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
1914	return (uint8_t)idxReg;
1915	}
1916
1917
1918
1919	/*********************************************************************************************************************************
1920	* Register Allocator (GPR) *
1921	*********************************************************************************************************************************/
1922
1923	/**
1924	* Marks host register @a idxHstReg as containing a shadow copy of guest
1925	* register @a enmGstReg.
1926	*
1927	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
1928	* host register before calling.
1929	*/
1930	DECL_FORCE_INLINE(void)
1931	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
1932	{
1933	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
1934	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
1935	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
1936
1937	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
1938	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
1939	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
1940	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
1941	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1942	iemNativeDbgInfoAddNativeOffset(pReNative, off);
1943	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
1944	#else
1945	RT_NOREF(off);
1946	#endif
1947	}
1948
1949
1950	/**
1951	* Clear any guest register shadow claims from @a idxHstReg.
1952	*
1953	* The register does not need to be shadowing any guest registers.
1954	*/
1955	DECL_FORCE_INLINE(void)
1956	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
1957	{
1958	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
1959	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
1960	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
1961	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
1962	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
1963
1964	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1965	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
1966	if (fGstRegs)
1967	{
1968	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
1969	iemNativeDbgInfoAddNativeOffset(pReNative, off);
1970	while (fGstRegs)
1971	{
1972	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
1973	fGstRegs &= ~RT_BIT_64(iGstReg);
1974	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
1975	}
1976	}
1977	#else
1978	RT_NOREF(off);
1979	#endif
1980
1981	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
1982	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
1983	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
1984	}
1985
1986
1987	/**
1988	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
1989	* and global overview flags.
1990	*/
1991	DECL_FORCE_INLINE(void)
1992	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
1993	{
1994	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
1995	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
1996	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
1997	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
1998	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
1999	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2000	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2001
2002	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2003	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2004	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2005	#else
2006	RT_NOREF(off);
2007	#endif
2008
2009	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2010	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2011	if (!fGstRegShadowsNew)
2012	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2013	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2014	}
2015
2016
2017	#if 0 /* unused */
2018	/**
2019	* Clear any guest register shadow claim for @a enmGstReg.
2020	*/
2021	DECL_FORCE_INLINE(void)
2022	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2023	{
2024	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2025	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2026	{
2027	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2028	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2029	}
2030	}
2031	#endif
2032
2033
2034	/**
2035	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2036	* as the new shadow of it.
2037	*
2038	* Unlike the other guest reg shadow helpers, this does the logging for you.
2039	* However, it is the liveness state is not asserted here, the caller must do
2040	* that.
2041	*/
2042	DECL_FORCE_INLINE(void)
2043	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2044	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2045	{
2046	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2047	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2048	{
2049	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2050	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2051	if (idxHstRegOld == idxHstRegNew)
2052	return;
2053	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2054	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2055	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2056	}
2057	else
2058	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2059	g_aGstShadowInfo[enmGstReg].pszName));
2060	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2061	}
2062
2063
2064	/**
2065	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2066	* to @a idxRegTo.
2067	*/
2068	DECL_FORCE_INLINE(void)
2069	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2070	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2071	{
2072	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2073	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2074	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2075	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2076	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2077	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2078	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2079	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2080	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2081
2082	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2083	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2084	if (!fGstRegShadowsFrom)
2085	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2086	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2087	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2088	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2089	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2090	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2091	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2092	#else
2093	RT_NOREF(off);
2094	#endif
2095	}
2096
2097
2098	/**
2099	* Flushes any delayed guest register writes.
2100	*
2101	* This must be called prior to calling CImpl functions and any helpers that use
2102	* the guest state (like raising exceptions) and such.
2103	*
2104	* This optimization has not yet been implemented. The first target would be
2105	* RIP updates, since these are the most common ones.
2106	*/
2107	DECL_INLINE_THROW(uint32_t)
2108	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, bool fFlushShadows = true)
2109	{
2110	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2111	if (!(fGstShwExcept & kIemNativeGstReg_Pc))
2112	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2113	#else
2114	RT_NOREF(pReNative, fGstShwExcept);
2115	#endif
2116
2117	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2118	/** @todo r=bird: There must be a quicker way to check if anything needs doing here! */
2119	/** @todo This doesn't mix well with fGstShwExcept but we ignore this for now and just flush everything. */
2120	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fFlushShadows);
2121	#else
2122	RT_NOREF(pReNative, fGstShwExcept, fFlushShadows);
2123	return off;
2124	#endif
2125	}
2126
2127
2128
2129	/*********************************************************************************************************************************
2130	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2131	*********************************************************************************************************************************/
2132
2133	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2134
2135	DECL_FORCE_INLINE(uint8_t)
2136	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2137	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2138	{
2139	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2140
2141	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2142	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2143	RT_NOREF(idxVar);
2144	return idxSimdReg;
2145	}
2146
2147
2148	/**
2149	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2150	* SIMD register @a enmGstSimdReg.
2151	*
2152	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2153	* host register before calling.
2154	*/
2155	DECL_FORCE_INLINE(void)
2156	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2157	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2158	{
2159	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2160	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2161	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2162
2163	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2164	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2165	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2166	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2167	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2168	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2169	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2170	#else
2171	RT_NOREF(off);
2172	#endif
2173	}
2174
2175
2176	/**
2177	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2178	* to @a idxSimdRegTo.
2179	*/
2180	DECL_FORCE_INLINE(void)
2181	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2182	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2183	{
2184	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2185	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2186	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2187	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2188	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2189	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2190	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2191	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2192	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2193	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2194	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2195
2196	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2197	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2198	if (!fGstRegShadowsFrom)
2199	{
2200	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2201	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2202	}
2203	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2204	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2205	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2206	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2207	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2208	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2209	#else
2210	RT_NOREF(off);
2211	#endif
2212	}
2213
2214
2215	/**
2216	* Clear any guest register shadow claims from @a idxHstSimdReg.
2217	*
2218	* The register does not need to be shadowing any guest registers.
2219	*/
2220	DECL_FORCE_INLINE(void)
2221	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2222	{
2223	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2224	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2225	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2226	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2227	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2228	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2229	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2230
2231	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2232	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2233	if (fGstRegs)
2234	{
2235	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2236	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2237	while (fGstRegs)
2238	{
2239	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2240	fGstRegs &= ~RT_BIT_64(iGstReg);
2241	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2242	}
2243	}
2244	#else
2245	RT_NOREF(off);
2246	#endif
2247
2248	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2249	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2250	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2251	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2252	}
2253
2254	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2255
2256
2257	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2258	/**
2259	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2260	*/
2261	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2262	{
2263	if (pReNative->Core.offPc)
2264	return iemNativeEmitPcWritebackSlow(pReNative, off);
2265	return off;
2266	}
2267	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2268
2269
2270	/** @} */
2271
2272	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2273

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103809

Download in other formats: