IEMN8veRecompiler.h@ 103649

Last change on this file since 103649 was 103649, checked in by vboxsync, 12 months ago
VMM/IEM: Delay the RIP update for each instruction if possible to save on potentially two instructions for each executed guest instruction of which one is a memory write, bugref:10373
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 61.1 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103649 2024-03-03 07:11:39Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	#ifdef VBOX_WITH_STATISTICS
54	/** Always count instructions for now. */
55	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
56	#endif
57
58
59	/** @name Stack Frame Layout
60	*
61	* @{ */
62	/** The size of the area for stack variables and spills and stuff.
63	* @note This limit is duplicated in the python script(s). We add 0x40 for
64	* alignment padding. */
65	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
66	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
67	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
68	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
69
70	#ifdef RT_ARCH_AMD64
71	/** An stack alignment adjustment (between non-volatile register pushes and
72	* the stack variable area, so the latter better aligned). */
73	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
74
75	/** Number of stack arguments slots for calls made from the frame. */
76	# ifdef RT_OS_WINDOWS
77	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
78	# else
79	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
80	# endif
81	/** Number of any shadow arguments (spill area) for calls we make. */
82	# ifdef RT_OS_WINDOWS
83	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
84	# else
85	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
86	# endif
87
88	/** Frame pointer (RBP) relative offset of the last push. */
89	# ifdef RT_OS_WINDOWS
90	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
91	# else
92	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
93	# endif
94	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
95	* address for it). */
96	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
97	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
98	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
99	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
100	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
101	# ifdef RT_OS_WINDOWS
102	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
103	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
104	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
105	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
106	# endif
107
108	# ifdef RT_OS_WINDOWS
109	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
110	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
111	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
112	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
113	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
114	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
115	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
116	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
117	# endif
118
119	#elif RT_ARCH_ARM64
120	/** No alignment padding needed for arm64. */
121	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
122	/** No stack argument slots, got 8 registers for arguments will suffice. */
123	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
124	/** There are no argument spill area. */
125	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
126
127	/** Number of saved registers at the top of our stack frame.
128	* This includes the return address and old frame pointer, so x19 thru x30. */
129	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
130	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
131	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
132
133	/** Frame pointer (BP) relative offset of the last push. */
134	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
135
136	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
137	* address for it). */
138	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
139
140	#else
141	# error "port me"
142	#endif
143	/** @} */
144
145
146	/** @name Fixed Register Allocation(s)
147	* @{ */
148	/** @def IEMNATIVE_REG_FIXED_PVMCPU
149	* The number of the register holding the pVCpu pointer. */
150	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
151	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
152	* @note This not available on AMD64, only ARM64. */
153	/** @def IEMNATIVE_REG_FIXED_TMP0
154	* Dedicated temporary register.
155	* @todo replace this by a register allocator and content tracker. */
156	/** @def IEMNATIVE_REG_FIXED_MASK
157	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
158	* architecture. */
159	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
160	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
161	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
162	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
163	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
164	\| RT_BIT_32(X86_GREG_xSP) \
165	\| RT_BIT_32(X86_GREG_xBP) )
166
167	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
168	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
169	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
170	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
171	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
172	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
173	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
174	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
175	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
176	# else
177	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
178	# endif
179	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
180	\| RT_BIT_32(ARMV8_A64_REG_LR) \
181	\| RT_BIT_32(ARMV8_A64_REG_BP) \
182	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
183	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
184	\| RT_BIT_32(ARMV8_A64_REG_X18) \
185	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
186	\| IEMNATIVE_REG_FIXED_MASK_ADD)
187
188	#else
189	# error "port me"
190	#endif
191	/** @} */
192
193	/** @name Call related registers.
194	* @{ */
195	/** @def IEMNATIVE_CALL_RET_GREG
196	* The return value register. */
197	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
198	* Number of arguments in registers. */
199	/** @def IEMNATIVE_CALL_ARG0_GREG
200	* The general purpose register carrying argument \#0. */
201	/** @def IEMNATIVE_CALL_ARG1_GREG
202	* The general purpose register carrying argument \#1. */
203	/** @def IEMNATIVE_CALL_ARG2_GREG
204	* The general purpose register carrying argument \#2. */
205	/** @def IEMNATIVE_CALL_ARG3_GREG
206	* The general purpose register carrying argument \#3. */
207	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
208	* Mask of registers the callee will not save and may trash. */
209	#ifdef RT_ARCH_AMD64
210	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
211
212	# ifdef RT_OS_WINDOWS
213	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
214	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
215	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
216	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
217	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
218	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
219	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
220	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
221	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
222	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
223	\| RT_BIT_32(X86_GREG_xCX) \
224	\| RT_BIT_32(X86_GREG_xDX) \
225	\| RT_BIT_32(X86_GREG_x8) \
226	\| RT_BIT_32(X86_GREG_x9) \
227	\| RT_BIT_32(X86_GREG_x10) \
228	\| RT_BIT_32(X86_GREG_x11) )
229	# else
230	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
231	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
232	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
233	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
234	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
235	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
236	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
237	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
238	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
239	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
240	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
241	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
242	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
243	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
244	\| RT_BIT_32(X86_GREG_xCX) \
245	\| RT_BIT_32(X86_GREG_xDX) \
246	\| RT_BIT_32(X86_GREG_xDI) \
247	\| RT_BIT_32(X86_GREG_xSI) \
248	\| RT_BIT_32(X86_GREG_x8) \
249	\| RT_BIT_32(X86_GREG_x9) \
250	\| RT_BIT_32(X86_GREG_x10) \
251	\| RT_BIT_32(X86_GREG_x11) )
252	# endif
253
254	#elif defined(RT_ARCH_ARM64)
255	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
257	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
258	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
259	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
260	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
261	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
262	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
263	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
264	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
265	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
266	\| RT_BIT_32(ARMV8_A64_REG_X1) \
267	\| RT_BIT_32(ARMV8_A64_REG_X2) \
268	\| RT_BIT_32(ARMV8_A64_REG_X3) \
269	\| RT_BIT_32(ARMV8_A64_REG_X4) \
270	\| RT_BIT_32(ARMV8_A64_REG_X5) \
271	\| RT_BIT_32(ARMV8_A64_REG_X6) \
272	\| RT_BIT_32(ARMV8_A64_REG_X7) )
273	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
274	\| RT_BIT_32(ARMV8_A64_REG_X1) \
275	\| RT_BIT_32(ARMV8_A64_REG_X2) \
276	\| RT_BIT_32(ARMV8_A64_REG_X3) \
277	\| RT_BIT_32(ARMV8_A64_REG_X4) \
278	\| RT_BIT_32(ARMV8_A64_REG_X5) \
279	\| RT_BIT_32(ARMV8_A64_REG_X6) \
280	\| RT_BIT_32(ARMV8_A64_REG_X7) \
281	\| RT_BIT_32(ARMV8_A64_REG_X8) \
282	\| RT_BIT_32(ARMV8_A64_REG_X9) \
283	\| RT_BIT_32(ARMV8_A64_REG_X10) \
284	\| RT_BIT_32(ARMV8_A64_REG_X11) \
285	\| RT_BIT_32(ARMV8_A64_REG_X12) \
286	\| RT_BIT_32(ARMV8_A64_REG_X13) \
287	\| RT_BIT_32(ARMV8_A64_REG_X14) \
288	\| RT_BIT_32(ARMV8_A64_REG_X15) \
289	\| RT_BIT_32(ARMV8_A64_REG_X16) \
290	\| RT_BIT_32(ARMV8_A64_REG_X17) )
291
292	#endif
293
294	/** This is the maximum argument count we'll ever be needing. */
295	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
296	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
297	#else
298	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
299	#endif
300	/** @} */
301
302
303	/** @def IEMNATIVE_HST_GREG_COUNT
304	* Number of host general purpose registers we tracker. */
305	/** @def IEMNATIVE_HST_GREG_MASK
306	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
307	* inverted register masks and such to get down to a correct set of regs. */
308	#ifdef RT_ARCH_AMD64
309	# define IEMNATIVE_HST_GREG_COUNT 16
310	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
311
312	#elif defined(RT_ARCH_ARM64)
313	# define IEMNATIVE_HST_GREG_COUNT 32
314	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
315	#else
316	# error "Port me!"
317	#endif
318
319
320	/** Native code generator label types. */
321	typedef enum
322	{
323	kIemNativeLabelType_Invalid = 0,
324	/* Labels w/o data, only once instance per TB: */
325	kIemNativeLabelType_Return,
326	kIemNativeLabelType_ReturnBreak,
327	kIemNativeLabelType_ReturnWithFlags,
328	kIemNativeLabelType_NonZeroRetOrPassUp,
329	kIemNativeLabelType_RaiseGp0,
330	kIemNativeLabelType_RaiseNm,
331	kIemNativeLabelType_RaiseUd,
332	kIemNativeLabelType_ObsoleteTb,
333	kIemNativeLabelType_NeedCsLimChecking,
334	kIemNativeLabelType_CheckBranchMiss,
335	/* Labels with data, potentially multiple instances per TB: */
336	kIemNativeLabelType_FirstWithMultipleInstances,
337	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
338	kIemNativeLabelType_Else,
339	kIemNativeLabelType_Endif,
340	kIemNativeLabelType_CheckIrq,
341	kIemNativeLabelType_TlbLookup,
342	kIemNativeLabelType_TlbMiss,
343	kIemNativeLabelType_TlbDone,
344	kIemNativeLabelType_End
345	} IEMNATIVELABELTYPE;
346
347	/** Native code generator label definition. */
348	typedef struct IEMNATIVELABEL
349	{
350	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
351	* the epilog. */
352	uint32_t off;
353	/** The type of label (IEMNATIVELABELTYPE). */
354	uint16_t enmType;
355	/** Additional label data, type specific. */
356	uint16_t uData;
357	} IEMNATIVELABEL;
358	/** Pointer to a label. */
359	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
360
361
362	/** Native code generator fixup types. */
363	typedef enum
364	{
365	kIemNativeFixupType_Invalid = 0,
366	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
367	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
368	kIemNativeFixupType_Rel32,
369	#elif defined(RT_ARCH_ARM64)
370	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
371	kIemNativeFixupType_RelImm26At0,
372	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
373	kIemNativeFixupType_RelImm19At5,
374	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
375	kIemNativeFixupType_RelImm14At5,
376	#endif
377	kIemNativeFixupType_End
378	} IEMNATIVEFIXUPTYPE;
379
380	/** Native code generator fixup. */
381	typedef struct IEMNATIVEFIXUP
382	{
383	/** Code offset of the fixup location. */
384	uint32_t off;
385	/** The IEMNATIVELABEL this is a fixup for. */
386	uint16_t idxLabel;
387	/** The fixup type (IEMNATIVEFIXUPTYPE). */
388	uint8_t enmType;
389	/** Addend or other data. */
390	int8_t offAddend;
391	} IEMNATIVEFIXUP;
392	/** Pointer to a native code generator fixup. */
393	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
394
395
396	/**
397	* One bit of the state.
398	*
399	* Each register state takes up two bits. We keep the two bits in two separate
400	* 64-bit words to simplify applying them to the guest shadow register mask in
401	* the register allocator.
402	*/
403	typedef union IEMLIVENESSBIT
404	{
405	uint64_t bm64;
406	RT_GCC_EXTENSION struct
407	{ /* bit no */
408	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
409	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
410	uint64_t fCr0 : 1; /*< 0x11 / 17: /
411	uint64_t fFcw : 1; /*< 0x12 / 18: /
412	uint64_t fFsw : 1; /*< 0x13 / 19: /
413	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
414	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
415	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
416	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
417	uint64_t fCr4 : 1; /*< 0x2c / 44: /
418	uint64_t fEflOther : 1; /*< 0x2d / 45: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
419	uint64_t fEflCf : 1; /*< 0x2e / 46: Carry flag (X86_EFL_CF / 0). /
420	uint64_t fEflPf : 1; /*< 0x2f / 47: Parity flag (X86_EFL_PF / 2). /
421	uint64_t fEflAf : 1; /*< 0x20 / 48: Auxilary carry flag (X86_EFL_AF / 4). /
422	uint64_t fEflZf : 1; /*< 0x31 / 49: Zero flag (X86_EFL_ZF / 6). /
423	uint64_t fEflSf : 1; /*< 0x32 / 50: Signed flag (X86_EFL_SF / 7). /
424	uint64_t fEflOf : 1; /*< 0x33 / 51: Overflow flag (X86_EFL_OF / 12). /
425	uint64_t uUnused : 12; /* 0x34 / 52 -> 0x40/64 */
426	};
427	} IEMLIVENESSBIT;
428	AssertCompileSize(IEMLIVENESSBIT, 8);
429
430	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
431	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
432	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
433	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
434	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
435	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
436	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
437
438
439	/**
440	* A liveness state entry.
441	*
442	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
443	* Once we add a SSE register shadowing, we'll add another 64-bit element for
444	* that.
445	*/
446	typedef union IEMLIVENESSENTRY
447	{
448	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
449	uint64_t bm64[16 / 8];
450	uint16_t bm32[16 / 4];
451	uint16_t bm16[16 / 2];
452	uint8_t bm8[ 16 / 1];
453	IEMLIVENESSBIT aBits[2];
454	#else
455	uint64_t bm64[32 / 8];
456	uint16_t bm32[32 / 4];
457	uint16_t bm16[32 / 2];
458	uint8_t bm8[ 32 / 1];
459	IEMLIVENESSBIT aBits[4];
460	#endif
461	RT_GCC_EXTENSION struct
462	{
463	/** Bit \#0 of the register states. */
464	IEMLIVENESSBIT Bit0;
465	/** Bit \#1 of the register states. */
466	IEMLIVENESSBIT Bit1;
467	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
468	/** Bit \#2 of the register states. */
469	IEMLIVENESSBIT Bit2;
470	/** Bit \#3 of the register states. */
471	IEMLIVENESSBIT Bit3;
472	#endif
473	};
474	} IEMLIVENESSENTRY;
475	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
476	AssertCompileSize(IEMLIVENESSENTRY, 16);
477	#else
478	AssertCompileSize(IEMLIVENESSENTRY, 32);
479	#endif
480	/** Pointer to a liveness state entry. */
481	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
482	/** Pointer to a const liveness state entry. */
483	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
484
485	/** @name 64-bit value masks for IEMLIVENESSENTRY.
486	* @{ / / 0xzzzzyyyyxxxxwwww */
487	#define IEMLIVENESSBIT_MASK UINT64_C(0x000ffffffffeffff)
488
489	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
490	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
491	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
492
493	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
494	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
495	#endif
496
497	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x000fe00000000000)
498
499	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
500	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
501	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
502	#endif
503	/** @} */
504
505
506	/** @name The liveness state for a register.
507	*
508	* The state values have been picked to with state accumulation in mind (what
509	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
510	* performance critical work done with the values.
511	*
512	* This is a compressed state that only requires 2 bits per register.
513	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
514	* 1. the incoming state from the following call,
515	* 2. the outgoing state for this call,
516	* 3. mask of the entries set in the 2nd.
517	*
518	* The mask entry (3rd one above) will be used both when updating the outgoing
519	* state and when merging in incoming state for registers not touched by the
520	* current call.
521	*
522	* @{ */
523	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
524	/** The register will be clobbered and the current value thrown away.
525	*
526	* When this is applied to the state (2) we'll simply be AND'ing it with the
527	* (old) mask (3) and adding the register to the mask. This way we'll
528	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
529	* IEMLIVENESS_STATE_INPUT states. */
530	# define IEMLIVENESS_STATE_CLOBBERED 0
531	/** The register is unused in the remainder of the TB.
532	*
533	* This is an initial state and can not be set by any of the
534	* iemNativeLivenessFunc_xxxx callbacks. */
535	# define IEMLIVENESS_STATE_UNUSED 1
536	/** The register value is required in a potential call or exception.
537	*
538	* This means that the register value must be calculated and is best written to
539	* the state, but that any shadowing registers can be flushed thereafter as it's
540	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
541	*
542	* It is typically applied across the board, but we preserve incoming
543	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
544	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
545	* 1. r0 = old & ~mask;
546	* 2. r0 = t1 & (t1 >> 1)'
547	* 3. state \|= r0 \| 0b10;
548	* 4. mask = ~0;
549	*/
550	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
551	/** The register value is used as input.
552	*
553	* This means that the register value must be calculated and it is best to keep
554	* it in a register. It does not need to be writtent out as such. This is the
555	* highest priority state.
556	*
557	* Whether the call modifies the register or not isn't relevant to earlier
558	* calls, so that's not recorded.
559	*
560	* When applying this state we just or in the value in the outgoing state and
561	* mask. */
562	# define IEMLIVENESS_STATE_INPUT 3
563	/** Mask of the state bits. */
564	# define IEMLIVENESS_STATE_MASK 3
565	/** The number of bits per state. */
566	# define IEMLIVENESS_STATE_BIT_COUNT 2
567	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
568	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
569	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
570	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
571	/** Check if a register clobbering is expected given the (previous) liveness state.
572	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
573	* include INPUT if the register is used in more than one place. */
574	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
575
576	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
577	/** The register is not used any more. */
578	# define IEMLIVENESS_STATE_UNUSED 0
579	/** Flag: The register is required in a potential exception or call. */
580	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
581	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
582	/** Flag: The register is read. */
583	# define IEMLIVENESS_STATE_READ 2
584	# define IEMLIVENESS_BIT_READ 1
585	/** Flag: The register is written. */
586	# define IEMLIVENESS_STATE_WRITE 4
587	# define IEMLIVENESS_BIT_WRITE 2
588	/** Flag: Unconditional call (not needed, can be redefined for research). */
589	# define IEMLIVENESS_STATE_CALL 8
590	# define IEMLIVENESS_BIT_CALL 3
591	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
592	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
593	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
594	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
595	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
596	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
597	/** @} */
598
599	/** @name Liveness helpers for builtin functions and similar.
600	*
601	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
602	* own set of manimulator macros for those.
603	*
604	* @{ */
605	/** Initializing the state as all unused. */
606	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
607	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
608	do { \
609	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
610	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
611	} while (0)
612	#else
613	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
614	do { \
615	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
616	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
617	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
618	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
619	} while (0)
620	#endif
621
622	/** Initializing the outgoing state with a potential xcpt or call state.
623	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
624	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
625	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
626	do { \
627	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
628	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
629	} while (0)
630	#else
631	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
632	do { \
633	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
634	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
635	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
636	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
637	} while (0)
638	#endif
639
640	/** Adds a segment base register as input to the outgoing state. */
641	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
642	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
643	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
644	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
645	} while (0)
646	#else
647	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
648	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
649	} while (0)
650	#endif
651
652	/** Adds a segment attribute register as input to the outgoing state. */
653	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
654	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
655	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
656	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
657	} while (0)
658	#else
659	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
660	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
661	} while (0)
662	#endif
663
664	/** Adds a segment limit register as input to the outgoing state. */
665	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
666	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
667	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
668	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
669	} while (0)
670	#else
671	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
672	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
673	} while (0)
674	#endif
675
676	/** Adds a segment limit register as input to the outgoing state. */
677	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
678	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
679	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
680	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
681	} while (0)
682	#else
683	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
684	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
685	} while (0)
686	#endif
687	/** @} */
688
689	/**
690	* Guest registers that can be shadowed in GPRs.
691	*
692	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
693	* must be placed last, as the liveness state tracks it as 7 subcomponents and
694	* we don't want to waste space here.
695	*
696	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
697	* friends as well as IEMAllN8veLiveness.cpp.
698	*/
699	typedef enum IEMNATIVEGSTREG : uint8_t
700	{
701	kIemNativeGstReg_GprFirst = 0,
702	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
703	kIemNativeGstReg_Pc,
704	kIemNativeGstReg_Cr0,
705	kIemNativeGstReg_FpuFcw,
706	kIemNativeGstReg_FpuFsw,
707	kIemNativeGstReg_SegBaseFirst,
708	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
709	kIemNativeGstReg_SegAttribFirst,
710	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
711	kIemNativeGstReg_SegLimitFirst,
712	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
713	kIemNativeGstReg_SegSelFirst,
714	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
715	kIemNativeGstReg_Cr4,
716	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
717	kIemNativeGstReg_End
718	} IEMNATIVEGSTREG;
719	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
720	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
721
722	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
723	* @{ */
724	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
725	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
726	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
727	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
728	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
729	/** @} */
730
731	/**
732	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
733	*/
734	typedef enum IEMNATIVEGSTREGUSE
735	{
736	/** The usage is read-only, the register holding the guest register
737	* shadow copy will not be modified by the caller. */
738	kIemNativeGstRegUse_ReadOnly = 0,
739	/** The caller will update the guest register (think: PC += cbInstr).
740	* The guest shadow copy will follow the returned register. */
741	kIemNativeGstRegUse_ForUpdate,
742	/** The call will put an entirely new value in the guest register, so
743	* if new register is allocate it will be returned uninitialized. */
744	kIemNativeGstRegUse_ForFullWrite,
745	/** The caller will use the guest register value as input in a calculation
746	* and the host register will be modified.
747	* This means that the returned host register will not be marked as a shadow
748	* copy of the guest register. */
749	kIemNativeGstRegUse_Calculation
750	} IEMNATIVEGSTREGUSE;
751
752	/**
753	* Guest registers (classes) that can be referenced.
754	*/
755	typedef enum IEMNATIVEGSTREGREF : uint8_t
756	{
757	kIemNativeGstRegRef_Invalid = 0,
758	kIemNativeGstRegRef_Gpr,
759	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
760	kIemNativeGstRegRef_EFlags,
761	kIemNativeGstRegRef_MxCsr,
762	kIemNativeGstRegRef_FpuReg,
763	kIemNativeGstRegRef_MReg,
764	kIemNativeGstRegRef_XReg,
765	//kIemNativeGstRegRef_YReg, - doesn't work.
766	kIemNativeGstRegRef_End
767	} IEMNATIVEGSTREGREF;
768
769
770	/** Variable kinds. */
771	typedef enum IEMNATIVEVARKIND : uint8_t
772	{
773	/** Customary invalid zero value. */
774	kIemNativeVarKind_Invalid = 0,
775	/** This is either in a register or on the stack. */
776	kIemNativeVarKind_Stack,
777	/** Immediate value - loaded into register when needed, or can live on the
778	* stack if referenced (in theory). */
779	kIemNativeVarKind_Immediate,
780	/** Variable reference - loaded into register when needed, never stack. */
781	kIemNativeVarKind_VarRef,
782	/** Guest register reference - loaded into register when needed, never stack. */
783	kIemNativeVarKind_GstRegRef,
784	/** End of valid values. */
785	kIemNativeVarKind_End
786	} IEMNATIVEVARKIND;
787
788
789	/** Variable or argument. */
790	typedef struct IEMNATIVEVAR
791	{
792	/** The kind of variable. */
793	IEMNATIVEVARKIND enmKind;
794	/** The variable size in bytes. */
795	uint8_t cbVar;
796	/** The first stack slot (uint64_t), except for immediate and references
797	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
798	* has a stack slot it has been initialized and has a value. Unused variables
799	* has neither a stack slot nor a host register assignment. */
800	uint8_t idxStackSlot;
801	/** The host register allocated for the variable, UINT8_MAX if not. */
802	uint8_t idxReg;
803	/** The argument number if argument, UINT8_MAX if regular variable. */
804	uint8_t uArgNo;
805	/** If referenced, the index (unpacked) of the variable referencing this one,
806	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
807	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
808	uint8_t idxReferrerVar;
809	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
810	* @todo not sure what this really is for... */
811	IEMNATIVEGSTREG enmGstReg;
812	/** Set if the registered is currently used exclusively, false if the
813	* variable is idle and the register can be grabbed. */
814	bool fRegAcquired;
815
816	union
817	{
818	/** kIemNativeVarKind_Immediate: The immediate value. */
819	uint64_t uValue;
820	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
821	uint8_t idxRefVar;
822	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
823	struct
824	{
825	/** The class of register. */
826	IEMNATIVEGSTREGREF enmClass;
827	/** Index within the class. */
828	uint8_t idx;
829	} GstRegRef;
830	} u;
831	} IEMNATIVEVAR;
832	/** Pointer to a variable or argument. */
833	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
834	/** Pointer to a const variable or argument. */
835	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
836
837	/** What is being kept in a host register. */
838	typedef enum IEMNATIVEWHAT : uint8_t
839	{
840	/** The traditional invalid zero value. */
841	kIemNativeWhat_Invalid = 0,
842	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
843	kIemNativeWhat_Var,
844	/** Temporary register, this is typically freed when a MC completes. */
845	kIemNativeWhat_Tmp,
846	/** Call argument w/o a variable mapping. This is free (via
847	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
848	kIemNativeWhat_Arg,
849	/** Return status code.
850	* @todo not sure if we need this... */
851	kIemNativeWhat_rc,
852	/** The fixed pVCpu (PVMCPUCC) register.
853	* @todo consider offsetting this on amd64 to use negative offsets to access
854	* more members using 8-byte disp. */
855	kIemNativeWhat_pVCpuFixed,
856	/** The fixed pCtx (PCPUMCTX) register.
857	* @todo consider offsetting this on amd64 to use negative offsets to access
858	* more members using 8-byte disp. */
859	kIemNativeWhat_pCtxFixed,
860	/** Fixed temporary register. */
861	kIemNativeWhat_FixedTmp,
862	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
863	/** Shadow RIP for the delayed RIP updating debugging. */
864	kIemNativeWhat_PcShadow,
865	#endif
866	/** Register reserved by the CPU or OS architecture. */
867	kIemNativeWhat_FixedReserved,
868	/** End of valid values. */
869	kIemNativeWhat_End
870	} IEMNATIVEWHAT;
871
872	/**
873	* Host general register entry.
874	*
875	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
876	*
877	* @todo Track immediate values in host registers similarlly to how we track the
878	* guest register shadow copies. For it to be real helpful, though,
879	* we probably need to know which will be reused and put them into
880	* non-volatile registers, otherwise it's going to be more or less
881	* restricted to an instruction or two.
882	*/
883	typedef struct IEMNATIVEHSTREG
884	{
885	/** Set of guest registers this one shadows.
886	*
887	* Using a bitmap here so we can designate the same host register as a copy
888	* for more than one guest register. This is expected to be useful in
889	* situations where one value is copied to several registers in a sequence.
890	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
891	* sequence we'd want to let this register follow to be a copy of and there
892	* will always be places where we'd be picking the wrong one.
893	*/
894	uint64_t fGstRegShadows;
895	/** What is being kept in this register. */
896	IEMNATIVEWHAT enmWhat;
897	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
898	uint8_t idxVar;
899	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
900	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
901	* that scope. */
902	uint8_t idxStackSlot;
903	/** Alignment padding. */
904	uint8_t abAlign[5];
905	} IEMNATIVEHSTREG;
906
907
908	/**
909	* Core state for the native recompiler, that is, things that needs careful
910	* handling when dealing with branches.
911	*/
912	typedef struct IEMNATIVECORESTATE
913	{
914	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
915	/** The current instruction offset in bytes from when the guest program counter
916	* was updated last. Used for delaying the write to the guest context program counter
917	* as long as possible. */
918	uint32_t offPc;
919	/** Number of instructions where we could skip the updating. */
920	uint32_t cInstrPcUpdateSkipped;
921	#endif
922	/** Allocation bitmap for aHstRegs. */
923	uint32_t bmHstRegs;
924
925	/** Bitmap marking which host register contains guest register shadow copies.
926	* This is used during register allocation to try preserve copies. */
927	uint32_t bmHstRegsWithGstShadow;
928	/** Bitmap marking valid entries in aidxGstRegShadows. */
929	uint64_t bmGstRegShadows;
930
931	union
932	{
933	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
934	uint8_t aidxArgVars[8];
935	/** For more efficient resetting. */
936	uint64_t u64ArgVars;
937	};
938
939	/** Allocation bitmap for the stack. */
940	uint32_t bmStack;
941	/** Allocation bitmap for aVars. */
942	uint32_t bmVars;
943
944	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
945	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
946	* (A shadow copy of a guest register can only be held in a one host register,
947	* there are no duplicate copies or ambiguities like that). */
948	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
949
950	/** Host register allocation tracking. */
951	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
952
953	/** Variables and arguments. */
954	IEMNATIVEVAR aVars[9];
955	} IEMNATIVECORESTATE;
956	/** Pointer to core state. */
957	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
958	/** Pointer to const core state. */
959	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
960
961	/** @def IEMNATIVE_VAR_IDX_UNPACK
962	* @returns Index into IEMNATIVECORESTATE::aVars.
963	* @param a_idxVar Variable index w/ magic (in strict builds).
964	*/
965	/** @def IEMNATIVE_VAR_IDX_PACK
966	* @returns Variable index w/ magic (in strict builds).
967	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
968	*/
969	#ifdef VBOX_STRICT
970	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
971	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
972	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
973	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
974	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
975	#else
976	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
977	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
978	#endif
979
980
981	/**
982	* Conditional stack entry.
983	*/
984	typedef struct IEMNATIVECOND
985	{
986	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
987	bool fInElse;
988	/** The label for the IEM_MC_ELSE. */
989	uint32_t idxLabelElse;
990	/** The label for the IEM_MC_ENDIF. */
991	uint32_t idxLabelEndIf;
992	/** The initial state snapshot as the if-block starts executing. */
993	IEMNATIVECORESTATE InitialState;
994	/** The state snapshot at the end of the if-block. */
995	IEMNATIVECORESTATE IfFinalState;
996	} IEMNATIVECOND;
997	/** Pointer to a condition stack entry. */
998	typedef IEMNATIVECOND *PIEMNATIVECOND;
999
1000
1001	/**
1002	* Native recompiler state.
1003	*/
1004	typedef struct IEMRECOMPILERSTATE
1005	{
1006	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1007	* IEMNATIVEINSTR units. */
1008	uint32_t cInstrBufAlloc;
1009	#ifdef VBOX_STRICT
1010	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1011	uint32_t offInstrBufChecked;
1012	#else
1013	uint32_t uPadding1; /* We don't keep track of the size here... */
1014	#endif
1015	/** Fixed temporary code buffer for native recompilation. */
1016	PIEMNATIVEINSTR pInstrBuf;
1017
1018	/** Bitmaps with the label types used. */
1019	uint64_t bmLabelTypes;
1020	/** Actual number of labels in paLabels. */
1021	uint32_t cLabels;
1022	/** Max number of entries allowed in paLabels before reallocating it. */
1023	uint32_t cLabelsAlloc;
1024	/** Labels defined while recompiling (referenced by fixups). */
1025	PIEMNATIVELABEL paLabels;
1026	/** Array with indexes of unique labels (uData always 0). */
1027	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1028
1029	/** Actual number of fixups paFixups. */
1030	uint32_t cFixups;
1031	/** Max number of entries allowed in paFixups before reallocating it. */
1032	uint32_t cFixupsAlloc;
1033	/** Buffer used by the recompiler for recording fixups when generating code. */
1034	PIEMNATIVEFIXUP paFixups;
1035
1036	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1037	/** Number of debug info entries allocated for pDbgInfo. */
1038	uint32_t cDbgInfoAlloc;
1039	uint32_t uPadding;
1040	/** Debug info. */
1041	PIEMTBDBG pDbgInfo;
1042	#endif
1043
1044	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1045	/** The current call index (liveness array and threaded calls in TB). */
1046	uint32_t idxCurCall;
1047	/** Number of liveness entries allocated. */
1048	uint32_t cLivenessEntriesAlloc;
1049	/** Liveness entries for all the calls in the TB begin recompiled.
1050	* The entry for idxCurCall contains the info for what the next call will
1051	* require wrt registers. (Which means the last entry is the initial liveness
1052	* state.) */
1053	PIEMLIVENESSENTRY paLivenessEntries;
1054	#endif
1055
1056	/** The translation block being recompiled. */
1057	PCIEMTB pTbOrg;
1058	/** The VMCPU structure of the EMT. */
1059	PVMCPUCC pVCpu;
1060
1061	/** Condition sequence number (for generating unique labels). */
1062	uint16_t uCondSeqNo;
1063	/** Check IRQ seqeunce number (for generating unique labels). */
1064	uint16_t uCheckIrqSeqNo;
1065	/** TLB load sequence number (for generating unique labels). */
1066	uint16_t uTlbSeqNo;
1067	/** The current condition stack depth (aCondStack). */
1068	uint8_t cCondDepth;
1069
1070	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1071	uint8_t cArgs;
1072	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1073	uint32_t fCImpl;
1074	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1075	uint32_t fMc;
1076	/** The expected IEMCPU::fExec value for the current call/instruction. */
1077	uint32_t fExec;
1078
1079	/** Core state requiring care with branches. */
1080	IEMNATIVECORESTATE Core;
1081
1082	/** The condition nesting stack. */
1083	IEMNATIVECOND aCondStack[2];
1084
1085	#ifndef IEM_WITH_THROW_CATCH
1086	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1087	* for recompilation error handling. */
1088	jmp_buf JmpBuf;
1089	#endif
1090	} IEMRECOMPILERSTATE;
1091	/** Pointer to a native recompiler state. */
1092	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1093
1094
1095	/** @def IEMNATIVE_TRY_SETJMP
1096	* Wrapper around setjmp / try, hiding all the ugly differences.
1097	*
1098	* @note Use with extreme care as this is a fragile macro.
1099	* @param a_pReNative The native recompile state.
1100	* @param a_rcTarget The variable that should receive the status code in case
1101	* of a longjmp/throw.
1102	*/
1103	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1104	* Start wrapper for catch / setjmp-else.
1105	*
1106	* This will set up a scope.
1107	*
1108	* @note Use with extreme care as this is a fragile macro.
1109	* @param a_pReNative The native recompile state.
1110	* @param a_rcTarget The variable that should receive the status code in case
1111	* of a longjmp/throw.
1112	*/
1113	/** @def IEMNATIVE_CATCH_LONGJMP_END
1114	* End wrapper for catch / setjmp-else.
1115	*
1116	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1117	* up the state.
1118	*
1119	* @note Use with extreme care as this is a fragile macro.
1120	* @param a_pReNative The native recompile state.
1121	*/
1122	/** @def IEMNATIVE_DO_LONGJMP
1123	*
1124	* Wrapper around longjmp / throw.
1125	*
1126	* @param a_pReNative The native recompile state.
1127	* @param a_rc The status code jump back with / throw.
1128	*/
1129	#ifdef IEM_WITH_THROW_CATCH
1130	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1131	a_rcTarget = VINF_SUCCESS; \
1132	try
1133	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1134	catch (int rcThrown) \
1135	{ \
1136	a_rcTarget = rcThrown
1137	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1138	} \
1139	((void)0)
1140	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1141	#else /* !IEM_WITH_THROW_CATCH */
1142	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1143	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1144	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1145	else \
1146	{ \
1147	((void)0)
1148	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1149	}
1150	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1151	#endif /* !IEM_WITH_THROW_CATCH */
1152
1153
1154	/**
1155	* Native recompiler worker for a threaded function.
1156	*
1157	* @returns New code buffer offset; throws VBox status code in case of a failure.
1158	* @param pReNative The native recompiler state.
1159	* @param off The current code buffer offset.
1160	* @param pCallEntry The threaded call entry.
1161	*
1162	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1163	*/
1164	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1165	/** Pointer to a native recompiler worker for a threaded function. */
1166	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1167
1168	/** Defines a native recompiler worker for a threaded function.
1169	* @see FNIEMNATIVERECOMPFUNC */
1170	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1171	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1172
1173	/** Prototypes a native recompiler function for a threaded function.
1174	* @see FNIEMNATIVERECOMPFUNC */
1175	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1176
1177
1178	/**
1179	* Native recompiler liveness analysis worker for a threaded function.
1180	*
1181	* @param pCallEntry The threaded call entry.
1182	* @param pIncoming The incoming liveness state entry.
1183	* @param pOutgoing The outgoing liveness state entry.
1184	*/
1185	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1186	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1187	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1188	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1189
1190	/** Defines a native recompiler liveness analysis worker for a threaded function.
1191	* @see FNIEMNATIVELIVENESSFUNC */
1192	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1193	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1194
1195	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1196	* @see FNIEMNATIVELIVENESSFUNC */
1197	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1198
1199
1200	/** Define a native recompiler helper function, safe to call from the TB code. */
1201	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1202	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1203	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1204	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1205	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1206
1207
1208	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1209	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1210	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1211	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1212	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1213	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1214
1215	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1216	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1217	bool fPreferVolatile = true);
1218	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1219	bool fPreferVolatile = true);
1220	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1221	IEMNATIVEGSTREG enmGstReg,
1222	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1223	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1224	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1225	IEMNATIVEGSTREG enmGstReg);
1226
1227	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1228	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1229	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1230	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1231	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1232	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1233	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1234	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1235	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1236	uint32_t fKeepVars = 0);
1237	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1238	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1239	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1240	uint32_t fHstRegsActiveShadows);
1241
1242	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1243	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1244	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1245	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1246	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1247	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1248	uint32_t fHstRegsNotToSave);
1249	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1250	uint32_t fHstRegsNotToSave);
1251
1252	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1253	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1254	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1255	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1256	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1257	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1258	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1259	PCIEMTHRDEDCALLENTRY pCallEntry);
1260
1261	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1262
1263
1264	/**
1265	* Ensures that there is sufficient space in the instruction output buffer.
1266	*
1267	* This will reallocate the buffer if needed and allowed.
1268	*
1269	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1270	* allocation size.
1271	*
1272	* @returns Pointer to the instruction output buffer on success; throws VBox
1273	* status code on failure, so no need to check it.
1274	* @param pReNative The native recompile state.
1275	* @param off Current instruction offset. Works safely for UINT32_MAX
1276	* as well.
1277	* @param cInstrReq Number of instruction about to be added. It's okay to
1278	* overestimate this a bit.
1279	*/
1280	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1281	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1282	{
1283	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1284	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1285	{
1286	#ifdef VBOX_STRICT
1287	pReNative->offInstrBufChecked = offChecked;
1288	#endif
1289	return pReNative->pInstrBuf;
1290	}
1291	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1292	}
1293
1294	/**
1295	* Checks that we didn't exceed the space requested in the last
1296	* iemNativeInstrBufEnsure() call.
1297	*/
1298	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1299	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1300	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1301
1302	/**
1303	* Checks that a variable index is valid.
1304	*/
1305	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1306	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1307	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1308	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1309	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1310	("%s=%#x\n", #a_idxVar, a_idxVar))
1311	#else
1312	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1313	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1314	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1315	#endif
1316
1317	/**
1318	* Checks that a variable index is valid and that the variable is assigned the
1319	* correct argument number.
1320	* This also adds a RT_NOREF of a_idxVar.
1321	*/
1322	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1323	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1324	RT_NOREF_PV(a_idxVar); \
1325	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1326	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1327	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1328	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1329	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1330	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1331	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1332	a_uArgNo)); \
1333	} while (0)
1334	#else
1335	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1336	RT_NOREF_PV(a_idxVar); \
1337	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1338	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1339	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1340	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1341	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1342	} while (0)
1343	#endif
1344
1345
1346	/**
1347	* Checks that a variable has the expected size.
1348	*/
1349	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1350	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1351	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1352	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1353
1354
1355	/**
1356	* Calculates the stack address of a variable as a [r]BP displacement value.
1357	*/
1358	DECL_FORCE_INLINE(int32_t)
1359	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1360	{
1361	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1362	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1363	}
1364
1365
1366	/**
1367	* Releases the variable's register.
1368	*
1369	* The register must have been previously acquired calling
1370	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1371	* iemNativeVarRegisterSetAndAcquire().
1372	*/
1373	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1374	{
1375	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1376	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1377	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1378	}
1379
1380	/** @} */
1381
1382	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1383

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103649

Download in other formats: