IEMN8veRecompiler.h@ 103665

Last change on this file since 103665 was 103665, checked in by vboxsync, 14 months ago
VMM/IEM: Native translation of IEM_MC_MAYBE_RAISE_FPU_XCPT() body, bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 61.2 KB

Line
1	/* $Id: IEMN8veRecompiler.h 103665 2024-03-04 12:50:11Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41	* Enables generating internal debug info for better TB disassembly dumping. */
42	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
43	# define IEMNATIVE_WITH_TB_DEBUG_INFO
44	#endif
45
46	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47	* Enables liveness analysis. */
48	#if 1 \|\| defined(DOXYGEN_RUNNING)
49	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50	/# define IEMLIVENESS_EXTENDED_LAYOUT/
51	#endif
52
53	#ifdef VBOX_WITH_STATISTICS
54	/** Always count instructions for now. */
55	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
56	#endif
57
58
59	/** @name Stack Frame Layout
60	*
61	* @{ */
62	/** The size of the area for stack variables and spills and stuff.
63	* @note This limit is duplicated in the python script(s). We add 0x40 for
64	* alignment padding. */
65	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
66	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
67	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
68	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
69
70	#ifdef RT_ARCH_AMD64
71	/** An stack alignment adjustment (between non-volatile register pushes and
72	* the stack variable area, so the latter better aligned). */
73	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
74
75	/** Number of stack arguments slots for calls made from the frame. */
76	# ifdef RT_OS_WINDOWS
77	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
78	# else
79	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
80	# endif
81	/** Number of any shadow arguments (spill area) for calls we make. */
82	# ifdef RT_OS_WINDOWS
83	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
84	# else
85	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
86	# endif
87
88	/** Frame pointer (RBP) relative offset of the last push. */
89	# ifdef RT_OS_WINDOWS
90	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
91	# else
92	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
93	# endif
94	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
95	* address for it). */
96	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
97	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
98	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
99	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
100	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
101	# ifdef RT_OS_WINDOWS
102	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
103	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
104	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
105	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
106	# endif
107
108	# ifdef RT_OS_WINDOWS
109	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
110	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
111	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
112	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
113	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
114	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
115	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
116	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
117	# endif
118
119	#elif RT_ARCH_ARM64
120	/** No alignment padding needed for arm64. */
121	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
122	/** No stack argument slots, got 8 registers for arguments will suffice. */
123	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
124	/** There are no argument spill area. */
125	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
126
127	/** Number of saved registers at the top of our stack frame.
128	* This includes the return address and old frame pointer, so x19 thru x30. */
129	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
130	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
131	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
132
133	/** Frame pointer (BP) relative offset of the last push. */
134	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
135
136	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
137	* address for it). */
138	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
139
140	#else
141	# error "port me"
142	#endif
143	/** @} */
144
145
146	/** @name Fixed Register Allocation(s)
147	* @{ */
148	/** @def IEMNATIVE_REG_FIXED_PVMCPU
149	* The number of the register holding the pVCpu pointer. */
150	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
151	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
152	* @note This not available on AMD64, only ARM64. */
153	/** @def IEMNATIVE_REG_FIXED_TMP0
154	* Dedicated temporary register.
155	* @todo replace this by a register allocator and content tracker. */
156	/** @def IEMNATIVE_REG_FIXED_MASK
157	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
158	* architecture. */
159	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
160	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
161	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
162	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
163	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
164	\| RT_BIT_32(X86_GREG_xSP) \
165	\| RT_BIT_32(X86_GREG_xBP) )
166
167	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
168	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
169	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
170	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
171	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
172	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
173	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
174	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
175	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
176	# else
177	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
178	# endif
179	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
180	\| RT_BIT_32(ARMV8_A64_REG_LR) \
181	\| RT_BIT_32(ARMV8_A64_REG_BP) \
182	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
183	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
184	\| RT_BIT_32(ARMV8_A64_REG_X18) \
185	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
186	\| IEMNATIVE_REG_FIXED_MASK_ADD)
187
188	#else
189	# error "port me"
190	#endif
191	/** @} */
192
193	/** @name Call related registers.
194	* @{ */
195	/** @def IEMNATIVE_CALL_RET_GREG
196	* The return value register. */
197	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
198	* Number of arguments in registers. */
199	/** @def IEMNATIVE_CALL_ARG0_GREG
200	* The general purpose register carrying argument \#0. */
201	/** @def IEMNATIVE_CALL_ARG1_GREG
202	* The general purpose register carrying argument \#1. */
203	/** @def IEMNATIVE_CALL_ARG2_GREG
204	* The general purpose register carrying argument \#2. */
205	/** @def IEMNATIVE_CALL_ARG3_GREG
206	* The general purpose register carrying argument \#3. */
207	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
208	* Mask of registers the callee will not save and may trash. */
209	#ifdef RT_ARCH_AMD64
210	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
211
212	# ifdef RT_OS_WINDOWS
213	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
214	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
215	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
216	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
217	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
218	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
219	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
220	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
221	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
222	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
223	\| RT_BIT_32(X86_GREG_xCX) \
224	\| RT_BIT_32(X86_GREG_xDX) \
225	\| RT_BIT_32(X86_GREG_x8) \
226	\| RT_BIT_32(X86_GREG_x9) \
227	\| RT_BIT_32(X86_GREG_x10) \
228	\| RT_BIT_32(X86_GREG_x11) )
229	# else
230	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
231	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
232	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
233	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
234	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
235	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
236	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
237	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
238	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
239	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
240	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
241	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
242	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
243	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
244	\| RT_BIT_32(X86_GREG_xCX) \
245	\| RT_BIT_32(X86_GREG_xDX) \
246	\| RT_BIT_32(X86_GREG_xDI) \
247	\| RT_BIT_32(X86_GREG_xSI) \
248	\| RT_BIT_32(X86_GREG_x8) \
249	\| RT_BIT_32(X86_GREG_x9) \
250	\| RT_BIT_32(X86_GREG_x10) \
251	\| RT_BIT_32(X86_GREG_x11) )
252	# endif
253
254	#elif defined(RT_ARCH_ARM64)
255	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
256	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
257	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
258	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
259	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
260	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
261	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
262	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
263	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
264	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
265	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
266	\| RT_BIT_32(ARMV8_A64_REG_X1) \
267	\| RT_BIT_32(ARMV8_A64_REG_X2) \
268	\| RT_BIT_32(ARMV8_A64_REG_X3) \
269	\| RT_BIT_32(ARMV8_A64_REG_X4) \
270	\| RT_BIT_32(ARMV8_A64_REG_X5) \
271	\| RT_BIT_32(ARMV8_A64_REG_X6) \
272	\| RT_BIT_32(ARMV8_A64_REG_X7) )
273	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
274	\| RT_BIT_32(ARMV8_A64_REG_X1) \
275	\| RT_BIT_32(ARMV8_A64_REG_X2) \
276	\| RT_BIT_32(ARMV8_A64_REG_X3) \
277	\| RT_BIT_32(ARMV8_A64_REG_X4) \
278	\| RT_BIT_32(ARMV8_A64_REG_X5) \
279	\| RT_BIT_32(ARMV8_A64_REG_X6) \
280	\| RT_BIT_32(ARMV8_A64_REG_X7) \
281	\| RT_BIT_32(ARMV8_A64_REG_X8) \
282	\| RT_BIT_32(ARMV8_A64_REG_X9) \
283	\| RT_BIT_32(ARMV8_A64_REG_X10) \
284	\| RT_BIT_32(ARMV8_A64_REG_X11) \
285	\| RT_BIT_32(ARMV8_A64_REG_X12) \
286	\| RT_BIT_32(ARMV8_A64_REG_X13) \
287	\| RT_BIT_32(ARMV8_A64_REG_X14) \
288	\| RT_BIT_32(ARMV8_A64_REG_X15) \
289	\| RT_BIT_32(ARMV8_A64_REG_X16) \
290	\| RT_BIT_32(ARMV8_A64_REG_X17) )
291
292	#endif
293
294	/** This is the maximum argument count we'll ever be needing. */
295	#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
296	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
297	#else
298	# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
299	#endif
300	/** @} */
301
302
303	/** @def IEMNATIVE_HST_GREG_COUNT
304	* Number of host general purpose registers we tracker. */
305	/** @def IEMNATIVE_HST_GREG_MASK
306	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
307	* inverted register masks and such to get down to a correct set of regs. */
308	#ifdef RT_ARCH_AMD64
309	# define IEMNATIVE_HST_GREG_COUNT 16
310	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
311
312	#elif defined(RT_ARCH_ARM64)
313	# define IEMNATIVE_HST_GREG_COUNT 32
314	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
315	#else
316	# error "Port me!"
317	#endif
318
319
320	/** Native code generator label types. */
321	typedef enum
322	{
323	kIemNativeLabelType_Invalid = 0,
324	/* Labels w/o data, only once instance per TB: */
325	kIemNativeLabelType_Return,
326	kIemNativeLabelType_ReturnBreak,
327	kIemNativeLabelType_ReturnWithFlags,
328	kIemNativeLabelType_NonZeroRetOrPassUp,
329	kIemNativeLabelType_RaiseGp0,
330	kIemNativeLabelType_RaiseNm,
331	kIemNativeLabelType_RaiseUd,
332	kIemNativeLabelType_RaiseMf,
333	kIemNativeLabelType_ObsoleteTb,
334	kIemNativeLabelType_NeedCsLimChecking,
335	kIemNativeLabelType_CheckBranchMiss,
336	/* Labels with data, potentially multiple instances per TB: */
337	kIemNativeLabelType_FirstWithMultipleInstances,
338	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
339	kIemNativeLabelType_Else,
340	kIemNativeLabelType_Endif,
341	kIemNativeLabelType_CheckIrq,
342	kIemNativeLabelType_TlbLookup,
343	kIemNativeLabelType_TlbMiss,
344	kIemNativeLabelType_TlbDone,
345	kIemNativeLabelType_End
346	} IEMNATIVELABELTYPE;
347
348	/** Native code generator label definition. */
349	typedef struct IEMNATIVELABEL
350	{
351	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
352	* the epilog. */
353	uint32_t off;
354	/** The type of label (IEMNATIVELABELTYPE). */
355	uint16_t enmType;
356	/** Additional label data, type specific. */
357	uint16_t uData;
358	} IEMNATIVELABEL;
359	/** Pointer to a label. */
360	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
361
362
363	/** Native code generator fixup types. */
364	typedef enum
365	{
366	kIemNativeFixupType_Invalid = 0,
367	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
368	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
369	kIemNativeFixupType_Rel32,
370	#elif defined(RT_ARCH_ARM64)
371	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
372	kIemNativeFixupType_RelImm26At0,
373	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
374	kIemNativeFixupType_RelImm19At5,
375	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
376	kIemNativeFixupType_RelImm14At5,
377	#endif
378	kIemNativeFixupType_End
379	} IEMNATIVEFIXUPTYPE;
380
381	/** Native code generator fixup. */
382	typedef struct IEMNATIVEFIXUP
383	{
384	/** Code offset of the fixup location. */
385	uint32_t off;
386	/** The IEMNATIVELABEL this is a fixup for. */
387	uint16_t idxLabel;
388	/** The fixup type (IEMNATIVEFIXUPTYPE). */
389	uint8_t enmType;
390	/** Addend or other data. */
391	int8_t offAddend;
392	} IEMNATIVEFIXUP;
393	/** Pointer to a native code generator fixup. */
394	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
395
396
397	/**
398	* One bit of the state.
399	*
400	* Each register state takes up two bits. We keep the two bits in two separate
401	* 64-bit words to simplify applying them to the guest shadow register mask in
402	* the register allocator.
403	*/
404	typedef union IEMLIVENESSBIT
405	{
406	uint64_t bm64;
407	RT_GCC_EXTENSION struct
408	{ /* bit no */
409	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
410	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
411	uint64_t fCr0 : 1; /*< 0x11 / 17: /
412	uint64_t fFcw : 1; /*< 0x12 / 18: /
413	uint64_t fFsw : 1; /*< 0x13 / 19: /
414	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
415	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
416	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
417	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
418	uint64_t fCr4 : 1; /*< 0x2c / 44: /
419	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
420	uint64_t fEflOther : 1; /*< 0x2e / 46: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
421	uint64_t fEflCf : 1; /*< 0x2f / 47: Carry flag (X86_EFL_CF / 0). /
422	uint64_t fEflPf : 1; /*< 0x30 / 48: Parity flag (X86_EFL_PF / 2). /
423	uint64_t fEflAf : 1; /*< 0x31 / 49: Auxilary carry flag (X86_EFL_AF / 4). /
424	uint64_t fEflZf : 1; /*< 0x32 / 50: Zero flag (X86_EFL_ZF / 6). /
425	uint64_t fEflSf : 1; /*< 0x33 / 51: Signed flag (X86_EFL_SF / 7). /
426	uint64_t fEflOf : 1; /*< 0x34 / 52: Overflow flag (X86_EFL_OF / 12). /
427	uint64_t uUnused : 11; /* 0x35 / 53 -> 0x40/64 */
428	};
429	} IEMLIVENESSBIT;
430	AssertCompileSize(IEMLIVENESSBIT, 8);
431
432	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
433	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
434	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
435	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
436	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
437	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
438	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
439
440
441	/**
442	* A liveness state entry.
443	*
444	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
445	* Once we add a SSE register shadowing, we'll add another 64-bit element for
446	* that.
447	*/
448	typedef union IEMLIVENESSENTRY
449	{
450	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
451	uint64_t bm64[16 / 8];
452	uint16_t bm32[16 / 4];
453	uint16_t bm16[16 / 2];
454	uint8_t bm8[ 16 / 1];
455	IEMLIVENESSBIT aBits[2];
456	#else
457	uint64_t bm64[32 / 8];
458	uint16_t bm32[32 / 4];
459	uint16_t bm16[32 / 2];
460	uint8_t bm8[ 32 / 1];
461	IEMLIVENESSBIT aBits[4];
462	#endif
463	RT_GCC_EXTENSION struct
464	{
465	/** Bit \#0 of the register states. */
466	IEMLIVENESSBIT Bit0;
467	/** Bit \#1 of the register states. */
468	IEMLIVENESSBIT Bit1;
469	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
470	/** Bit \#2 of the register states. */
471	IEMLIVENESSBIT Bit2;
472	/** Bit \#3 of the register states. */
473	IEMLIVENESSBIT Bit3;
474	#endif
475	};
476	} IEMLIVENESSENTRY;
477	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
478	AssertCompileSize(IEMLIVENESSENTRY, 16);
479	#else
480	AssertCompileSize(IEMLIVENESSENTRY, 32);
481	#endif
482	/** Pointer to a liveness state entry. */
483	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
484	/** Pointer to a const liveness state entry. */
485	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
486
487	/** @name 64-bit value masks for IEMLIVENESSENTRY.
488	* @{ / / 0xzzzzyyyyxxxxwwww */
489	#define IEMLIVENESSBIT_MASK UINT64_C(0x001ffffffffeffff)
490
491	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
492	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
493	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
494
495	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
496	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
497	#endif
498
499	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x001fc00000000000)
500
501	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
502	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
503	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
504	#endif
505	/** @} */
506
507
508	/** @name The liveness state for a register.
509	*
510	* The state values have been picked to with state accumulation in mind (what
511	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
512	* performance critical work done with the values.
513	*
514	* This is a compressed state that only requires 2 bits per register.
515	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
516	* 1. the incoming state from the following call,
517	* 2. the outgoing state for this call,
518	* 3. mask of the entries set in the 2nd.
519	*
520	* The mask entry (3rd one above) will be used both when updating the outgoing
521	* state and when merging in incoming state for registers not touched by the
522	* current call.
523	*
524	* @{ */
525	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
526	/** The register will be clobbered and the current value thrown away.
527	*
528	* When this is applied to the state (2) we'll simply be AND'ing it with the
529	* (old) mask (3) and adding the register to the mask. This way we'll
530	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
531	* IEMLIVENESS_STATE_INPUT states. */
532	# define IEMLIVENESS_STATE_CLOBBERED 0
533	/** The register is unused in the remainder of the TB.
534	*
535	* This is an initial state and can not be set by any of the
536	* iemNativeLivenessFunc_xxxx callbacks. */
537	# define IEMLIVENESS_STATE_UNUSED 1
538	/** The register value is required in a potential call or exception.
539	*
540	* This means that the register value must be calculated and is best written to
541	* the state, but that any shadowing registers can be flushed thereafter as it's
542	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
543	*
544	* It is typically applied across the board, but we preserve incoming
545	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
546	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
547	* 1. r0 = old & ~mask;
548	* 2. r0 = t1 & (t1 >> 1)'
549	* 3. state \|= r0 \| 0b10;
550	* 4. mask = ~0;
551	*/
552	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
553	/** The register value is used as input.
554	*
555	* This means that the register value must be calculated and it is best to keep
556	* it in a register. It does not need to be writtent out as such. This is the
557	* highest priority state.
558	*
559	* Whether the call modifies the register or not isn't relevant to earlier
560	* calls, so that's not recorded.
561	*
562	* When applying this state we just or in the value in the outgoing state and
563	* mask. */
564	# define IEMLIVENESS_STATE_INPUT 3
565	/** Mask of the state bits. */
566	# define IEMLIVENESS_STATE_MASK 3
567	/** The number of bits per state. */
568	# define IEMLIVENESS_STATE_BIT_COUNT 2
569	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
570	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
571	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
572	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
573	/** Check if a register clobbering is expected given the (previous) liveness state.
574	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
575	* include INPUT if the register is used in more than one place. */
576	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
577
578	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
579	/** The register is not used any more. */
580	# define IEMLIVENESS_STATE_UNUSED 0
581	/** Flag: The register is required in a potential exception or call. */
582	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
583	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
584	/** Flag: The register is read. */
585	# define IEMLIVENESS_STATE_READ 2
586	# define IEMLIVENESS_BIT_READ 1
587	/** Flag: The register is written. */
588	# define IEMLIVENESS_STATE_WRITE 4
589	# define IEMLIVENESS_BIT_WRITE 2
590	/** Flag: Unconditional call (not needed, can be redefined for research). */
591	# define IEMLIVENESS_STATE_CALL 8
592	# define IEMLIVENESS_BIT_CALL 3
593	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
594	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
595	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
596	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
597	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
598	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
599	/** @} */
600
601	/** @name Liveness helpers for builtin functions and similar.
602	*
603	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
604	* own set of manimulator macros for those.
605	*
606	* @{ */
607	/** Initializing the state as all unused. */
608	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
609	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
610	do { \
611	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
612	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
613	} while (0)
614	#else
615	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
616	do { \
617	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
618	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
619	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
620	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
621	} while (0)
622	#endif
623
624	/** Initializing the outgoing state with a potential xcpt or call state.
625	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
626	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
627	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
628	do { \
629	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
630	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
631	} while (0)
632	#else
633	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
634	do { \
635	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
636	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
637	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
638	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
639	} while (0)
640	#endif
641
642	/** Adds a segment base register as input to the outgoing state. */
643	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
644	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
645	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
646	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
647	} while (0)
648	#else
649	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
650	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
651	} while (0)
652	#endif
653
654	/** Adds a segment attribute register as input to the outgoing state. */
655	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
656	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
657	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
658	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
659	} while (0)
660	#else
661	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
662	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
663	} while (0)
664	#endif
665
666	/** Adds a segment limit register as input to the outgoing state. */
667	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
668	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
669	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
670	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
671	} while (0)
672	#else
673	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
674	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
675	} while (0)
676	#endif
677
678	/** Adds a segment limit register as input to the outgoing state. */
679	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
680	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
681	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
682	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
683	} while (0)
684	#else
685	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
686	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
687	} while (0)
688	#endif
689	/** @} */
690
691	/**
692	* Guest registers that can be shadowed in GPRs.
693	*
694	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
695	* must be placed last, as the liveness state tracks it as 7 subcomponents and
696	* we don't want to waste space here.
697	*
698	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
699	* friends as well as IEMAllN8veLiveness.cpp.
700	*/
701	typedef enum IEMNATIVEGSTREG : uint8_t
702	{
703	kIemNativeGstReg_GprFirst = 0,
704	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
705	kIemNativeGstReg_Pc,
706	kIemNativeGstReg_Cr0,
707	kIemNativeGstReg_FpuFcw,
708	kIemNativeGstReg_FpuFsw,
709	kIemNativeGstReg_SegBaseFirst,
710	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
711	kIemNativeGstReg_SegAttribFirst,
712	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
713	kIemNativeGstReg_SegLimitFirst,
714	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
715	kIemNativeGstReg_SegSelFirst,
716	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
717	kIemNativeGstReg_Cr4,
718	kIemNativeGstReg_Xcr0,
719	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
720	kIemNativeGstReg_End
721	} IEMNATIVEGSTREG;
722	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
723	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
724
725	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
726	* @{ */
727	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
728	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
729	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
730	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
731	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
732	/** @} */
733
734	/**
735	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
736	*/
737	typedef enum IEMNATIVEGSTREGUSE
738	{
739	/** The usage is read-only, the register holding the guest register
740	* shadow copy will not be modified by the caller. */
741	kIemNativeGstRegUse_ReadOnly = 0,
742	/** The caller will update the guest register (think: PC += cbInstr).
743	* The guest shadow copy will follow the returned register. */
744	kIemNativeGstRegUse_ForUpdate,
745	/** The call will put an entirely new value in the guest register, so
746	* if new register is allocate it will be returned uninitialized. */
747	kIemNativeGstRegUse_ForFullWrite,
748	/** The caller will use the guest register value as input in a calculation
749	* and the host register will be modified.
750	* This means that the returned host register will not be marked as a shadow
751	* copy of the guest register. */
752	kIemNativeGstRegUse_Calculation
753	} IEMNATIVEGSTREGUSE;
754
755	/**
756	* Guest registers (classes) that can be referenced.
757	*/
758	typedef enum IEMNATIVEGSTREGREF : uint8_t
759	{
760	kIemNativeGstRegRef_Invalid = 0,
761	kIemNativeGstRegRef_Gpr,
762	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
763	kIemNativeGstRegRef_EFlags,
764	kIemNativeGstRegRef_MxCsr,
765	kIemNativeGstRegRef_FpuReg,
766	kIemNativeGstRegRef_MReg,
767	kIemNativeGstRegRef_XReg,
768	//kIemNativeGstRegRef_YReg, - doesn't work.
769	kIemNativeGstRegRef_End
770	} IEMNATIVEGSTREGREF;
771
772
773	/** Variable kinds. */
774	typedef enum IEMNATIVEVARKIND : uint8_t
775	{
776	/** Customary invalid zero value. */
777	kIemNativeVarKind_Invalid = 0,
778	/** This is either in a register or on the stack. */
779	kIemNativeVarKind_Stack,
780	/** Immediate value - loaded into register when needed, or can live on the
781	* stack if referenced (in theory). */
782	kIemNativeVarKind_Immediate,
783	/** Variable reference - loaded into register when needed, never stack. */
784	kIemNativeVarKind_VarRef,
785	/** Guest register reference - loaded into register when needed, never stack. */
786	kIemNativeVarKind_GstRegRef,
787	/** End of valid values. */
788	kIemNativeVarKind_End
789	} IEMNATIVEVARKIND;
790
791
792	/** Variable or argument. */
793	typedef struct IEMNATIVEVAR
794	{
795	/** The kind of variable. */
796	IEMNATIVEVARKIND enmKind;
797	/** The variable size in bytes. */
798	uint8_t cbVar;
799	/** The first stack slot (uint64_t), except for immediate and references
800	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
801	* has a stack slot it has been initialized and has a value. Unused variables
802	* has neither a stack slot nor a host register assignment. */
803	uint8_t idxStackSlot;
804	/** The host register allocated for the variable, UINT8_MAX if not. */
805	uint8_t idxReg;
806	/** The argument number if argument, UINT8_MAX if regular variable. */
807	uint8_t uArgNo;
808	/** If referenced, the index (unpacked) of the variable referencing this one,
809	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
810	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
811	uint8_t idxReferrerVar;
812	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
813	* @todo not sure what this really is for... */
814	IEMNATIVEGSTREG enmGstReg;
815	/** Set if the registered is currently used exclusively, false if the
816	* variable is idle and the register can be grabbed. */
817	bool fRegAcquired;
818
819	union
820	{
821	/** kIemNativeVarKind_Immediate: The immediate value. */
822	uint64_t uValue;
823	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
824	uint8_t idxRefVar;
825	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
826	struct
827	{
828	/** The class of register. */
829	IEMNATIVEGSTREGREF enmClass;
830	/** Index within the class. */
831	uint8_t idx;
832	} GstRegRef;
833	} u;
834	} IEMNATIVEVAR;
835	/** Pointer to a variable or argument. */
836	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
837	/** Pointer to a const variable or argument. */
838	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
839
840	/** What is being kept in a host register. */
841	typedef enum IEMNATIVEWHAT : uint8_t
842	{
843	/** The traditional invalid zero value. */
844	kIemNativeWhat_Invalid = 0,
845	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
846	kIemNativeWhat_Var,
847	/** Temporary register, this is typically freed when a MC completes. */
848	kIemNativeWhat_Tmp,
849	/** Call argument w/o a variable mapping. This is free (via
850	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
851	kIemNativeWhat_Arg,
852	/** Return status code.
853	* @todo not sure if we need this... */
854	kIemNativeWhat_rc,
855	/** The fixed pVCpu (PVMCPUCC) register.
856	* @todo consider offsetting this on amd64 to use negative offsets to access
857	* more members using 8-byte disp. */
858	kIemNativeWhat_pVCpuFixed,
859	/** The fixed pCtx (PCPUMCTX) register.
860	* @todo consider offsetting this on amd64 to use negative offsets to access
861	* more members using 8-byte disp. */
862	kIemNativeWhat_pCtxFixed,
863	/** Fixed temporary register. */
864	kIemNativeWhat_FixedTmp,
865	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
866	/** Shadow RIP for the delayed RIP updating debugging. */
867	kIemNativeWhat_PcShadow,
868	#endif
869	/** Register reserved by the CPU or OS architecture. */
870	kIemNativeWhat_FixedReserved,
871	/** End of valid values. */
872	kIemNativeWhat_End
873	} IEMNATIVEWHAT;
874
875	/**
876	* Host general register entry.
877	*
878	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
879	*
880	* @todo Track immediate values in host registers similarlly to how we track the
881	* guest register shadow copies. For it to be real helpful, though,
882	* we probably need to know which will be reused and put them into
883	* non-volatile registers, otherwise it's going to be more or less
884	* restricted to an instruction or two.
885	*/
886	typedef struct IEMNATIVEHSTREG
887	{
888	/** Set of guest registers this one shadows.
889	*
890	* Using a bitmap here so we can designate the same host register as a copy
891	* for more than one guest register. This is expected to be useful in
892	* situations where one value is copied to several registers in a sequence.
893	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
894	* sequence we'd want to let this register follow to be a copy of and there
895	* will always be places where we'd be picking the wrong one.
896	*/
897	uint64_t fGstRegShadows;
898	/** What is being kept in this register. */
899	IEMNATIVEWHAT enmWhat;
900	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
901	uint8_t idxVar;
902	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
903	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
904	* that scope. */
905	uint8_t idxStackSlot;
906	/** Alignment padding. */
907	uint8_t abAlign[5];
908	} IEMNATIVEHSTREG;
909
910
911	/**
912	* Core state for the native recompiler, that is, things that needs careful
913	* handling when dealing with branches.
914	*/
915	typedef struct IEMNATIVECORESTATE
916	{
917	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
918	/** The current instruction offset in bytes from when the guest program counter
919	* was updated last. Used for delaying the write to the guest context program counter
920	* as long as possible. */
921	uint32_t offPc;
922	/** Number of instructions where we could skip the updating. */
923	uint32_t cInstrPcUpdateSkipped;
924	#endif
925	/** Allocation bitmap for aHstRegs. */
926	uint32_t bmHstRegs;
927
928	/** Bitmap marking which host register contains guest register shadow copies.
929	* This is used during register allocation to try preserve copies. */
930	uint32_t bmHstRegsWithGstShadow;
931	/** Bitmap marking valid entries in aidxGstRegShadows. */
932	uint64_t bmGstRegShadows;
933
934	union
935	{
936	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
937	uint8_t aidxArgVars[8];
938	/** For more efficient resetting. */
939	uint64_t u64ArgVars;
940	};
941
942	/** Allocation bitmap for the stack. */
943	uint32_t bmStack;
944	/** Allocation bitmap for aVars. */
945	uint32_t bmVars;
946
947	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
948	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
949	* (A shadow copy of a guest register can only be held in a one host register,
950	* there are no duplicate copies or ambiguities like that). */
951	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
952
953	/** Host register allocation tracking. */
954	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
955
956	/** Variables and arguments. */
957	IEMNATIVEVAR aVars[9];
958	} IEMNATIVECORESTATE;
959	/** Pointer to core state. */
960	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
961	/** Pointer to const core state. */
962	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
963
964	/** @def IEMNATIVE_VAR_IDX_UNPACK
965	* @returns Index into IEMNATIVECORESTATE::aVars.
966	* @param a_idxVar Variable index w/ magic (in strict builds).
967	*/
968	/** @def IEMNATIVE_VAR_IDX_PACK
969	* @returns Variable index w/ magic (in strict builds).
970	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
971	*/
972	#ifdef VBOX_STRICT
973	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
974	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
975	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
976	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
977	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
978	#else
979	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
980	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
981	#endif
982
983
984	/**
985	* Conditional stack entry.
986	*/
987	typedef struct IEMNATIVECOND
988	{
989	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
990	bool fInElse;
991	/** The label for the IEM_MC_ELSE. */
992	uint32_t idxLabelElse;
993	/** The label for the IEM_MC_ENDIF. */
994	uint32_t idxLabelEndIf;
995	/** The initial state snapshot as the if-block starts executing. */
996	IEMNATIVECORESTATE InitialState;
997	/** The state snapshot at the end of the if-block. */
998	IEMNATIVECORESTATE IfFinalState;
999	} IEMNATIVECOND;
1000	/** Pointer to a condition stack entry. */
1001	typedef IEMNATIVECOND *PIEMNATIVECOND;
1002
1003
1004	/**
1005	* Native recompiler state.
1006	*/
1007	typedef struct IEMRECOMPILERSTATE
1008	{
1009	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1010	* IEMNATIVEINSTR units. */
1011	uint32_t cInstrBufAlloc;
1012	#ifdef VBOX_STRICT
1013	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1014	uint32_t offInstrBufChecked;
1015	#else
1016	uint32_t uPadding1; /* We don't keep track of the size here... */
1017	#endif
1018	/** Fixed temporary code buffer for native recompilation. */
1019	PIEMNATIVEINSTR pInstrBuf;
1020
1021	/** Bitmaps with the label types used. */
1022	uint64_t bmLabelTypes;
1023	/** Actual number of labels in paLabels. */
1024	uint32_t cLabels;
1025	/** Max number of entries allowed in paLabels before reallocating it. */
1026	uint32_t cLabelsAlloc;
1027	/** Labels defined while recompiling (referenced by fixups). */
1028	PIEMNATIVELABEL paLabels;
1029	/** Array with indexes of unique labels (uData always 0). */
1030	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1031
1032	/** Actual number of fixups paFixups. */
1033	uint32_t cFixups;
1034	/** Max number of entries allowed in paFixups before reallocating it. */
1035	uint32_t cFixupsAlloc;
1036	/** Buffer used by the recompiler for recording fixups when generating code. */
1037	PIEMNATIVEFIXUP paFixups;
1038
1039	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1040	/** Number of debug info entries allocated for pDbgInfo. */
1041	uint32_t cDbgInfoAlloc;
1042	uint32_t uPadding;
1043	/** Debug info. */
1044	PIEMTBDBG pDbgInfo;
1045	#endif
1046
1047	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1048	/** The current call index (liveness array and threaded calls in TB). */
1049	uint32_t idxCurCall;
1050	/** Number of liveness entries allocated. */
1051	uint32_t cLivenessEntriesAlloc;
1052	/** Liveness entries for all the calls in the TB begin recompiled.
1053	* The entry for idxCurCall contains the info for what the next call will
1054	* require wrt registers. (Which means the last entry is the initial liveness
1055	* state.) */
1056	PIEMLIVENESSENTRY paLivenessEntries;
1057	#endif
1058
1059	/** The translation block being recompiled. */
1060	PCIEMTB pTbOrg;
1061	/** The VMCPU structure of the EMT. */
1062	PVMCPUCC pVCpu;
1063
1064	/** Condition sequence number (for generating unique labels). */
1065	uint16_t uCondSeqNo;
1066	/** Check IRQ seqeunce number (for generating unique labels). */
1067	uint16_t uCheckIrqSeqNo;
1068	/** TLB load sequence number (for generating unique labels). */
1069	uint16_t uTlbSeqNo;
1070	/** The current condition stack depth (aCondStack). */
1071	uint8_t cCondDepth;
1072
1073	/** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
1074	uint8_t cArgs;
1075	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1076	uint32_t fCImpl;
1077	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1078	uint32_t fMc;
1079	/** The expected IEMCPU::fExec value for the current call/instruction. */
1080	uint32_t fExec;
1081
1082	/** Core state requiring care with branches. */
1083	IEMNATIVECORESTATE Core;
1084
1085	/** The condition nesting stack. */
1086	IEMNATIVECOND aCondStack[2];
1087
1088	#ifndef IEM_WITH_THROW_CATCH
1089	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1090	* for recompilation error handling. */
1091	jmp_buf JmpBuf;
1092	#endif
1093	} IEMRECOMPILERSTATE;
1094	/** Pointer to a native recompiler state. */
1095	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1096
1097
1098	/** @def IEMNATIVE_TRY_SETJMP
1099	* Wrapper around setjmp / try, hiding all the ugly differences.
1100	*
1101	* @note Use with extreme care as this is a fragile macro.
1102	* @param a_pReNative The native recompile state.
1103	* @param a_rcTarget The variable that should receive the status code in case
1104	* of a longjmp/throw.
1105	*/
1106	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1107	* Start wrapper for catch / setjmp-else.
1108	*
1109	* This will set up a scope.
1110	*
1111	* @note Use with extreme care as this is a fragile macro.
1112	* @param a_pReNative The native recompile state.
1113	* @param a_rcTarget The variable that should receive the status code in case
1114	* of a longjmp/throw.
1115	*/
1116	/** @def IEMNATIVE_CATCH_LONGJMP_END
1117	* End wrapper for catch / setjmp-else.
1118	*
1119	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1120	* up the state.
1121	*
1122	* @note Use with extreme care as this is a fragile macro.
1123	* @param a_pReNative The native recompile state.
1124	*/
1125	/** @def IEMNATIVE_DO_LONGJMP
1126	*
1127	* Wrapper around longjmp / throw.
1128	*
1129	* @param a_pReNative The native recompile state.
1130	* @param a_rc The status code jump back with / throw.
1131	*/
1132	#ifdef IEM_WITH_THROW_CATCH
1133	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1134	a_rcTarget = VINF_SUCCESS; \
1135	try
1136	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1137	catch (int rcThrown) \
1138	{ \
1139	a_rcTarget = rcThrown
1140	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1141	} \
1142	((void)0)
1143	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1144	#else /* !IEM_WITH_THROW_CATCH */
1145	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1146	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1147	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1148	else \
1149	{ \
1150	((void)0)
1151	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1152	}
1153	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1154	#endif /* !IEM_WITH_THROW_CATCH */
1155
1156
1157	/**
1158	* Native recompiler worker for a threaded function.
1159	*
1160	* @returns New code buffer offset; throws VBox status code in case of a failure.
1161	* @param pReNative The native recompiler state.
1162	* @param off The current code buffer offset.
1163	* @param pCallEntry The threaded call entry.
1164	*
1165	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1166	*/
1167	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1168	/** Pointer to a native recompiler worker for a threaded function. */
1169	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1170
1171	/** Defines a native recompiler worker for a threaded function.
1172	* @see FNIEMNATIVERECOMPFUNC */
1173	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1174	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1175
1176	/** Prototypes a native recompiler function for a threaded function.
1177	* @see FNIEMNATIVERECOMPFUNC */
1178	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1179
1180
1181	/**
1182	* Native recompiler liveness analysis worker for a threaded function.
1183	*
1184	* @param pCallEntry The threaded call entry.
1185	* @param pIncoming The incoming liveness state entry.
1186	* @param pOutgoing The outgoing liveness state entry.
1187	*/
1188	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1189	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1190	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1191	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1192
1193	/** Defines a native recompiler liveness analysis worker for a threaded function.
1194	* @see FNIEMNATIVELIVENESSFUNC */
1195	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1196	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1197
1198	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1199	* @see FNIEMNATIVELIVENESSFUNC */
1200	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1201
1202
1203	/** Define a native recompiler helper function, safe to call from the TB code. */
1204	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1205	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1206	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1207	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1208	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1209
1210
1211	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1212	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1213	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1214	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1215	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1216	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1217
1218	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1219	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1220	bool fPreferVolatile = true);
1221	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1222	bool fPreferVolatile = true);
1223	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1224	IEMNATIVEGSTREG enmGstReg,
1225	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1226	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1227	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1228	IEMNATIVEGSTREG enmGstReg);
1229
1230	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1231	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1232	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1233	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1234	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1235	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1236	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1237	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExept = 0);
1238	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1239	uint32_t fKeepVars = 0);
1240	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1241	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1242	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1243	uint32_t fHstRegsActiveShadows);
1244
1245	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1246	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1247	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1248	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1249	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1250	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1251	uint32_t fHstRegsNotToSave);
1252	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1253	uint32_t fHstRegsNotToSave);
1254
1255	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1256	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1257	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1258	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1259	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1260	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1261	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1262	PCIEMTHRDEDCALLENTRY pCallEntry);
1263
1264	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1265
1266
1267	/**
1268	* Ensures that there is sufficient space in the instruction output buffer.
1269	*
1270	* This will reallocate the buffer if needed and allowed.
1271	*
1272	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1273	* allocation size.
1274	*
1275	* @returns Pointer to the instruction output buffer on success; throws VBox
1276	* status code on failure, so no need to check it.
1277	* @param pReNative The native recompile state.
1278	* @param off Current instruction offset. Works safely for UINT32_MAX
1279	* as well.
1280	* @param cInstrReq Number of instruction about to be added. It's okay to
1281	* overestimate this a bit.
1282	*/
1283	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1284	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1285	{
1286	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1287	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1288	{
1289	#ifdef VBOX_STRICT
1290	pReNative->offInstrBufChecked = offChecked;
1291	#endif
1292	return pReNative->pInstrBuf;
1293	}
1294	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1295	}
1296
1297	/**
1298	* Checks that we didn't exceed the space requested in the last
1299	* iemNativeInstrBufEnsure() call.
1300	*/
1301	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1302	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1303	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1304
1305	/**
1306	* Checks that a variable index is valid.
1307	*/
1308	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1309	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1310	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1311	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1312	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1313	("%s=%#x\n", #a_idxVar, a_idxVar))
1314	#else
1315	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1316	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1317	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1318	#endif
1319
1320	/**
1321	* Checks that a variable index is valid and that the variable is assigned the
1322	* correct argument number.
1323	* This also adds a RT_NOREF of a_idxVar.
1324	*/
1325	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1326	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1327	RT_NOREF_PV(a_idxVar); \
1328	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1329	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1330	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1331	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1332	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1333	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1334	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1335	a_uArgNo)); \
1336	} while (0)
1337	#else
1338	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1339	RT_NOREF_PV(a_idxVar); \
1340	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1341	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1342	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1343	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1344	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1345	} while (0)
1346	#endif
1347
1348
1349	/**
1350	* Checks that a variable has the expected size.
1351	*/
1352	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
1353	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
1354	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
1355	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
1356
1357
1358	/**
1359	* Calculates the stack address of a variable as a [r]BP displacement value.
1360	*/
1361	DECL_FORCE_INLINE(int32_t)
1362	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1363	{
1364	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1365	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1366	}
1367
1368
1369	/**
1370	* Releases the variable's register.
1371	*
1372	* The register must have been previously acquired calling
1373	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1374	* iemNativeVarRegisterSetAndAcquire().
1375	*/
1376	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1377	{
1378	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1379	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
1380	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
1381	}
1382
1383	/** @} */
1384
1385	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1386

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103665

Download in other formats: