IEMInternal.h@ 108250

Last change on this file since 108250 was 108226, checked in by vboxsync, 3 months ago
VMM/IEM: Splitting up IEMAll.cpp. jiraref:VBP-1531
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 147.7 KB

Line
1	/* $Id: IEMInternal.h 108226 2025-02-14 15:54:48Z vboxsync $ */
2	/** @file
3	* IEM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
29	#define VMM_INCLUDED_SRC_include_IEMInternal_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#ifndef RT_IN_ASSEMBLER
35	# include <VBox/vmm/cpum.h>
36	# include <VBox/vmm/iem.h>
37	# include <VBox/vmm/pgm.h>
38	# include <VBox/vmm/stam.h>
39	# include <VBox/param.h>
40
41	# include <iprt/setjmp-without-sigmask.h>
42	# include <iprt/list.h>
43	#endif /* !RT_IN_ASSEMBLER */
44
45
46	RT_C_DECLS_BEGIN
47
48
49	/** @defgroup grp_iem_int Internals
50	* @ingroup grp_iem
51	* @internal
52	* @{
53	*/
54
55	/* Make doxygen happy w/o overcomplicating the #if checks. */
56	#ifdef DOXYGEN_RUNNING
57	# define IEM_WITH_THROW_CATCH
58	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
59	#endif
60
61	/** For expanding symbol in slickedit and other products tagging and
62	* crossreferencing IEM symbols. */
63	#ifndef IEM_STATIC
64	# define IEM_STATIC static
65	#endif
66
67	/** @def IEM_WITH_SETJMP
68	* Enables alternative status code handling using setjmps.
69	*
70	* This adds a bit of expense via the setjmp() call since it saves all the
71	* non-volatile registers. However, it eliminates return code checks and allows
72	* for more optimal return value passing (return regs instead of stack buffer).
73	*/
74	#if defined(DOXYGEN_RUNNING) \|\| defined(RT_OS_WINDOWS) \|\| 1
75	# define IEM_WITH_SETJMP
76	#endif
77
78	/** @def IEM_WITH_THROW_CATCH
79	* Enables using C++ throw/catch as an alternative to setjmp/longjmp in user
80	* mode code when IEM_WITH_SETJMP is in effect.
81	*
82	* With GCC 11.3.1 and code TLB on linux, using throw/catch instead of
83	* setjmp/long resulted in bs2-test-1 running 3.00% faster and all but on test
84	* result value improving by more than 1%. (Best out of three.)
85	*
86	* With Visual C++ 2019 and code TLB on windows, using throw/catch instead of
87	* setjmp/long resulted in bs2-test-1 running 3.68% faster and all but some of
88	* the MMIO and CPUID tests ran noticeably faster. Variation is greater than on
89	* Linux, but it should be quite a bit faster for normal code.
90	*/
91	#if defined(__cplusplus) && defined(IEM_WITH_SETJMP) && defined(IN_RING3) && (defined(__GNUC__) \|\| defined(_MSC_VER)) /* ASM-NOINC-START */
92	# define IEM_WITH_THROW_CATCH
93	#endif /ASM-NOINC-END/
94
95	/** @def IEM_WITH_ADAPTIVE_TIMER_POLLING
96	* Enables the adaptive timer polling code.
97	*/
98	#if defined(DOXYGEN_RUNNING) \|\| 1
99	# define IEM_WITH_ADAPTIVE_TIMER_POLLING
100	#endif
101
102	/** @def IEM_WITH_INTRA_TB_JUMPS
103	* Enables loop-jumps within a TB (currently only to the first call).
104	*/
105	#if defined(DOXYGEN_RUNNING) \|\| 1
106	# define IEM_WITH_INTRA_TB_JUMPS
107	#endif
108
109	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
110	* Enables the delayed PC updating optimization (see @bugref{10373}).
111	*/
112	#if defined(DOXYGEN_RUNNING) \|\| 1
113	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING
114	#endif
115	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
116	* Enabled delayed PC updating debugging code.
117	* This is an alternative to the ARM64-only IEMNATIVE_REG_FIXED_PC_DBG. */
118	#if defined(DOXYGEN_RUNNING) \|\| 0
119	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
120	#endif
121
122	/** Enables access to even callee saved registers. */
123	/# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS/
124
125	#if defined(DOXYGEN_RUNNING) \|\| 1
126	/** @def IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
127	* Delay the writeback or dirty registers as long as possible. */
128	# define IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
129	#endif
130
131	/** @def IEM_WITH_TLB_STATISTICS
132	* Enables all TLB statistics. */
133	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
134	# define IEM_WITH_TLB_STATISTICS
135	#endif
136
137	/** @def IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
138	* Enable this to use native emitters for certain SIMD FP operations. */
139	#if 1 \|\| defined(DOXYGEN_RUNNING)
140	# define IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
141	#endif
142
143	/** @def VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
144	* Enable this to create a saved state file with the threaded translation
145	* blocks fed to the native recompiler on VCPU \#0. The resulting file can
146	* then be fed into the native recompiler for code profiling purposes.
147	* This is not a feature that should be normally be enabled! */
148	#if 0 \|\| defined(DOXYGEN_RUNNING)
149	# define VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
150	#endif
151
152	/** @def VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
153	* Enables a quicker alternative to throw/longjmp for IEM_DO_LONGJMP when
154	* executing native translation blocks.
155	*
156	* This exploits the fact that we save all non-volatile registers in the TB
157	* prologue and thus just need to do the same as the TB epilogue to get the
158	* effect of a longjmp/throw. Since MSC marks XMM6 thru XMM15 as
159	* non-volatile (and does something even more crazy for ARM), this probably
160	* won't work reliably on Windows. */
161	#ifdef RT_ARCH_ARM64
162	# ifndef RT_OS_WINDOWS
163	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
164	# endif
165	#endif
166	/* ASM-NOINC-START */
167	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
168	# if !defined(IN_RING3) \
169	\|\| !defined(VBOX_WITH_IEM_RECOMPILER) \
170	\|\| !defined(VBOX_WITH_IEM_NATIVE_RECOMPILER)
171	# undef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
172	# elif defined(RT_OS_WINDOWS)
173	# pragma message("VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP is not safe to use on windows")
174	# endif
175	#endif
176
177
178	/** @def IEM_DO_LONGJMP
179	*
180	* Wrapper around longjmp / throw.
181	*
182	* @param a_pVCpu The CPU handle.
183	* @param a_rc The status code jump back with / throw.
184	*/
185	#if defined(IEM_WITH_SETJMP) \|\| defined(DOXYGEN_RUNNING)
186	# ifdef IEM_WITH_THROW_CATCH
187	# ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
188	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) do { \
189	if ((a_pVCpu)->iem.s.pvTbFramePointerR3) \
190	iemNativeTbLongJmp((a_pVCpu)->iem.s.pvTbFramePointerR3, (a_rc)); \
191	throw int(a_rc); \
192	} while (0)
193	# else
194	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) throw int(a_rc)
195	# endif
196	# else
197	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) longjmp(*(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf), (a_rc))
198	# endif
199	#endif
200
201	/** For use with IEM function that may do a longjmp (when enabled).
202	*
203	* Visual C++ has trouble longjmp'ing from/over functions with the noexcept
204	* attribute. So, we indicate that function that may be part of a longjmp may
205	* throw "exceptions" and that the compiler should definitely not generate and
206	* std::terminate calling unwind code.
207	*
208	* Here is one example of this ending in std::terminate:
209	* @code{.txt}
210	00 00000041`cadfda10 00007ffc`5d5a1f9f ucrtbase!abort+0x4e
211	01 00000041`cadfda40 00007ffc`57af229a ucrtbase!terminate+0x1f
212	02 00000041`cadfda70 00007ffb`eec91030 VCRUNTIME140!__std_terminate+0xa [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\ehhelpers.cpp @ 192]
213	03 00000041`cadfdaa0 00007ffb`eec92c6d VCRUNTIME140_1!_CallSettingFrame+0x20 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\handlers.asm @ 50]
214	04 00000041`cadfdad0 00007ffb`eec93ae5 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToState+0x241 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 1085]
215	05 00000041`cadfdc00 00007ffb`eec92258 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToEmptyState+0x2d [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 218]
216	06 00000041`cadfdc30 00007ffb`eec940e9 VCRUNTIME140_1!__InternalCxxFrameHandler<__FrameHandler4>+0x194 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 304]
217	07 00000041`cadfdcd0 00007ffc`5f9f249f VCRUNTIME140_1!__CxxFrameHandler4+0xa9 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 290]
218	08 00000041`cadfdd40 00007ffc`5f980939 ntdll!RtlpExecuteHandlerForUnwind+0xf
219	09 00000041`cadfdd70 00007ffc`5f9a0edd ntdll!RtlUnwindEx+0x339
220	0a 00000041`cadfe490 00007ffc`57aff976 ntdll!RtlUnwind+0xcd
221	0b 00000041`cadfea00 00007ffb`e1b5de01 VCRUNTIME140!__longjmp_internal+0xe6 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\longjmp.asm @ 140]
222	0c (Inline Function) --------`-------- VBoxVMM!iemOpcodeGetNextU8SlowJmp+0x95 [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 1155]
223	0d 00000041`cadfea50 00007ffb`e1b60f6b VBoxVMM!iemOpcodeGetNextU8Jmp+0xc1 [L:\vbox-intern\src\VBox\VMM\include\IEMInline.h @ 402]
224	0e 00000041`cadfea90 00007ffb`e1cc6201 VBoxVMM!IEMExecForExits+0xdb [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 10185]
225	0f 00000041`cadfec70 00007ffb`e1d0df8d VBoxVMM!EMHistoryExec+0x4f1 [L:\vbox-intern\src\VBox\VMM\VMMAll\EMAll.cpp @ 452]
226	10 00000041`cadfed60 00007ffb`e1d0d4c0 VBoxVMM!nemR3WinHandleExitCpuId+0x79d [L:\vbox-intern\src\VBox\VMM\VMMAll\NEMAllNativeTemplate-win.cpp.h @ 1829] @encode
227	@endcode
228	*
229	* @see https://developercommunity.visualstudio.com/t/fragile-behavior-of-longjmp-called-from-noexcept-f/1532859
230	*/
231	#if defined(IEM_WITH_SETJMP) && (defined(_MSC_VER) \|\| defined(IEM_WITH_THROW_CATCH))
232	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT_EX(false)
233	#else
234	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT
235	#endif
236	/* ASM-NOINC-END */
237
238
239	//#define IEM_WITH_CODE_TLB // - work in progress
240	//#define IEM_WITH_DATA_TLB // - work in progress
241
242
243	/** @def IEM_USE_UNALIGNED_DATA_ACCESS
244	* Use unaligned accesses instead of elaborate byte assembly. */
245	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86) \|\| defined(DOXYGEN_RUNNING) /ASM-NOINC/
246	# define IEM_USE_UNALIGNED_DATA_ACCESS
247	#endif /ASM-NOINC/
248
249	//#define IEM_LOG_MEMORY_WRITES
250
251
252	/** @def IEM_CFG_TARGET_CPU
253	* The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
254	*
255	* By default we allow this to be configured by the user via the
256	* CPUM/GuestCpuName config string, but this comes at a slight cost during
257	* decoding. So, for applications of this code where there is no need to
258	* be dynamic wrt target CPU, just modify this define.
259	*/
260	#if !defined(IEM_CFG_TARGET_CPU) \|\| defined(DOXYGEN_RUNNING)
261	# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
262	#endif
263
264
265	/*
266	* X86 config.
267	*/
268
269	#define IEM_IMPLEMENTS_TASKSWITCH
270
271	/** @def IEM_WITH_3DNOW
272	* Includes the 3DNow decoding. */
273	#if !defined(IEM_WITH_3DNOW) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
274	# ifndef IEM_WITHOUT_3DNOW
275	# define IEM_WITH_3DNOW
276	# endif
277	#endif
278
279	/** @def IEM_WITH_THREE_0F_38
280	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
281	#if !defined(IEM_WITH_THREE_0F_38) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
282	# ifndef IEM_WITHOUT_THREE_0F_38
283	# define IEM_WITH_THREE_0F_38
284	# endif
285	#endif
286
287	/** @def IEM_WITH_THREE_0F_3A
288	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
289	#if !defined(IEM_WITH_THREE_0F_3A) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
290	# ifndef IEM_WITHOUT_THREE_0F_3A
291	# define IEM_WITH_THREE_0F_3A
292	# endif
293	#endif
294
295	/** @def IEM_WITH_VEX
296	* Includes the VEX decoding. */
297	#if !defined(IEM_WITH_VEX) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
298	# ifndef IEM_WITHOUT_VEX
299	# define IEM_WITH_VEX
300	# endif
301	#endif
302
303
304	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
305
306	# if !defined(IEM_WITHOUT_INSTRUCTION_STATS) && !defined(DOXYGEN_RUNNING)
307	/** Instruction statistics. */
308	typedef struct IEMINSTRSTATS
309	{
310	# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
311	# include "IEMInstructionStatisticsTmpl.h"
312	# undef IEM_DO_INSTR_STAT
313	} IEMINSTRSTATS;
314	#else
315	struct IEMINSTRSTATS;
316	typedef struct IEMINSTRSTATS IEMINSTRSTATS;
317	#endif
318	/** Pointer to IEM instruction statistics. */
319	typedef IEMINSTRSTATS *PIEMINSTRSTATS;
320
321
322	/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
323	* @{ */
324	#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /*< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. /
325	#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /*< Intel EFLAGS result. /
326	#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /*< AMD EFLAGS result /
327	#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /*< Reserved/dummy entry slot that's the same as 0. /
328	#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /*< For masking the index before use. /
329	/** Selects the right variant from a_aArray.
330	* pVCpu is implicit in the caller context. */
331	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
332	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
333	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
334	* be used because the host CPU does not support the operation. */
335	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
336	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
337	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
338	* array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
339	* into the two.
340	* @sa IEM_SELECT_NATIVE_OR_FALLBACK */
341	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
342	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
343	(a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
344	#else
345	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
346	(a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
347	#endif
348	/** @} */
349
350	/**
351	* Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
352	* indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
353	*
354	* On non-x86 hosts, this will shortcut to the fallback w/o checking the
355	* indicator.
356	*
357	* @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
358	*/
359	#if (defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
360	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
361	(g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
362	#else
363	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
364	#endif
365
366	/** @name Helpers for passing C++ template arguments to an
367	* IEM_MC_NATIVE_EMIT_3/4/5 style macro.
368	* @{
369	*/
370	#define IEM_TEMPL_ARG_1(a1) <a1>
371	#define IEM_TEMPL_ARG_2(a1, a2) <a1,a2>
372	#define IEM_TEMPL_ARG_3(a1, a2, a3) <a1,a2,a3>
373	/** @} */
374
375
376	/**
377	* IEM TLB entry.
378	*
379	* Lookup assembly:
380	* @code{.asm}
381	; Calculate tag.
382	mov rax, [VA]
383	shl rax, 16
384	shr rax, 16 + X86_PAGE_SHIFT
385	or rax, [uTlbRevision]
386
387	; Do indexing.
388	movzx ecx, al
389	lea rcx, [pTlbEntries + rcx]
390
391	; Check tag.
392	cmp [rcx + IEMTLBENTRY.uTag], rax
393	jne .TlbMiss
394
395	; Check access.
396	mov rax, ACCESS_FLAGS \| MAPPING_R3_NOT_VALID \| 0xffffff00
397	and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
398	cmp rax, [uTlbPhysRev]
399	jne .TlbMiss
400
401	; Calc address and we're done.
402	mov eax, X86_PAGE_OFFSET_MASK
403	and eax, [VA]
404	or rax, [rcx + IEMTLBENTRY.pMappingR3]
405	%ifdef VBOX_WITH_STATISTICS
406	inc qword [cTlbHits]
407	%endif
408	jmp .Done
409
410	.TlbMiss:
411	mov r8d, ACCESS_FLAGS
412	mov rdx, [VA]
413	mov rcx, [pVCpu]
414	call iemTlbTypeMiss
415	.Done:
416
417	@endcode
418	*
419	*/
420	typedef struct IEMTLBENTRY
421	{
422	/** The TLB entry tag.
423	* Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
424	* is ASSUMING a virtual address width of 48 bits.
425	*
426	* Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
427	*
428	* The TLB lookup code uses the current TLB revision, which won't ever be zero,
429	* enabling an extremely cheap TLB invalidation most of the time. When the TLB
430	* revision wraps around though, the tags needs to be zeroed.
431	*
432	* @note Try use SHRD instruction? After seeing
433	* https://gmplib.org/~tege/x86-timing.pdf, maybe not.
434	*
435	* @todo This will need to be reorganized for 57-bit wide virtual address and
436	* PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
437	* have to move the TLB entry versioning entirely to the
438	* fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
439	* 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
440	* consumed by PCID and ASID (12 + 6 = 18).
441	*/
442	uint64_t uTag;
443	/** Access flags and physical TLB revision.
444	*
445	* - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
446	* - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
447	* - Bit 2 - page tables - not user (complemented X86_PTE_US).
448	* - Bit 3 - pgm phys/virt - not directly writable.
449	* - Bit 4 - pgm phys page - not directly readable.
450	* - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
451	* - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
452	* - Bit 7 - tlb entry - pMappingR3 member not valid.
453	* - Bits 63 thru 8 are used for the physical TLB revision number.
454	*
455	* We're using complemented bit meanings here because it makes it easy to check
456	* whether special action is required. For instance a user mode write access
457	* would do a "TEST fFlags, (X86_PTE_RW \| X86_PTE_US \| X86_PTE_D)" and a
458	* non-zero result would mean special handling needed because either it wasn't
459	* writable, or it wasn't user, or the page wasn't dirty. A user mode read
460	* access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
461	* need to check any PTE flag.
462	*/
463	uint64_t fFlagsAndPhysRev;
464	/** The guest physical page address. */
465	uint64_t GCPhys;
466	/** Pointer to the ring-3 mapping. */
467	R3PTRTYPE(uint8_t *) pbMappingR3;
468	#if HC_ARCH_BITS == 32
469	uint32_t u32Padding1;
470	#endif
471	} IEMTLBENTRY;
472	AssertCompileSize(IEMTLBENTRY, 32);
473	/** Pointer to an IEM TLB entry. */
474	typedef IEMTLBENTRY *PIEMTLBENTRY;
475	/** Pointer to a const IEM TLB entry. */
476	typedef IEMTLBENTRY const *PCIEMTLBENTRY;
477
478	/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
479	* @{ */
480	#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /*< Page tables: Not executable. /
481	#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /*< Page tables: Not writable. /
482	#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /*< Page tables: Not user accessible (supervisor only). /
483	#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /*< Phys page: Not writable (access handler, ROM, whatever). /
484	#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /*< Phys page: Not readable (MMIO / access handler, ROM) /
485	#define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /*< Phys tables: Not accessed (need to be marked accessed). /
486	#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /*< Page tables: Not dirty (needs to be made dirty on write). /
487	#define IEMTLBE_F_PT_LARGE_PAGE RT_BIT_64(7) /*< Page tables: Large 2 or 4 MiB page (for flushing). /
488	#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(8) /*< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. /
489	#define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(9) /*< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). /
490	#define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(10) /*< Phys page: Code page. /
491	#define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffff800) /*< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR /
492	/** @} */
493	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
494	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
495	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
496	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
497	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
498	AssertCompile(PGM_WALKINFO_BIG_PAGE == IEMTLBE_F_PT_LARGE_PAGE);
499	/** The bits set by PGMPhysIemGCPhys2PtrNoLock. */
500	#define IEMTLBE_GCPHYS2PTR_MASK ( PGMIEMGCPHYS2PTR_F_NO_WRITE \
501	\| PGMIEMGCPHYS2PTR_F_NO_READ \
502	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \
503	\| PGMIEMGCPHYS2PTR_F_UNASSIGNED \
504	\| PGMIEMGCPHYS2PTR_F_CODE_PAGE \
505	\| IEMTLBE_F_PHYS_REV )
506
507
508	/** The TLB size (power of two).
509	* We initially chose 256 because that way we can obtain the result directly
510	* from a 8-bit register without an additional AND instruction.
511	* See also @bugref{10687}. */
512	#if defined(RT_ARCH_AMD64)
513	# define IEMTLB_ENTRY_COUNT 256
514	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 8
515	#else
516	# define IEMTLB_ENTRY_COUNT 8192
517	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 13
518	#endif
519	AssertCompile(RT_BIT_32(IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO) == IEMTLB_ENTRY_COUNT);
520
521	/** TLB slot format spec (assumes uint32_t or unsigned value). */
522	#if IEMTLB_ENTRY_COUNT <= 0x100 / 2
523	# define IEMTLB_SLOT_FMT "%02x"
524	#elif IEMTLB_ENTRY_COUNT <= 0x1000 / 2
525	# define IEMTLB_SLOT_FMT "%03x"
526	#elif IEMTLB_ENTRY_COUNT <= 0x10000 / 2
527	# define IEMTLB_SLOT_FMT "%04x"
528	#else
529	# define IEMTLB_SLOT_FMT "%05x"
530	#endif
531
532	/** Enable the large page bitmap TLB optimization.
533	*
534	* The idea here is to avoid scanning the full 32 KB (2MB pages, 2*512 TLB
535	* entries) or 64 KB (4MB pages, 2*1024 TLB entries) worth of TLB entries during
536	* invlpg when large pages are used, and instead just scan 128 or 256 bytes of
537	* the bmLargePage bitmap to determin which TLB entires that might be containing
538	* large pages and actually require checking.
539	*
540	* There is a good posibility of false positives since we currently don't clear
541	* the bitmap when flushing the TLB, but it should help reduce the workload when
542	* the large pages aren't fully loaded into the TLB in their entirity...
543	*/
544	#define IEMTLB_WITH_LARGE_PAGE_BITMAP
545
546	/**
547	* An IEM TLB.
548	*
549	* We've got two of these, one for data and one for instructions.
550	*/
551	typedef struct IEMTLB
552	{
553	/** The non-global TLB revision.
554	* This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
555	* by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
556	* the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
557	* (The revision zero indicates an invalid TLB entry.)
558	*
559	* The initial value is choosen to cause an early wraparound. */
560	uint64_t uTlbRevision;
561	/** The TLB physical address revision - shadow of PGM variable.
562	*
563	* This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
564	* incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
565	* a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
566	* as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
567	*
568	* The initial value is choosen to cause an early wraparound.
569	*
570	* @note This is placed between the two TLB revisions because we
571	* load it in pair with one or the other on arm64. */
572	uint64_t volatile uTlbPhysRev;
573	/** The global TLB revision.
574	* Same as uTlbRevision, but only increased for global flushes. */
575	uint64_t uTlbRevisionGlobal;
576
577	/** Large page tag range.
578	*
579	* This is used to avoid scanning a large page's worth of TLB entries for each
580	* INVLPG instruction, and only to do so iff we've loaded any and when the
581	* address is in this range. This is kept up to date when we loading new TLB
582	* entries.
583	*/
584	struct LARGEPAGERANGE
585	{
586	/** The lowest large page address tag, UINT64_MAX if none. */
587	uint64_t uFirstTag;
588	/** The highest large page address tag (with offset mask part set), 0 if none. */
589	uint64_t uLastTag;
590	}
591	/** Large page range for non-global pages. */
592	NonGlobalLargePageRange,
593	/** Large page range for global pages. */
594	GlobalLargePageRange;
595	/** Number of non-global entries for large pages loaded since last TLB flush. */
596	uint32_t cTlbNonGlobalLargePageCurLoads;
597	/** Number of global entries for large pages loaded since last TLB flush. */
598	uint32_t cTlbGlobalLargePageCurLoads;
599
600	/* Statistics: */
601
602	/** TLB hits in IEMAll.cpp code (IEM_WITH_TLB_STATISTICS only; both).
603	* @note For the data TLB this is only used in iemMemMap and and for direct (i.e.
604	* not via safe read/write path) calls to iemMemMapJmp. */
605	uint64_t cTlbCoreHits;
606	/** Safe read/write TLB hits in iemMemMapJmp (IEM_WITH_TLB_STATISTICS
607	* only; data tlb only). */
608	uint64_t cTlbSafeHits;
609	/** TLB hits in IEMAllMemRWTmplInline.cpp.h (data + IEM_WITH_TLB_STATISTICS only). */
610	uint64_t cTlbInlineCodeHits;
611
612	/** TLB misses in IEMAll.cpp code (both).
613	* @note For the data TLB this is only used in iemMemMap and for direct (i.e.
614	* not via safe read/write path) calls to iemMemMapJmp. So,
615	* for the data TLB this more like 'other misses', while for the code
616	* TLB is all misses. */
617	uint64_t cTlbCoreMisses;
618	/** Subset of cTlbCoreMisses that results in PTE.G=1 loads (odd entries). */
619	uint64_t cTlbCoreGlobalLoads;
620	/** Safe read/write TLB misses in iemMemMapJmp (so data only). */
621	uint64_t cTlbSafeMisses;
622	/** Subset of cTlbSafeMisses that results in PTE.G=1 loads (odd entries). */
623	uint64_t cTlbSafeGlobalLoads;
624	/** Safe read path taken (data only). */
625	uint64_t cTlbSafeReadPath;
626	/** Safe write path taken (data only). */
627	uint64_t cTlbSafeWritePath;
628
629	/** @name Details for native code TLB misses.
630	* @note These counts are included in the above counters (cTlbSafeReadPath,
631	* cTlbSafeWritePath, cTlbInlineCodeHits).
632	* @{ */
633	/** TLB misses in native code due to tag mismatch. */
634	STAMCOUNTER cTlbNativeMissTag;
635	/** TLB misses in native code due to flags or physical revision mismatch. */
636	STAMCOUNTER cTlbNativeMissFlagsAndPhysRev;
637	/** TLB misses in native code due to misaligned access. */
638	STAMCOUNTER cTlbNativeMissAlignment;
639	/** TLB misses in native code due to cross page access. */
640	uint32_t cTlbNativeMissCrossPage;
641	/** TLB misses in native code due to non-canonical address. */
642	uint32_t cTlbNativeMissNonCanonical;
643	/** @} */
644
645	/** Slow read path (code only). */
646	uint32_t cTlbSlowCodeReadPath;
647
648	/** Regular TLB flush count. */
649	uint32_t cTlsFlushes;
650	/** Global TLB flush count. */
651	uint32_t cTlsGlobalFlushes;
652	/** Revision rollovers. */
653	uint32_t cTlbRevisionRollovers;
654	/** Physical revision flushes. */
655	uint32_t cTlbPhysRevFlushes;
656	/** Physical revision rollovers. */
657	uint32_t cTlbPhysRevRollovers;
658
659	/** Number of INVLPG (and similar) operations. */
660	uint32_t cTlbInvlPg;
661	/** Subset of cTlbInvlPg that involved non-global large pages. */
662	uint32_t cTlbInvlPgLargeNonGlobal;
663	/** Subset of cTlbInvlPg that involved global large pages. */
664	uint32_t cTlbInvlPgLargeGlobal;
665
666	uint32_t au32Padding[13];
667
668	/** The TLB entries.
669	* Even entries are for PTE.G=0 and uses uTlbRevision.
670	* Odd entries are for PTE.G=1 and uses uTlbRevisionGlobal. */
671	IEMTLBENTRY aEntries[IEMTLB_ENTRY_COUNT * 2];
672	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
673	/** Bitmap tracking TLB entries for large pages.
674	* This duplicates IEMTLBE_F_PT_LARGE_PAGE for each TLB entry. */
675	uint64_t bmLargePage[IEMTLB_ENTRY_COUNT * 2 / 64];
676	#endif
677	} IEMTLB;
678	AssertCompileSizeAlignment(IEMTLB, 64);
679	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
680	AssertCompile(IEMTLB_ENTRY_COUNT >= 32 /* bmLargePage ASSUMPTION */);
681	#endif
682	/** The width (in bits) of the address portion of the TLB tag. */
683	#define IEMTLB_TAG_ADDR_WIDTH 36
684	/** IEMTLB::uTlbRevision increment. */
685	#define IEMTLB_REVISION_INCR RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH)
686	/** IEMTLB::uTlbRevision mask. */
687	#define IEMTLB_REVISION_MASK (~(RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH) - 1))
688
689	/** IEMTLB::uTlbPhysRev increment.
690	* @sa IEMTLBE_F_PHYS_REV */
691	#define IEMTLB_PHYS_REV_INCR RT_BIT_64(11)
692	AssertCompile(IEMTLBE_F_PHYS_REV == ~(IEMTLB_PHYS_REV_INCR - 1U));
693
694	/**
695	* Calculates the TLB tag for a virtual address but without TLB revision.
696	* @returns Tag value for indexing and comparing with IEMTLB::uTag.
697	* @param a_GCPtr The virtual address. Must be RTGCPTR or same size or
698	* the clearing of the top 16 bits won't work (if 32-bit
699	* we'll end up with mostly zeros).
700	*/
701	#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
702	/**
703	* Converts a TLB tag value into a even TLB index.
704	* @returns Index into IEMTLB::aEntries.
705	* @param a_uTag Value returned by IEMTLB_CALC_TAG.
706	*/
707	#if IEMTLB_ENTRY_COUNT == 256
708	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( (uint8_t)(a_uTag) * 2U )
709	#else
710	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( ((a_uTag) & (IEMTLB_ENTRY_COUNT - 1U)) * 2U )
711	AssertCompile(RT_IS_POWER_OF_TWO(IEMTLB_ENTRY_COUNT));
712	#endif
713	/**
714	* Converts a TLB tag value into an even TLB index.
715	* @returns Pointer into IEMTLB::aEntries corresponding to .
716	* @param a_pTlb The TLB.
717	* @param a_uTag Value returned by IEMTLB_CALC_TAG or
718	* IEMTLB_CALC_TAG_NO_REV.
719	*/
720	#define IEMTLB_TAG_TO_EVEN_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_EVEN_INDEX(a_uTag)] )
721
722	/** Converts a GC address to an even TLB index. */
723	#define IEMTLB_ADDR_TO_EVEN_INDEX(a_GCPtr) IEMTLB_TAG_TO_EVEN_INDEX(IEMTLB_CALC_TAG_NO_REV(a_GCPtr))
724
725
726	/** @def IEM_WITH_TLB_TRACE
727	* Enables the TLB tracing.
728	* Adjust buffer size in IEMR3Init. */
729	#if defined(DOXYGEN_RUNNING) \|\| 0
730	# define IEM_WITH_TLB_TRACE
731	#endif
732
733	#ifdef IEM_WITH_TLB_TRACE
734
735	/** TLB trace entry types. */
736	typedef enum : uint8_t
737	{
738	kIemTlbTraceType_Invalid,
739	kIemTlbTraceType_InvlPg,
740	kIemTlbTraceType_EvictSlot,
741	kIemTlbTraceType_LargeEvictSlot,
742	kIemTlbTraceType_LargeScan,
743	kIemTlbTraceType_Flush,
744	kIemTlbTraceType_FlushGlobal,
745	kIemTlbTraceType_Load,
746	kIemTlbTraceType_LoadGlobal,
747	kIemTlbTraceType_Load_Cr0, /*< x86 specific /
748	kIemTlbTraceType_Load_Cr3, /*< x86 specific /
749	kIemTlbTraceType_Load_Cr4, /*< x86 specific /
750	kIemTlbTraceType_Load_Efer, /*< x86 specific /
751	kIemTlbTraceType_Irq,
752	kIemTlbTraceType_Xcpt,
753	kIemTlbTraceType_IRet, /*< x86 specific /
754	kIemTlbTraceType_Tb_Compile,
755	kIemTlbTraceType_Tb_Exec_Threaded,
756	kIemTlbTraceType_Tb_Exec_Native,
757	kIemTlbTraceType_User0,
758	kIemTlbTraceType_User1,
759	kIemTlbTraceType_User2,
760	kIemTlbTraceType_User3,
761	} IEMTLBTRACETYPE;
762
763	/** TLB trace entry. */
764	typedef struct IEMTLBTRACEENTRY
765	{
766	/** The flattened RIP for the event. */
767	uint64_t rip;
768	/** The event type. */
769	IEMTLBTRACETYPE enmType;
770	/** Byte parameter - typically used as 'bool fDataTlb'. */
771	uint8_t bParam;
772	/** 16-bit parameter value. */
773	uint16_t u16Param;
774	/** 32-bit parameter value. */
775	uint32_t u32Param;
776	/** 64-bit parameter value. */
777	uint64_t u64Param;
778	/** 64-bit parameter value. */
779	uint64_t u64Param2;
780	} IEMTLBTRACEENTRY;
781	AssertCompileSize(IEMTLBTRACEENTRY, 32);
782	/** Pointer to a TLB trace entry. */
783	typedef IEMTLBTRACEENTRY *PIEMTLBTRACEENTRY;
784	/** Pointer to a const TLB trace entry. */
785	typedef IEMTLBTRACEENTRY const *PCIEMTLBTRACEENTRY;
786	#endif /* !IEM_WITH_TLB_TRACE */
787
788	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
789	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) \
790	iemTlbTrace(a_pVCpu, kIemTlbTraceType_InvlPg, a_GCPtr)
791	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
792	iemTlbTrace(a_pVCpu, kIemTlbTraceType_EvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
793	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
794	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeEvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
795	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) \
796	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeScan, 0, 0, a_fDataTlb, (uint8_t)a_fGlobal \| ((uint8_t)a_fNonGlobal << 1))
797	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) \
798	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Flush, a_uRev, 0, a_fDataTlb)
799	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) \
800	iemTlbTrace(a_pVCpu, kIemTlbTraceType_FlushGlobal, a_uRev, a_uGRev, a_fDataTlb)
801	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
802	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
803	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
804	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LoadGlobal, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
805	#else
806	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) do { } while (0)
807	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
808	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
809	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) do { } while (0)
810	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) do { } while (0)
811	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) do { } while (0)
812	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
813	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
814	#endif
815
816	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
817	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr0, a_uNew, a_uOld)
818	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr3, a_uNew, a_uOld)
819	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr4, a_uNew, a_uOld)
820	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Efer, a_uNew, a_uOld)
821	#else
822	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) do { } while (0)
823	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) do { } while (0)
824	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) do { } while (0)
825	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) do { } while (0)
826	#endif
827
828	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
829	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) \
830	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Irq, a_fEFlags, 0, a_uVector, a_fFlags)
831	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) \
832	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Xcpt, a_uErr, a_uCr2, a_uVector, a_fFlags)
833	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) \
834	iemTlbTrace(a_pVCpu, kIemTlbTraceType_IRet, a_uRetRip, a_fEFlags, 0, a_uRetCs)
835	#else
836	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) do { } while (0)
837	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) do { } while (0)
838	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) do { } while (0)
839	#endif
840
841	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
842	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) \
843	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Compile, a_GCPhysPc)
844	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) \
845	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Threaded, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
846	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) \
847	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Native, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
848	#else
849	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) do { } while (0)
850	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) do { } while (0)
851	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) do { } while (0)
852	#endif
853
854	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
855	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
856	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User0, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
857	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
858	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User1, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
859	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
860	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User2, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
861	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
862	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User3, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
863	#else
864	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
865	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
866	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
867	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
868	#endif
869
870
871	/** @name IEM_MC_F_XXX - MC block flags/clues.
872	* @note x86 specific
873	* @todo Merge with IEM_CIMPL_F_XXX
874	* @{ */
875	#define IEM_MC_F_ONLY_8086 RT_BIT_32(0)
876	#define IEM_MC_F_MIN_186 RT_BIT_32(1)
877	#define IEM_MC_F_MIN_286 RT_BIT_32(2)
878	#define IEM_MC_F_NOT_286_OR_OLDER IEM_MC_F_MIN_386
879	#define IEM_MC_F_MIN_386 RT_BIT_32(3)
880	#define IEM_MC_F_MIN_486 RT_BIT_32(4)
881	#define IEM_MC_F_MIN_PENTIUM RT_BIT_32(5)
882	#define IEM_MC_F_MIN_PENTIUM_II IEM_MC_F_MIN_PENTIUM
883	#define IEM_MC_F_MIN_CORE IEM_MC_F_MIN_PENTIUM
884	#define IEM_MC_F_64BIT RT_BIT_32(6)
885	#define IEM_MC_F_NOT_64BIT RT_BIT_32(7)
886	/** This is set by IEMAllN8vePython.py to indicate a variation with the
887	* flags-clearing-and-checking. */
888	#define IEM_MC_F_WITH_FLAGS RT_BIT_32(8)
889	/** This is set by IEMAllN8vePython.py to indicate a variation without the
890	* flags-clearing-and-checking, when there is also a variation with that.
891	* @note Do not set this manully, it's only for python and for testing in
892	* the native recompiler! */
893	#define IEM_MC_F_WITHOUT_FLAGS RT_BIT_32(9)
894	/** @} */
895
896	/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
897	*
898	* These clues are mainly for the recompiler, so that it can emit correct code.
899	*
900	* They are processed by the python script and which also automatically
901	* calculates flags for MC blocks based on the statements, extending the use of
902	* these flags to describe MC block behavior to the recompiler core. The python
903	* script pass the flags to the IEM_MC2_END_EMIT_CALLS macro, but mainly for
904	* error checking purposes. The script emits the necessary fEndTb = true and
905	* similar statements as this reduces compile time a tiny bit.
906	*
907	* @{ */
908	/** Flag set if direct branch, clear if absolute or indirect. */
909	#define IEM_CIMPL_F_BRANCH_DIRECT RT_BIT_32(0)
910	/** Flag set if indirect branch, clear if direct or relative.
911	* This is also used for all system control transfers (SYSCALL, SYSRET, INT, ++)
912	* as well as for return instructions (RET, IRET, RETF). */
913	#define IEM_CIMPL_F_BRANCH_INDIRECT RT_BIT_32(1)
914	/** Flag set if relative branch, clear if absolute or indirect. */
915	#define IEM_CIMPL_F_BRANCH_RELATIVE RT_BIT_32(2)
916	/** Flag set if conditional branch, clear if unconditional. */
917	#define IEM_CIMPL_F_BRANCH_CONDITIONAL RT_BIT_32(3)
918	/** Flag set if it's a far branch (changes CS).
919	* @note x86 specific */
920	#define IEM_CIMPL_F_BRANCH_FAR RT_BIT_32(4)
921	/** Convenience: Testing any kind of branch. */
922	#define IEM_CIMPL_F_BRANCH_ANY (IEM_CIMPL_F_BRANCH_DIRECT \| IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_RELATIVE)
923
924	/** Execution flags may change (IEMCPU::fExec). */
925	#define IEM_CIMPL_F_MODE RT_BIT_32(5)
926	/** May change significant portions of RFLAGS.
927	* @note x86 specific */
928	#define IEM_CIMPL_F_RFLAGS RT_BIT_32(6)
929	/** May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS.
930	* @note x86 specific */
931	#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(7)
932	/** May trigger interrupt shadowing.
933	* @note x86 specific */
934	#define IEM_CIMPL_F_INHIBIT_SHADOW RT_BIT_32(8)
935	/** May enable interrupts, so recheck IRQ immediately afterwards executing
936	* the instruction. */
937	#define IEM_CIMPL_F_CHECK_IRQ_AFTER RT_BIT_32(9)
938	/** May disable interrupts, so recheck IRQ immediately before executing the
939	* instruction. */
940	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE RT_BIT_32(10)
941	/** Convenience: Check for IRQ both before and after an instruction. */
942	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE_AND_AFTER (IEM_CIMPL_F_CHECK_IRQ_BEFORE \| IEM_CIMPL_F_CHECK_IRQ_AFTER)
943	/** May trigger a VM exit (treated like IEM_CIMPL_F_MODE atm). */
944	#define IEM_CIMPL_F_VMEXIT RT_BIT_32(11)
945	/** May modify FPU state.
946	* @todo Not sure if this is useful yet. */
947	#define IEM_CIMPL_F_FPU RT_BIT_32(12)
948	/** REP prefixed instruction which may yield before updating PC.
949	* @todo Not sure if this is useful, REP functions now return non-zero
950	* status if they don't update the PC.
951	* @note x86 specific */
952	#define IEM_CIMPL_F_REP RT_BIT_32(13)
953	/** I/O instruction.
954	* @todo Not sure if this is useful yet.
955	* @note x86 specific */
956	#define IEM_CIMPL_F_IO RT_BIT_32(14)
957	/** Force end of TB after the instruction. */
958	#define IEM_CIMPL_F_END_TB RT_BIT_32(15)
959	/** Flag set if a branch may also modify the stack (push/pop return address). */
960	#define IEM_CIMPL_F_BRANCH_STACK RT_BIT_32(16)
961	/** Flag set if a branch may also modify the stack (push/pop return address)
962	* and switch it (load/restore SS:RSP).
963	* @note x86 specific */
964	#define IEM_CIMPL_F_BRANCH_STACK_FAR RT_BIT_32(17)
965	/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
966	#define IEM_CIMPL_F_XCPT \
967	(IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_FAR \| IEM_CIMPL_F_BRANCH_STACK_FAR \
968	\| IEM_CIMPL_F_MODE \| IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_VMEXIT)
969
970	/** The block calls a C-implementation instruction function with two implicit arguments.
971	* Mutually exclusive with IEM_CIMPL_F_CALLS_AIMPL and
972	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
973	* @note The python scripts will add this if missing. */
974	#define IEM_CIMPL_F_CALLS_CIMPL RT_BIT_32(18)
975	/** The block calls an ASM-implementation instruction function.
976	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and
977	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
978	* @note The python scripts will add this if missing. */
979	#define IEM_CIMPL_F_CALLS_AIMPL RT_BIT_32(19)
980	/** The block calls an ASM-implementation instruction function with an implicit
981	* X86FXSTATE pointer argument.
982	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
983	* IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE.
984	* @note The python scripts will add this if missing.
985	* @note x86 specific */
986	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE RT_BIT_32(20)
987	/** The block calls an ASM-implementation instruction function with an implicit
988	* X86XSAVEAREA pointer argument.
989	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
990	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
991	* @note No different from IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE, so same value.
992	* @note The python scripts will add this if missing.
993	* @note x86 specific */
994	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE
995	/** @} */
996
997
998	/** @name IEM_F_XXX - Execution mode flags (IEMCPU::fExec, IEMTB::fFlags).
999	*
1000	* These flags are set when entering IEM and adjusted as code is executed, such
1001	* that they will always contain the current values as instructions are
1002	* finished.
1003	*
1004	* In recompiled execution mode, (most of) these flags are included in the
1005	* translation block selection key and stored in IEMTB::fFlags alongside the
1006	* IEMTB_F_XXX flags. The latter flags uses bits 31 thru 24, which are all zero
1007	* in IEMCPU::fExec.
1008	*
1009	* @{ */
1010	/** Mode: The block target mode mask. */
1011	#define IEM_F_MODE_MASK UINT32_C(0x0000001f)
1012	/** Mode: The IEMMODE part of the IEMTB_F_MODE_MASK value. */
1013	#define IEM_F_MODE_CPUMODE_MASK UINT32_C(0x00000003)
1014	/** X86 Mode: Bit used to indicating pre-386 CPU in 16-bit mode (for eliminating
1015	* conditional in EIP/IP updating), and flat wide open CS, SS, DS, and ES in
1016	* 32-bit mode (for simplifying most memory accesses). */
1017	#define IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK UINT32_C(0x00000004)
1018	/** X86 Mode: Bit indicating protected mode, real mode (or SMM) when not set. */
1019	#define IEM_F_MODE_X86_PROT_MASK UINT32_C(0x00000008)
1020	/** X86 Mode: Bit used to indicate virtual 8086 mode (only 16-bit). */
1021	#define IEM_F_MODE_X86_V86_MASK UINT32_C(0x00000010)
1022
1023	/** X86 Mode: 16-bit on 386 or later. */
1024	#define IEM_F_MODE_X86_16BIT UINT32_C(0x00000000)
1025	/** X86 Mode: 80286, 80186 and 8086/88 targetting blocks (EIP update opt). */
1026	#define IEM_F_MODE_X86_16BIT_PRE_386 UINT32_C(0x00000004)
1027	/** X86 Mode: 16-bit protected mode on 386 or later. */
1028	#define IEM_F_MODE_X86_16BIT_PROT UINT32_C(0x00000008)
1029	/** X86 Mode: 16-bit protected mode on 386 or later. */
1030	#define IEM_F_MODE_X86_16BIT_PROT_PRE_386 UINT32_C(0x0000000c)
1031	/** X86 Mode: 16-bit virtual 8086 protected mode (on 386 or later). */
1032	#define IEM_F_MODE_X86_16BIT_PROT_V86 UINT32_C(0x00000018)
1033
1034	/** X86 Mode: 32-bit on 386 or later. */
1035	#define IEM_F_MODE_X86_32BIT UINT32_C(0x00000001)
1036	/** X86 Mode: 32-bit mode with wide open flat CS, SS, DS and ES. */
1037	#define IEM_F_MODE_X86_32BIT_FLAT UINT32_C(0x00000005)
1038	/** X86 Mode: 32-bit protected mode. */
1039	#define IEM_F_MODE_X86_32BIT_PROT UINT32_C(0x00000009)
1040	/** X86 Mode: 32-bit protected mode with wide open flat CS, SS, DS and ES. */
1041	#define IEM_F_MODE_X86_32BIT_PROT_FLAT UINT32_C(0x0000000d)
1042
1043	/** X86 Mode: 64-bit (includes protected, but not the flat bit). */
1044	#define IEM_F_MODE_X86_64BIT UINT32_C(0x0000000a)
1045
1046	/** X86 Mode: Checks if @a a_fExec represent a FLAT mode. */
1047	#define IEM_F_MODE_X86_IS_FLAT(a_fExec) ( ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT \
1048	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT \
1049	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT)
1050
1051	/** Bypass access handlers when set. */
1052	#define IEM_F_BYPASS_HANDLERS UINT32_C(0x00010000)
1053	/** Have pending hardware instruction breakpoints. */
1054	#define IEM_F_PENDING_BRK_INSTR UINT32_C(0x00020000)
1055	/** Have pending hardware data breakpoints. */
1056	#define IEM_F_PENDING_BRK_DATA UINT32_C(0x00040000)
1057
1058	/** X86: Have pending hardware I/O breakpoints. */
1059	#define IEM_F_PENDING_BRK_X86_IO UINT32_C(0x00000400)
1060	/** X86: Disregard the lock prefix (implied or not) when set. */
1061	#define IEM_F_X86_DISREGARD_LOCK UINT32_C(0x00000800)
1062
1063	/** Pending breakpoint mask (what iemCalcExecDbgFlags works out). */
1064	#define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR \| IEM_F_PENDING_BRK_DATA \| IEM_F_PENDING_BRK_X86_IO)
1065
1066	/** Caller configurable options. */
1067	#define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS \| IEM_F_X86_DISREGARD_LOCK)
1068
1069	/** X86: The current protection level (CPL) shift factor. */
1070	#define IEM_F_X86_CPL_SHIFT 8
1071	/** X86: The current protection level (CPL) mask. */
1072	#define IEM_F_X86_CPL_MASK UINT32_C(0x00000300)
1073	/** X86: The current protection level (CPL) shifted mask. */
1074	#define IEM_F_X86_CPL_SMASK UINT32_C(0x00000003)
1075
1076	/** X86: Alignment checks enabled (CR0.AM=1 & EFLAGS.AC=1). */
1077	#define IEM_F_X86_AC UINT32_C(0x00080000)
1078
1079	/** X86 execution context.
1080	* The IEM_F_X86_CTX_XXX values are individual flags that can be combined (with
1081	* the exception of IEM_F_X86_CTX_NORMAL). This allows running VMs from SMM
1082	* mode. */
1083	#define IEM_F_X86_CTX_MASK UINT32_C(0x0000f000)
1084	/** X86 context: Plain regular execution context. */
1085	#define IEM_F_X86_CTX_NORMAL UINT32_C(0x00000000)
1086	/** X86 context: VT-x enabled. */
1087	#define IEM_F_X86_CTX_VMX UINT32_C(0x00001000)
1088	/** X86 context: AMD-V enabled. */
1089	#define IEM_F_X86_CTX_SVM UINT32_C(0x00002000)
1090	/** X86 context: In AMD-V or VT-x guest mode. */
1091	#define IEM_F_X86_CTX_IN_GUEST UINT32_C(0x00004000)
1092	/** X86 context: System management mode (SMM). */
1093	#define IEM_F_X86_CTX_SMM UINT32_C(0x00008000)
1094
1095	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1096	* iemRegFinishClearingRF() most for most situations (CPUMCTX_DBG_HIT_DRX_MASK
1097	* and CPUMCTX_DBG_DBGF_MASK are covered by the IEM_F_PENDING_BRK_XXX bits
1098	* alread). */
1099
1100	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1101	* iemRegFinishClearingRF() most for most situations
1102	* (CPUMCTX_DBG_HIT_DRX_MASK and CPUMCTX_DBG_DBGF_MASK are covered by
1103	* the IEM_F_PENDING_BRK_XXX bits alread). */
1104
1105	/** @} */
1106
1107
1108	/** @name IEMTB_F_XXX - Translation block flags (IEMTB::fFlags).
1109	*
1110	* Extends the IEM_F_XXX flags (subject to IEMTB_F_IEM_F_MASK) to make up the
1111	* translation block flags. The combined flag mask (subject to
1112	* IEMTB_F_KEY_MASK) is used as part of the lookup key for translation blocks.
1113	*
1114	* @{ */
1115	/** Mask of IEM_F_XXX flags included in IEMTB_F_XXX. */
1116	#define IEMTB_F_IEM_F_MASK UINT32_C(0x00ffffff)
1117
1118	/** Type: The block type mask. */
1119	#define IEMTB_F_TYPE_MASK UINT32_C(0x03000000)
1120	/** Type: Purly threaded recompiler (via tables). */
1121	#define IEMTB_F_TYPE_THREADED UINT32_C(0x01000000)
1122	/** Type: Native recompilation. */
1123	#define IEMTB_F_TYPE_NATIVE UINT32_C(0x02000000)
1124
1125	/** Set when we're starting the block in an "interrupt shadow".
1126	* We don't need to distingish between the two types of this mask, thus the one.
1127	* @see CPUMCTX_INHIBIT_SHADOW, CPUMIsInInterruptShadow() */
1128	#define IEMTB_F_INHIBIT_SHADOW UINT32_C(0x04000000)
1129	/** Set when we're currently inhibiting NMIs
1130	* @see CPUMCTX_INHIBIT_NMI, CPUMAreInterruptsInhibitedByNmi() */
1131	#define IEMTB_F_INHIBIT_NMI UINT32_C(0x08000000)
1132
1133	/** Checks that EIP/IP is wihin CS.LIM before each instruction. Used when
1134	* we're close the limit before starting a TB, as determined by
1135	* iemGetTbFlagsForCurrentPc().
1136	* @note x86 specific */
1137	#define IEMTB_F_CS_LIM_CHECKS UINT32_C(0x10000000)
1138
1139	/** Mask of the IEMTB_F_XXX flags that are part of the TB lookup key.
1140	*
1141	* @note We skip all of IEM_F_X86_CTX_MASK, with the exception of SMM (which we
1142	* don't implement), because we don't currently generate any context
1143	* specific code - that's all handled in CIMPL functions.
1144	*
1145	* For the threaded recompiler we don't generate any CPL specific code
1146	* either, but the native recompiler does for memory access (saves getting
1147	* the CPL from fExec and turning it into IEMTLBE_F_PT_NO_USER).
1148	* Since most OSes will not share code between rings, this shouldn't
1149	* have any real effect on TB/memory/recompiling load.
1150	*/
1151	#define IEMTB_F_KEY_MASK ((UINT32_MAX & ~(IEM_F_X86_CTX_MASK \| IEMTB_F_TYPE_MASK)) \| IEM_F_X86_CTX_SMM)
1152	/** @} */
1153
1154	AssertCompile( (IEM_F_MODE_X86_16BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1155	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1156	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_PROT_MASK));
1157	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_V86_MASK));
1158	AssertCompile( (IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1159	AssertCompile( IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1160	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_PROT_MASK));
1161	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1162	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1163	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1164	AssertCompile( IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1165	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_V86_MASK));
1166	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1167	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1168	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_PROT_MASK);
1169	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1170	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_PROT_MASK);
1171	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1172	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_V86_MASK);
1173
1174	AssertCompile( (IEM_F_MODE_X86_32BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1175	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1176	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_PROT_MASK));
1177	AssertCompile( (IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1178	AssertCompile( IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1179	AssertCompile(!(IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_PROT_MASK));
1180	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1181	AssertCompile(!(IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1182	AssertCompile( IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1183	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1184	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1185	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_PROT_MASK);
1186
1187	AssertCompile( (IEM_F_MODE_X86_64BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT);
1188	AssertCompile( IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_PROT_MASK);
1189	AssertCompile(!(IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1190
1191	/** Native instruction type for use with the native code generator.
1192	* This is a byte (uint8_t) for x86 and amd64 and uint32_t for the other(s). */
1193	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
1194	typedef uint8_t IEMNATIVEINSTR;
1195	#else
1196	typedef uint32_t IEMNATIVEINSTR;
1197	#endif
1198	/** Pointer to a native instruction unit. */
1199	typedef IEMNATIVEINSTR *PIEMNATIVEINSTR;
1200	/** Pointer to a const native instruction unit. */
1201	typedef IEMNATIVEINSTR const *PCIEMNATIVEINSTR;
1202
1203	/**
1204	* A call for the threaded call table.
1205	*/
1206	typedef struct IEMTHRDEDCALLENTRY
1207	{
1208	/** The function to call (IEMTHREADEDFUNCS). */
1209	uint16_t enmFunction;
1210
1211	/** Instruction number in the TB (for statistics). */
1212	uint8_t idxInstr;
1213	/** The opcode length. */
1214	uint8_t cbOpcode;
1215	/** Offset into IEMTB::pabOpcodes. */
1216	uint16_t offOpcode;
1217
1218	/** TB lookup table index (7 bits) and large size (1 bits).
1219	*
1220	* The default size is 1 entry, but for indirect calls and returns we set the
1221	* top bit and allocate 4 (IEM_TB_LOOKUP_TAB_LARGE_SIZE) entries. The large
1222	* tables uses RIP for selecting the entry to use, as it is assumed a hash table
1223	* lookup isn't that slow compared to sequentially trying out 4 TBs.
1224	*
1225	* By default lookup table entry 0 for a TB is reserved as a fallback for
1226	* calltable entries w/o explicit entreis, so this member will be non-zero if
1227	* there is a lookup entry associated with this call.
1228	*
1229	* @sa IEM_TB_LOOKUP_TAB_GET_SIZE, IEM_TB_LOOKUP_TAB_GET_IDX
1230	*/
1231	uint8_t uTbLookup;
1232
1233	/** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
1234	uint8_t fFlags;
1235
1236	/** Generic parameters. */
1237	uint64_t auParams[3];
1238	} IEMTHRDEDCALLENTRY;
1239	AssertCompileSize(IEMTHRDEDCALLENTRY, sizeof(uint64_t) * 4);
1240	/** Pointer to a threaded call entry. */
1241	typedef struct IEMTHRDEDCALLENTRY *PIEMTHRDEDCALLENTRY;
1242	/** Pointer to a const threaded call entry. */
1243	typedef IEMTHRDEDCALLENTRY const *PCIEMTHRDEDCALLENTRY;
1244
1245	/** The number of TB lookup table entries for a large allocation
1246	* (IEMTHRDEDCALLENTRY::uTbLookup bit 7 set). */
1247	#define IEM_TB_LOOKUP_TAB_LARGE_SIZE 4
1248	/** Get the lookup table size from IEMTHRDEDCALLENTRY::uTbLookup. */
1249	#define IEM_TB_LOOKUP_TAB_GET_SIZE(a_uTbLookup) (!((a_uTbLookup) & 0x80) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE)
1250	/** Get the first lookup table index from IEMTHRDEDCALLENTRY::uTbLookup. */
1251	#define IEM_TB_LOOKUP_TAB_GET_IDX(a_uTbLookup) ((a_uTbLookup) & 0x7f)
1252	/** Get the lookup table index from IEMTHRDEDCALLENTRY::uTbLookup and RIP. */
1253	#define IEM_TB_LOOKUP_TAB_GET_IDX_WITH_RIP(a_uTbLookup, a_Rip) \
1254	(!((a_uTbLookup) & 0x80) ? (a_uTbLookup) & 0x7f : ((a_uTbLookup) & 0x7f) + ((a_Rip) & (IEM_TB_LOOKUP_TAB_LARGE_SIZE - 1)) )
1255
1256	/** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
1257	#define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge) ((a_idxTable) \| ((a_fLarge) ? 0x80 : 0))
1258
1259
1260	/** The call entry is a jump target. */
1261	#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET UINT8_C(0x01)
1262
1263
1264	/**
1265	* Native IEM TB 'function' typedef.
1266	*
1267	* This will throw/longjmp on occation.
1268	*
1269	* @note AMD64 doesn't have that many non-volatile registers and does sport
1270	* 32-bit address displacments, so we don't need pCtx.
1271	*
1272	* On ARM64 pCtx allows us to directly address the whole register
1273	* context without requiring a separate indexing register holding the
1274	* offset. This saves an instruction loading the offset for each guest
1275	* CPU context access, at the cost of a non-volatile register.
1276	* Fortunately, ARM64 has quite a lot more registers.
1277	*/
1278	typedef
1279	#ifdef RT_ARCH_AMD64
1280	int FNIEMTBNATIVE(PVMCPUCC pVCpu)
1281	#else
1282	int FNIEMTBNATIVE(PVMCPUCC pVCpu, PCPUMCTX pCtx)
1283	#endif
1284	#if RT_CPLUSPLUS_PREREQ(201700)
1285	IEM_NOEXCEPT_MAY_LONGJMP
1286	#endif
1287	;
1288	/** Pointer to a native IEM TB entry point function.
1289	* This will throw/longjmp on occation. */
1290	typedef FNIEMTBNATIVE *PFNIEMTBNATIVE;
1291
1292
1293	/**
1294	* Translation block.
1295	*
1296	* The current plan is to just keep TBs and associated lookup hash table private
1297	* to each VCpu as that simplifies TB removal greatly (no races) and generally
1298	* avoids using expensive atomic primitives for updating lists and stuff.
1299	*/
1300	#pragma pack(2) /* to prevent the Thrd structure from being padded unnecessarily */
1301	typedef struct IEMTB
1302	{
1303	/** Next block with the same hash table entry. */
1304	struct IEMTB *pNext;
1305	/** Usage counter. */
1306	uint32_t cUsed;
1307	/** The IEMCPU::msRecompilerPollNow last time it was used. */
1308	uint32_t msLastUsed;
1309
1310	/** @name What uniquely identifies the block.
1311	* @{ */
1312	RTGCPHYS GCPhysPc;
1313	/** IEMTB_F_XXX (i.e. IEM_F_XXX ++). */
1314	uint32_t fFlags;
1315	union
1316	{
1317	struct
1318	{
1319	/*< Relevant CS X86DESCATTR_XXX bits. /
1320	uint16_t fAttr;
1321	} x86;
1322	};
1323	/** @} */
1324
1325	/** Number of opcode ranges. */
1326	uint8_t cRanges;
1327	/** Statistics: Number of instructions in the block. */
1328	uint8_t cInstructions;
1329
1330	/** Type specific info. */
1331	union
1332	{
1333	struct
1334	{
1335	/** The call sequence table. */
1336	PIEMTHRDEDCALLENTRY paCalls;
1337	/** Number of calls in paCalls. */
1338	uint16_t cCalls;
1339	/** Number of calls allocated. */
1340	uint16_t cAllocated;
1341	} Thrd;
1342	struct
1343	{
1344	/** The native instructions (PFNIEMTBNATIVE). */
1345	PIEMNATIVEINSTR paInstructions;
1346	/** Number of instructions pointed to by paInstructions. */
1347	uint32_t cInstructions;
1348	} Native;
1349	/** Generic view for zeroing when freeing. */
1350	struct
1351	{
1352	uintptr_t uPtr;
1353	uint32_t uData;
1354	} Gen;
1355	};
1356
1357	/** The allocation chunk this TB belongs to. */
1358	uint8_t idxAllocChunk;
1359	/** The number of entries in the lookup table.
1360	* Because we're out of space, the TB lookup table is located before the
1361	* opcodes pointed to by pabOpcodes. */
1362	uint8_t cTbLookupEntries;
1363
1364	/** Number of bytes of opcodes stored in pabOpcodes.
1365	* @todo this field isn't really needed, aRanges keeps the actual info. */
1366	uint16_t cbOpcodes;
1367	/** Pointer to the opcode bytes this block was recompiled from.
1368	* This also points to the TB lookup table, which starts cTbLookupEntries
1369	* entries before the opcodes (we don't have room atm for another point). */
1370	uint8_t *pabOpcodes;
1371
1372	union
1373	{
1374	/** Native recompilation debug info if enabled.
1375	* This is only generated by the native recompiler. */
1376	struct IEMTBDBG *pDbgInfo;
1377	/** For threaded TBs and natives when debug info is disabled, this is the flat
1378	* PC corresponding to GCPhysPc. */
1379	RTGCPTR FlatPc;
1380	};
1381
1382	/* --- 64 byte cache line end --- */
1383
1384	/** Opcode ranges.
1385	*
1386	* The opcode checkers and maybe TLB loading functions will use this to figure
1387	* out what to do. The parameter will specify an entry and the opcode offset to
1388	* start at and the minimum number of bytes to verify (instruction length).
1389	*
1390	* When VT-x and AMD-V looks up the opcode bytes for an exitting instruction,
1391	* they'll first translate RIP (+ cbInstr - 1) to a physical address using the
1392	* code TLB (must have a valid entry for that address) and scan the ranges to
1393	* locate the corresponding opcodes. Probably.
1394	*/
1395	struct IEMTBOPCODERANGE
1396	{
1397	/** Offset within pabOpcodes. */
1398	uint16_t offOpcodes;
1399	/** Number of bytes. */
1400	uint16_t cbOpcodes;
1401	/** The page offset. */
1402	RT_GCC_EXTENSION
1403	uint16_t offPhysPage : 12;
1404	/** Unused bits. */
1405	RT_GCC_EXTENSION
1406	uint16_t u2Unused : 2;
1407	/** Index into GCPhysPc + aGCPhysPages for the physical page address. */
1408	RT_GCC_EXTENSION
1409	uint16_t idxPhysPage : 2;
1410	} aRanges[8];
1411
1412	/** Physical pages that this TB covers.
1413	* The GCPhysPc w/o page offset is element zero, so starting here with 1. */
1414	RTGCPHYS aGCPhysPages[2];
1415	} IEMTB;
1416	#pragma pack()
1417	AssertCompileMemberAlignment(IEMTB, GCPhysPc, sizeof(RTGCPHYS));
1418	AssertCompileMemberAlignment(IEMTB, Thrd, sizeof(void *));
1419	AssertCompileMemberAlignment(IEMTB, pabOpcodes, sizeof(void *));
1420	AssertCompileMemberAlignment(IEMTB, pDbgInfo, sizeof(void *));
1421	AssertCompileMemberAlignment(IEMTB, aGCPhysPages, sizeof(RTGCPHYS));
1422	AssertCompileMemberOffset(IEMTB, aRanges, 64);
1423	AssertCompileMemberSize(IEMTB, aRanges[0], 6);
1424	#if 1
1425	AssertCompileSize(IEMTB, 128);
1426	# define IEMTB_SIZE_IS_POWER_OF_TWO /*< The IEMTB size is a power of two. /
1427	#else
1428	AssertCompileSize(IEMTB, 168);
1429	# undef IEMTB_SIZE_IS_POWER_OF_TWO
1430	#endif
1431
1432	/** Pointer to a translation block. */
1433	typedef IEMTB *PIEMTB;
1434	/** Pointer to a const translation block. */
1435	typedef IEMTB const *PCIEMTB;
1436
1437	/** Gets address of the given TB lookup table entry. */
1438	#define IEMTB_GET_TB_LOOKUP_TAB_ENTRY(a_pTb, a_idx) \
1439	((PIEMTB )&(a_pTb)->pabOpcodes[-(int)((a_pTb)->cTbLookupEntries - (a_idx)) sizeof(PIEMTB)])
1440
1441	/**
1442	* Gets the physical address for a TB opcode range.
1443	*/
1444	DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1445	{
1446	Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1447	uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1448	Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1449	if (idxPage == 0)
1450	return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1451	Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1452	return pTb->aGCPhysPages[idxPage - 1];
1453	}
1454
1455
1456	/**
1457	* A chunk of memory in the TB allocator.
1458	*/
1459	typedef struct IEMTBCHUNK
1460	{
1461	/** Pointer to the translation blocks in this chunk. */
1462	PIEMTB paTbs;
1463	#ifdef IN_RING0
1464	/** Allocation handle. */
1465	RTR0MEMOBJ hMemObj;
1466	#endif
1467	} IEMTBCHUNK;
1468
1469	/**
1470	* A per-CPU translation block allocator.
1471	*
1472	* Because of how the IEMTBCACHE uses the lower 6 bits of the TB address to keep
1473	* the length of the collision list, and of course also for cache line alignment
1474	* reasons, the TBs must be allocated with at least 64-byte alignment.
1475	* Memory is there therefore allocated using one of the page aligned allocators.
1476	*
1477	*
1478	* To avoid wasting too much memory, it is allocated piecemeal as needed,
1479	* in chunks (IEMTBCHUNK) of 2 MiB or more. The TB has an 8-bit chunk index
1480	* that enables us to quickly calculate the allocation bitmap position when
1481	* freeing the translation block.
1482	*/
1483	typedef struct IEMTBALLOCATOR
1484	{
1485	/** Magic value (IEMTBALLOCATOR_MAGIC). */
1486	uint32_t uMagic;
1487
1488	#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
1489	/** Mask corresponding to cTbsPerChunk - 1. */
1490	uint32_t fChunkMask;
1491	/** Shift count corresponding to cTbsPerChunk. */
1492	uint8_t cChunkShift;
1493	#else
1494	uint32_t uUnused;
1495	uint8_t bUnused;
1496	#endif
1497	/** Number of chunks we're allowed to allocate. */
1498	uint8_t cMaxChunks;
1499	/** Number of chunks currently populated. */
1500	uint16_t cAllocatedChunks;
1501	/** Number of translation blocks per chunk. */
1502	uint32_t cTbsPerChunk;
1503	/** Chunk size. */
1504	uint32_t cbPerChunk;
1505
1506	/** The maximum number of TBs. */
1507	uint32_t cMaxTbs;
1508	/** Total number of TBs in the populated chunks.
1509	* (cAllocatedChunks * cTbsPerChunk) */
1510	uint32_t cTotalTbs;
1511	/** The current number of TBs in use.
1512	* The number of free TBs: cAllocatedTbs - cInUseTbs; */
1513	uint32_t cInUseTbs;
1514	/** Statistics: Number of the cInUseTbs that are native ones. */
1515	uint32_t cNativeTbs;
1516	/** Statistics: Number of the cInUseTbs that are threaded ones. */
1517	uint32_t cThreadedTbs;
1518
1519	/** Where to start pruning TBs from when we're out.
1520	* See iemTbAllocatorAllocSlow for details. */
1521	uint32_t iPruneFrom;
1522	/** Where to start pruning native TBs from when we're out of executable memory.
1523	* See iemTbAllocatorFreeupNativeSpace for details. */
1524	uint32_t iPruneNativeFrom;
1525	uint64_t u64Padding;
1526
1527	/** Statistics: Number of TB allocation calls. */
1528	STAMCOUNTER StatAllocs;
1529	/** Statistics: Number of TB free calls. */
1530	STAMCOUNTER StatFrees;
1531	/** Statistics: Time spend pruning. */
1532	STAMPROFILE StatPrune;
1533	/** Statistics: Time spend pruning native TBs. */
1534	STAMPROFILE StatPruneNative;
1535
1536	/** The delayed free list (see iemTbAlloctorScheduleForFree). */
1537	PIEMTB pDelayedFreeHead;
1538	/* Head of the list of free TBs. */
1539	PIEMTB pTbsFreeHead;
1540
1541	/** Allocation chunks. */
1542	IEMTBCHUNK aChunks[256];
1543	} IEMTBALLOCATOR;
1544	/** Pointer to a TB allocator. */
1545	typedef struct IEMTBALLOCATOR *PIEMTBALLOCATOR;
1546
1547	/** Magic value for the TB allocator (Emmet Harley Cohen). */
1548	#define IEMTBALLOCATOR_MAGIC UINT32_C(0x19900525)
1549
1550
1551	/**
1552	* A per-CPU translation block cache (hash table).
1553	*
1554	* The hash table is allocated once during IEM initialization and size double
1555	* the max TB count, rounded up to the nearest power of two (so we can use and
1556	* AND mask rather than a rest division when hashing).
1557	*/
1558	typedef struct IEMTBCACHE
1559	{
1560	/** Magic value (IEMTBCACHE_MAGIC). */
1561	uint32_t uMagic;
1562	/** Size of the hash table. This is a power of two. */
1563	uint32_t cHash;
1564	/** The mask corresponding to cHash. */
1565	uint32_t uHashMask;
1566	uint32_t uPadding;
1567
1568	/** @name Statistics
1569	* @{ */
1570	/** Number of collisions ever. */
1571	STAMCOUNTER cCollisions;
1572
1573	/** Statistics: Number of TB lookup misses. */
1574	STAMCOUNTER cLookupMisses;
1575	/** Statistics: Number of TB lookup hits via hash table (debug only). */
1576	STAMCOUNTER cLookupHits;
1577	/** Statistics: Number of TB lookup hits via TB associated lookup table (debug only). */
1578	STAMCOUNTER cLookupHitsViaTbLookupTable;
1579	STAMCOUNTER auPadding2[2];
1580	/** Statistics: Collision list length pruning. */
1581	STAMPROFILE StatPrune;
1582	/** @} */
1583
1584	/** The hash table itself.
1585	* @note The lower 6 bits of the pointer is used for keeping the collision
1586	* list length, so we can take action when it grows too long.
1587	* This works because TBs are allocated using a 64 byte (or
1588	* higher) alignment from page aligned chunks of memory, so the lower
1589	* 6 bits of the address will always be zero.
1590	* See IEMTBCACHE_PTR_COUNT_MASK, IEMTBCACHE_PTR_MAKE and friends.
1591	*/
1592	RT_FLEXIBLE_ARRAY_EXTENSION
1593	PIEMTB apHash[RT_FLEXIBLE_ARRAY];
1594	} IEMTBCACHE;
1595	/** Pointer to a per-CPU translation block cache. */
1596	typedef IEMTBCACHE *PIEMTBCACHE;
1597
1598	/** Magic value for IEMTBCACHE (Johnny O'Neal). */
1599	#define IEMTBCACHE_MAGIC UINT32_C(0x19561010)
1600
1601	/** The collision count mask for IEMTBCACHE::apHash entries. */
1602	#define IEMTBCACHE_PTR_COUNT_MASK ((uintptr_t)0x3f)
1603	/** The max collision count for IEMTBCACHE::apHash entries before pruning. */
1604	#define IEMTBCACHE_PTR_MAX_COUNT ((uintptr_t)0x30)
1605	/** Combine a TB pointer and a collision list length into a value for an
1606	* IEMTBCACHE::apHash entry. */
1607	#define IEMTBCACHE_PTR_MAKE(a_pTb, a_cCount) (PIEMTB)((uintptr_t)(a_pTb) \| (a_cCount))
1608	/** Combine a TB pointer and a collision list length into a value for an
1609	* IEMTBCACHE::apHash entry. */
1610	#define IEMTBCACHE_PTR_GET_TB(a_pHashEntry) (PIEMTB)((uintptr_t)(a_pHashEntry) & ~IEMTBCACHE_PTR_COUNT_MASK)
1611	/** Combine a TB pointer and a collision list length into a value for an
1612	* IEMTBCACHE::apHash entry. */
1613	#define IEMTBCACHE_PTR_GET_COUNT(a_pHashEntry) ((uintptr_t)(a_pHashEntry) & IEMTBCACHE_PTR_COUNT_MASK)
1614
1615	/**
1616	* Calculates the hash table slot for a TB from physical PC address and TB flags.
1617	*/
1618	#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
1619	IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, (a_fTbFlags) & IEMTB_F_KEY_MASK, a_GCPhysPc)
1620
1621	/**
1622	* Calculates the hash table slot for a TB from physical PC address and TB
1623	* flags, ASSUMING the caller has applied IEMTB_F_KEY_MASK to @a a_fTbFlags.
1624	*/
1625	#define IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, a_fTbFlags, a_GCPhysPc) \
1626	(((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
1627
1628
1629	/** @name IEMBRANCHED_F_XXX - Branched indicator (IEMCPU::fTbBranched).
1630	*
1631	* These flags parallels the main IEM_CIMPL_F_BRANCH_XXX flags.
1632	*
1633	* @{ */
1634	/** Value if no branching happened recently. */
1635	#define IEMBRANCHED_F_NO UINT8_C(0x00)
1636	/** Flag set if direct branch, clear if absolute or indirect. */
1637	#define IEMBRANCHED_F_DIRECT UINT8_C(0x01)
1638	/** Flag set if indirect branch, clear if direct or relative. */
1639	#define IEMBRANCHED_F_INDIRECT UINT8_C(0x02)
1640	/** Flag set if relative branch, clear if absolute or indirect. */
1641	#define IEMBRANCHED_F_RELATIVE UINT8_C(0x04)
1642	/** Flag set if conditional branch, clear if unconditional. */
1643	#define IEMBRANCHED_F_CONDITIONAL UINT8_C(0x08)
1644	/** Flag set if it's a far branch.
1645	* @note x86 specific */
1646	#define IEMBRANCHED_F_FAR UINT8_C(0x10)
1647	/** Flag set if the stack pointer is modified. */
1648	#define IEMBRANCHED_F_STACK UINT8_C(0x20)
1649	/** Flag set if the stack pointer and (maybe) the stack segment are modified.
1650	* @note x86 specific */
1651	#define IEMBRANCHED_F_STACK_FAR UINT8_C(0x40)
1652	/** Flag set (by IEM_MC_REL_JMP_XXX) if it's a zero bytes relative jump. */
1653	#define IEMBRANCHED_F_ZERO UINT8_C(0x80)
1654	/** @} */
1655
1656
1657	/**
1658	* The per-CPU IEM state.
1659	*/
1660	typedef struct IEMCPU
1661	{
1662	/** Info status code that needs to be propagated to the IEM caller.
1663	* This cannot be passed internally, as it would complicate all success
1664	* checks within the interpreter making the code larger and almost impossible
1665	* to get right. Instead, we'll store status codes to pass on here. Each
1666	* source of these codes will perform appropriate sanity checks. */
1667	int32_t rcPassUp; /* 0x00 */
1668	/** Execution flag, IEM_F_XXX. */
1669	uint32_t fExec; /* 0x04 */
1670
1671	/** @name Decoder state.
1672	* @{ */
1673	#ifdef IEM_WITH_CODE_TLB
1674	/** The offset of the next instruction byte. */
1675	uint32_t offInstrNextByte; /* 0x08 */
1676	/** The number of bytes available at pbInstrBuf for the current instruction.
1677	* This takes the max opcode length into account so that doesn't need to be
1678	* checked separately. */
1679	uint32_t cbInstrBuf; /* 0x0c */
1680	/** Pointer to the page containing RIP, user specified buffer or abOpcode.
1681	* This can be NULL if the page isn't mappable for some reason, in which
1682	* case we'll do fallback stuff.
1683	*
1684	* If we're executing an instruction from a user specified buffer,
1685	* IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
1686	* aligned pointer but pointer to the user data.
1687	*
1688	* For instructions crossing pages, this will start on the first page and be
1689	* advanced to the next page by the time we've decoded the instruction. This
1690	* therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
1691	*/
1692	uint8_t const pbInstrBuf; / 0x10 */
1693	# if ARCH_BITS == 32
1694	uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
1695	# endif
1696	/** The program counter corresponding to pbInstrBuf.
1697	* This is set to a non-canonical address when we need to invalidate it. */
1698	uint64_t uInstrBufPc; /* 0x18 */
1699	/** The guest physical address corresponding to pbInstrBuf. */
1700	RTGCPHYS GCPhysInstrBuf; /* 0x20 */
1701	/** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1702	* This takes the CS segment limit into account.
1703	* @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1704	uint16_t cbInstrBufTotal; /* 0x28 */
1705	/** Offset into pbInstrBuf of the first byte of the current instruction.
1706	* Can be negative to efficiently handle cross page instructions. */
1707	int16_t offCurInstrStart; /* 0x2a */
1708
1709	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1710	/** The prefix mask (IEM_OP_PRF_XXX). */
1711	uint32_t fPrefixes; /* 0x2c */
1712	/** The extra REX ModR/M register field bit (REX.R << 3). */
1713	uint8_t uRexReg; /* 0x30 */
1714	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
1715	* (REX.B << 3). */
1716	uint8_t uRexB; /* 0x31 */
1717	/** The extra REX SIB index field bit (REX.X << 3). */
1718	uint8_t uRexIndex; /* 0x32 */
1719
1720	/** The effective segment register (X86_SREG_XXX). */
1721	uint8_t iEffSeg; /* 0x33 */
1722
1723	/** The offset of the ModR/M byte relative to the start of the instruction. */
1724	uint8_t offModRm; /* 0x34 */
1725
1726	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1727	/** The current offset into abOpcode. */
1728	uint8_t offOpcode; /* 0x35 */
1729	# else
1730	uint8_t bUnused; /* 0x35 */
1731	# endif
1732	# else /* IEM_WITH_OPAQUE_DECODER_STATE */
1733	uint8_t abOpaqueDecoderPart1[0x36 - 0x2c];
1734	# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1735
1736	#else /* !IEM_WITH_CODE_TLB */
1737	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1738	/** The size of what has currently been fetched into abOpcode. */
1739	uint8_t cbOpcode; /* 0x08 */
1740	/** The current offset into abOpcode. */
1741	uint8_t offOpcode; /* 0x09 */
1742	/** The offset of the ModR/M byte relative to the start of the instruction. */
1743	uint8_t offModRm; /* 0x0a */
1744
1745	/** The effective segment register (X86_SREG_XXX). */
1746	uint8_t iEffSeg; /* 0x0b */
1747
1748	/** The prefix mask (IEM_OP_PRF_XXX). */
1749	uint32_t fPrefixes; /* 0x0c */
1750	/** The extra REX ModR/M register field bit (REX.R << 3). */
1751	uint8_t uRexReg; /* 0x10 */
1752	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
1753	* (REX.B << 3). */
1754	uint8_t uRexB; /* 0x11 */
1755	/** The extra REX SIB index field bit (REX.X << 3). */
1756	uint8_t uRexIndex; /* 0x12 */
1757
1758	# else /* IEM_WITH_OPAQUE_DECODER_STATE */
1759	uint8_t abOpaqueDecoderPart1[0x13 - 0x08];
1760	# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1761	#endif /* !IEM_WITH_CODE_TLB */
1762
1763	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1764	/** The effective operand mode. */
1765	IEMMODE enmEffOpSize; /* 0x36, 0x13 */
1766	/** The default addressing mode. */
1767	IEMMODE enmDefAddrMode; /* 0x37, 0x14 */
1768	/** The effective addressing mode. */
1769	IEMMODE enmEffAddrMode; /* 0x38, 0x15 */
1770	/** The default operand mode. */
1771	IEMMODE enmDefOpSize; /* 0x39, 0x16 */
1772
1773	/** Prefix index (VEX.pp) for two byte and three byte tables. */
1774	uint8_t idxPrefix; /* 0x3a, 0x17 */
1775	/** 3rd VEX/EVEX/XOP register.
1776	* Please use IEM_GET_EFFECTIVE_VVVV to access. */
1777	uint8_t uVex3rdReg; /* 0x3b, 0x18 */
1778	/** The VEX/EVEX/XOP length field. */
1779	uint8_t uVexLength; /* 0x3c, 0x19 */
1780	/** Additional EVEX stuff. */
1781	uint8_t fEvexStuff; /* 0x3d, 0x1a */
1782
1783	# ifndef IEM_WITH_CODE_TLB
1784	/** Explicit alignment padding. */
1785	uint8_t abAlignment2a[1]; /* 0x1b */
1786	# endif
1787	/** The FPU opcode (FOP). */
1788	uint16_t uFpuOpcode; /* 0x3e, 0x1c */
1789	# ifndef IEM_WITH_CODE_TLB
1790	/** Explicit alignment padding. */
1791	uint8_t abAlignment2b[2]; /* 0x1e */
1792	# endif
1793
1794	/** The opcode bytes. */
1795	uint8_t abOpcode[15]; /* 0x40, 0x20 */
1796	/** Explicit alignment padding. */
1797	# ifdef IEM_WITH_CODE_TLB
1798	//uint8_t abAlignment2c[0x4f - 0x4f]; /* 0x4f */
1799	# else
1800	uint8_t abAlignment2c[0x4f - 0x2f]; /* 0x2f */
1801	# endif
1802
1803	#else /* IEM_WITH_OPAQUE_DECODER_STATE */
1804	# ifdef IEM_WITH_CODE_TLB
1805	uint8_t abOpaqueDecoderPart2[0x4f - 0x36];
1806	# else
1807	uint8_t abOpaqueDecoderPart2[0x4f - 0x13];
1808	# endif
1809	#endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1810	/** @} */
1811
1812
1813	/** The number of active guest memory mappings. */
1814	uint8_t cActiveMappings; /* 0x4f, 0x4f */
1815
1816	/** Records for tracking guest memory mappings. */
1817	struct
1818	{
1819	/** The address of the mapped bytes. */
1820	R3R0PTRTYPE(void *) pv;
1821	/** The access flags (IEM_ACCESS_XXX).
1822	* IEM_ACCESS_INVALID if the entry is unused. */
1823	uint32_t fAccess;
1824	#if HC_ARCH_BITS == 64
1825	uint32_t u32Alignment4; /*< Alignment padding. /
1826	#endif
1827	} aMemMappings[3]; /* 0x50 LB 0x30 */
1828
1829	/** Locking records for the mapped memory. */
1830	union
1831	{
1832	PGMPAGEMAPLOCK Lock;
1833	uint64_t au64Padding[2];
1834	} aMemMappingLocks[3]; /* 0x80 LB 0x30 */
1835
1836	/** Bounce buffer info.
1837	* This runs in parallel to aMemMappings. */
1838	struct
1839	{
1840	/** The physical address of the first byte. */
1841	RTGCPHYS GCPhysFirst;
1842	/** The physical address of the second page. */
1843	RTGCPHYS GCPhysSecond;
1844	/** The number of bytes in the first page. */
1845	uint16_t cbFirst;
1846	/** The number of bytes in the second page. */
1847	uint16_t cbSecond;
1848	/** Whether it's unassigned memory. */
1849	bool fUnassigned;
1850	/** Explicit alignment padding. */
1851	bool afAlignment5[3];
1852	} aMemBbMappings[3]; /* 0xb0 LB 0x48 */
1853
1854	/** The flags of the current exception / interrupt. */
1855	uint32_t fCurXcpt; /* 0xf8 */
1856	/** The current exception / interrupt. */
1857	uint8_t uCurXcpt; /* 0xfc */
1858	/** Exception / interrupt recursion depth. */
1859	int8_t cXcptRecursions; /* 0xfb */
1860
1861	/** The next unused mapping index.
1862	* @todo try find room for this up with cActiveMappings. */
1863	uint8_t iNextMapping; /* 0xfd */
1864	uint8_t abAlignment7[1];
1865
1866	/** Bounce buffer storage.
1867	* This runs in parallel to aMemMappings and aMemBbMappings. */
1868	struct
1869	{
1870	uint8_t ab[512];
1871	} aBounceBuffers[3]; /* 0x100 LB 0x600 */
1872
1873
1874	/** Pointer set jump buffer - ring-3 context. */
1875	R3PTRTYPE(jmp_buf *) pJmpBufR3;
1876	/** Pointer set jump buffer - ring-0 context. */
1877	R0PTRTYPE(jmp_buf *) pJmpBufR0;
1878
1879	/** @todo Should move this near @a fCurXcpt later. */
1880	/** The CR2 for the current exception / interrupt. */
1881	uint64_t uCurXcptCr2;
1882	/** The error code for the current exception / interrupt. */
1883	uint32_t uCurXcptErr;
1884
1885	/** @name Statistics
1886	* @{ */
1887	/** The number of instructions we've executed. */
1888	uint32_t cInstructions;
1889	/** The number of potential exits. */
1890	uint32_t cPotentialExits;
1891	/** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
1892	uint32_t cRetInstrNotImplemented;
1893	/** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
1894	uint32_t cRetAspectNotImplemented;
1895	/** Counts informational statuses returned (other than VINF_SUCCESS). */
1896	uint32_t cRetInfStatuses;
1897	/** Counts other error statuses returned. */
1898	uint32_t cRetErrStatuses;
1899	/** Number of times rcPassUp has been used. */
1900	uint32_t cRetPassUpStatus;
1901	/** Number of times RZ left with instruction commit pending for ring-3. */
1902	uint32_t cPendingCommit;
1903	/** Number of misaligned (host sense) atomic instruction accesses. */
1904	uint32_t cMisalignedAtomics;
1905	/** Number of long jumps. */
1906	uint32_t cLongJumps;
1907	/** @} */
1908
1909	/** @name Target CPU information.
1910	* @{ */
1911	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1912	/** The target CPU. */
1913	uint8_t uTargetCpu;
1914	#else
1915	uint8_t bTargetCpuPadding;
1916	#endif
1917	/** For selecting assembly works matching the target CPU EFLAGS behaviour, see
1918	* IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
1919	* native host support and the 2nd for when there is.
1920	*
1921	* The two values are typically indexed by a g_CpumHostFeatures bit.
1922	*
1923	* This is for instance used for the BSF & BSR instructions where AMD and
1924	* Intel CPUs produce different EFLAGS. */
1925	uint8_t aidxTargetCpuEflFlavour[2];
1926
1927	/** The CPU vendor. */
1928	CPUMCPUVENDOR enmCpuVendor;
1929	/** @} */
1930
1931	/** Counts RDMSR \#GP(0) LogRel(). */
1932	uint8_t cLogRelRdMsr;
1933	/** Counts WRMSR \#GP(0) LogRel(). */
1934	uint8_t cLogRelWrMsr;
1935	/** Alignment padding. */
1936	uint8_t abAlignment9[50];
1937
1938
1939	/** @name Recompiled Exection
1940	* @{ */
1941	/** Pointer to the current translation block.
1942	* This can either be one being executed or one being compiled. */
1943	R3PTRTYPE(PIEMTB) pCurTbR3;
1944	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
1945	/** Frame pointer for the last native TB to execute. */
1946	R3PTRTYPE(void *) pvTbFramePointerR3;
1947	#else
1948	R3PTRTYPE(void *) pvUnusedR3;
1949	#endif
1950	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1951	/** The saved host floating point control register (MXCSR on x86, FPCR on arm64)
1952	* needing restore when the TB finished, IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED indicates the TB
1953	* didn't modify it so we don't need to restore it. */
1954	# ifdef RT_ARCH_AMD64
1955	uint32_t uRegFpCtrl;
1956	/** Temporary copy of MXCSR for stmxcsr/ldmxcsr (so we don't have to fiddle with stack pointers). */
1957	uint32_t uRegMxcsrTmp;
1958	# elif defined(RT_ARCH_ARM64)
1959	uint64_t uRegFpCtrl;
1960	# else
1961	# error "Port me"
1962	# endif
1963	#else
1964	uint64_t u64Unused;
1965	#endif
1966	/** Pointer to the ring-3 TB cache for this EMT. */
1967	R3PTRTYPE(PIEMTBCACHE) pTbCacheR3;
1968	/** Pointer to the ring-3 TB lookup entry.
1969	* This either points to pTbLookupEntryDummyR3 or an actually lookuptable
1970	* entry, thus it can always safely be used w/o NULL checking. */
1971	R3PTRTYPE(PIEMTB *) ppTbLookupEntryR3;
1972	#if 0 /* unused */
1973	/** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
1974	* The TBs are based on physical addresses, so this is needed to correleated
1975	* RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
1976	uint64_t uCurTbStartPc;
1977	#endif
1978
1979	/** Number of threaded TBs executed. */
1980	uint64_t cTbExecThreaded;
1981	/** Number of native TBs executed. */
1982	uint64_t cTbExecNative;
1983
1984	/** The number of IRQ/FF checks till the next timer poll call. */
1985	uint32_t cTbsTillNextTimerPoll;
1986	/** The virtual sync time at the last timer poll call in milliseconds. */
1987	uint32_t msRecompilerPollNow;
1988	/** The virtual sync time at the last timer poll call in nanoseconds. */
1989	uint64_t nsRecompilerPollNow;
1990	/** The previous cTbsTillNextTimerPoll value. */
1991	uint32_t cTbsTillNextTimerPollPrev;
1992
1993	/** The current instruction number in a native TB.
1994	* This is set by code that may trigger an unexpected TB exit (throw/longjmp)
1995	* and will be picked up by the TB execution loop. Only used when
1996	* IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
1997	uint8_t idxTbCurInstr;
1998	/** @} */
1999
2000	/** @name Recompilation
2001	* @{ */
2002	/** Whether we need to check the opcode bytes for the current instruction.
2003	* This is set by a previous instruction if it modified memory or similar. */
2004	bool fTbCheckOpcodes;
2005	/** Indicates whether and how we just branched - IEMBRANCHED_F_XXX. */
2006	uint8_t fTbBranched;
2007	/** Set when GCPhysInstrBuf is updated because of a page crossing. */
2008	bool fTbCrossedPage;
2009	/** Whether to end the current TB. */
2010	bool fEndTb;
2011	/** Indicates that the current instruction is an STI. This is set by the
2012	* iemCImpl_sti code and subsequently cleared by the recompiler. */
2013	bool fTbCurInstrIsSti;
2014	/** Spaced reserved for recompiler data / alignment. */
2015	bool afRecompilerStuff1[1];
2016	/** Number of instructions before we need emit an IRQ check call again.
2017	* This helps making sure we don't execute too long w/o checking for
2018	* interrupts and immediately following instructions that may enable
2019	* interrupts (e.g. POPF, IRET, STI). With STI an additional hack is
2020	* required to make sure we check following the next instruction as well, see
2021	* fTbCurInstrIsSti. */
2022	uint8_t cInstrTillIrqCheck;
2023	/** The index of the last CheckIrq call during threaded recompilation. */
2024	uint16_t idxLastCheckIrqCallNo;
2025	/** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
2026	uint16_t cbOpcodesAllocated;
2027	/** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
2028	uint32_t uTbNativeRecompileAtUsedCount;
2029	/** The IEM_CIMPL_F_XXX mask for the current instruction. */
2030	uint32_t fTbCurInstr;
2031	/** The IEM_CIMPL_F_XXX mask for the previous instruction. */
2032	uint32_t fTbPrevInstr;
2033	/** Strict: Tracking skipped EFLAGS calculations. Any bits set here are
2034	* currently not up to date in EFLAGS. */
2035	uint32_t fSkippingEFlags;
2036	#if 0 /* unused */
2037	/** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set. */
2038	RTGCPHYS GCPhysInstrBufPrev;
2039	#endif
2040
2041	/** Fixed TB used for threaded recompilation.
2042	* This is allocated once with maxed-out sizes and re-used afterwards. */
2043	R3PTRTYPE(PIEMTB) pThrdCompileTbR3;
2044	/** Pointer to the ring-3 TB allocator for this EMT. */
2045	R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
2046	/** Pointer to the ring-3 executable memory allocator for this EMT. */
2047	R3PTRTYPE(struct IEMEXECMEMALLOCATOR *) pExecMemAllocatorR3;
2048	/** Pointer to the native recompiler state for ring-3. */
2049	R3PTRTYPE(struct IEMRECOMPILERSTATE *) pNativeRecompilerStateR3;
2050	/** Dummy entry for ppTbLookupEntryR3. */
2051	R3PTRTYPE(PIEMTB) pTbLookupEntryDummyR3;
2052	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2053	/** The debug code advances this register as if it was CPUMCTX::rip and we
2054	* didn't do delayed PC updating. When CPUMCTX::rip is finally updated,
2055	* the result is compared with this value. */
2056	uint64_t uPcUpdatingDebug;
2057	#elif defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
2058	/** The SSM handle used for saving threaded TBs for recompiler profiling. */
2059	R3PTRTYPE(PSSMHANDLE) pSsmThreadedTbsForProfiling;
2060	#else
2061	uint64_t u64Placeholder;
2062	#endif
2063	/**
2064	* Whether we should use the host instruction invalidation APIs of the
2065	* host OS or our own version of it (macOS). */
2066	uint8_t fHostICacheInvalidation;
2067	#define IEMNATIVE_ICACHE_F_USE_HOST_API UINT8_C(0x01) /*< Use the host API (macOS) instead of our code. /
2068	#define IEMNATIVE_ICACHE_F_END_WITH_ISH UINT8_C(0x02) /*< Whether to end with a ISH barrier (arm). /
2069	bool afRecompilerStuff2[7];
2070	/** @} */
2071
2072	/** Dummy TLB entry used for accesses to pages with databreakpoints. */
2073	IEMTLBENTRY DataBreakpointTlbe;
2074
2075	/** Threaded TB statistics: Times TB execution was broken off before reaching the end. */
2076	STAMCOUNTER StatTbThreadedExecBreaks;
2077	/** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
2078	STAMCOUNTER StatCheckIrqBreaks;
2079	/** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
2080	STAMCOUNTER StatCheckTimersBreaks;
2081	/** Statistics: Times BltIn_CheckMode breaks out of the TB. */
2082	STAMCOUNTER StatCheckModeBreaks;
2083	/** Threaded TB statistics: Times execution break on call with lookup entries. */
2084	STAMCOUNTER StatTbThreadedExecBreaksWithLookup;
2085	/** Threaded TB statistics: Times execution break on call without lookup entries. */
2086	STAMCOUNTER StatTbThreadedExecBreaksWithoutLookup;
2087	/** Statistics: Times a post jump target check missed and had to find new TB. */
2088	STAMCOUNTER StatCheckBranchMisses;
2089	/** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
2090	STAMCOUNTER StatCheckNeedCsLimChecking;
2091	/** Statistics: Times a loop was detected within a TB. */
2092	STAMCOUNTER StatTbLoopInTbDetected;
2093	/** Statistics: Times a loop back to the start of the TB was detected. */
2094	STAMCOUNTER StatTbLoopFullTbDetected;
2095	/** Statistics: Times a loop back to the start of the TB was detected, var 2. */
2096	STAMCOUNTER StatTbLoopFullTbDetected2;
2097	/** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
2098	STAMCOUNTER StatNativeExecMemInstrBufAllocFailed;
2099	/** Native TB statistics: Number of fully recompiled TBs. */
2100	STAMCOUNTER StatNativeFullyRecompiledTbs;
2101	/** TB statistics: Number of instructions per TB. */
2102	STAMPROFILE StatTbInstr;
2103	/** TB statistics: Number of TB lookup table entries per TB. */
2104	STAMPROFILE StatTbLookupEntries;
2105	/** Threaded TB statistics: Number of calls per TB. */
2106	STAMPROFILE StatTbThreadedCalls;
2107	/** Native TB statistics: Native code size per TB. */
2108	STAMPROFILE StatTbNativeCode;
2109	/** Native TB statistics: Profiling native recompilation. */
2110	STAMPROFILE StatNativeRecompilation;
2111	/** Native TB statistics: Number of calls per TB that were recompiled properly. */
2112	STAMPROFILE StatNativeCallsRecompiled;
2113	/** Native TB statistics: Number of threaded calls per TB that weren't recompiled. */
2114	STAMPROFILE StatNativeCallsThreaded;
2115	/** Native recompiled execution: TLB hits for data fetches. */
2116	STAMCOUNTER StatNativeTlbHitsForFetch;
2117	/** Native recompiled execution: TLB hits for data stores. */
2118	STAMCOUNTER StatNativeTlbHitsForStore;
2119	/** Native recompiled execution: TLB hits for stack accesses. */
2120	STAMCOUNTER StatNativeTlbHitsForStack;
2121	/** Native recompiled execution: TLB hits for mapped accesses. */
2122	STAMCOUNTER StatNativeTlbHitsForMapped;
2123	/** Native recompiled execution: Code TLB misses for new page. */
2124	STAMCOUNTER StatNativeCodeTlbMissesNewPage;
2125	/** Native recompiled execution: Code TLB hits for new page. */
2126	STAMCOUNTER StatNativeCodeTlbHitsForNewPage;
2127	/** Native recompiled execution: Code TLB misses for new page with offset. */
2128	STAMCOUNTER StatNativeCodeTlbMissesNewPageWithOffset;
2129	/** Native recompiled execution: Code TLB hits for new page with offset. */
2130	STAMCOUNTER StatNativeCodeTlbHitsForNewPageWithOffset;
2131
2132	/** Native recompiler: Number of calls to iemNativeRegAllocFindFree. */
2133	STAMCOUNTER StatNativeRegFindFree;
2134	/** Native recompiler: Number of times iemNativeRegAllocFindFree needed
2135	* to free a variable. */
2136	STAMCOUNTER StatNativeRegFindFreeVar;
2137	/** Native recompiler: Number of times iemNativeRegAllocFindFree did
2138	* not need to free any variables. */
2139	STAMCOUNTER StatNativeRegFindFreeNoVar;
2140	/** Native recompiler: Liveness info freed shadowed guest registers in
2141	* iemNativeRegAllocFindFree. */
2142	STAMCOUNTER StatNativeRegFindFreeLivenessUnshadowed;
2143	/** Native recompiler: Liveness info helped with the allocation in
2144	* iemNativeRegAllocFindFree. */
2145	STAMCOUNTER StatNativeRegFindFreeLivenessHelped;
2146
2147	/** Native recompiler: Number of times status flags calc has been skipped. */
2148	STAMCOUNTER StatNativeEflSkippedArithmetic;
2149	/** Native recompiler: Number of times status flags calc has been postponed. */
2150	STAMCOUNTER StatNativeEflPostponedArithmetic;
2151	/** Native recompiler: Total number instructions in this category. */
2152	STAMCOUNTER StatNativeEflTotalArithmetic;
2153
2154	/** Native recompiler: Number of times status flags calc has been skipped. */
2155	STAMCOUNTER StatNativeEflSkippedLogical;
2156	/** Native recompiler: Number of times status flags calc has been postponed. */
2157	STAMCOUNTER StatNativeEflPostponedLogical;
2158	/** Native recompiler: Total number instructions in this category. */
2159	STAMCOUNTER StatNativeEflTotalLogical;
2160
2161	/** Native recompiler: Number of times status flags calc has been skipped. */
2162	STAMCOUNTER StatNativeEflSkippedShift;
2163	/** Native recompiler: Number of times status flags calc has been postponed. */
2164	STAMCOUNTER StatNativeEflPostponedShift;
2165	/** Native recompiler: Total number instructions in this category. */
2166	STAMCOUNTER StatNativeEflTotalShift;
2167
2168	/** Native recompiler: Number of emits per postponement. */
2169	STAMPROFILE StatNativeEflPostponedEmits;
2170
2171	/** Native recompiler: Number of opportunities to skip EFLAGS.CF updating. */
2172	STAMCOUNTER StatNativeLivenessEflCfSkippable;
2173	/** Native recompiler: Number of opportunities to skip EFLAGS.PF updating. */
2174	STAMCOUNTER StatNativeLivenessEflPfSkippable;
2175	/** Native recompiler: Number of opportunities to skip EFLAGS.AF updating. */
2176	STAMCOUNTER StatNativeLivenessEflAfSkippable;
2177	/** Native recompiler: Number of opportunities to skip EFLAGS.ZF updating. */
2178	STAMCOUNTER StatNativeLivenessEflZfSkippable;
2179	/** Native recompiler: Number of opportunities to skip EFLAGS.SF updating. */
2180	STAMCOUNTER StatNativeLivenessEflSfSkippable;
2181	/** Native recompiler: Number of opportunities to skip EFLAGS.OF updating. */
2182	STAMCOUNTER StatNativeLivenessEflOfSkippable;
2183	/** Native recompiler: Number of required EFLAGS.CF updates. */
2184	STAMCOUNTER StatNativeLivenessEflCfRequired;
2185	/** Native recompiler: Number of required EFLAGS.PF updates. */
2186	STAMCOUNTER StatNativeLivenessEflPfRequired;
2187	/** Native recompiler: Number of required EFLAGS.AF updates. */
2188	STAMCOUNTER StatNativeLivenessEflAfRequired;
2189	/** Native recompiler: Number of required EFLAGS.ZF updates. */
2190	STAMCOUNTER StatNativeLivenessEflZfRequired;
2191	/** Native recompiler: Number of required EFLAGS.SF updates. */
2192	STAMCOUNTER StatNativeLivenessEflSfRequired;
2193	/** Native recompiler: Number of required EFLAGS.OF updates. */
2194	STAMCOUNTER StatNativeLivenessEflOfRequired;
2195	/** Native recompiler: Number of potentially delayable EFLAGS.CF updates. */
2196	STAMCOUNTER StatNativeLivenessEflCfDelayable;
2197	/** Native recompiler: Number of potentially delayable EFLAGS.PF updates. */
2198	STAMCOUNTER StatNativeLivenessEflPfDelayable;
2199	/** Native recompiler: Number of potentially delayable EFLAGS.AF updates. */
2200	STAMCOUNTER StatNativeLivenessEflAfDelayable;
2201	/** Native recompiler: Number of potentially delayable EFLAGS.ZF updates. */
2202	STAMCOUNTER StatNativeLivenessEflZfDelayable;
2203	/** Native recompiler: Number of potentially delayable EFLAGS.SF updates. */
2204	STAMCOUNTER StatNativeLivenessEflSfDelayable;
2205	/** Native recompiler: Number of potentially delayable EFLAGS.OF updates. */
2206	STAMCOUNTER StatNativeLivenessEflOfDelayable;
2207
2208	/** Native recompiler: Number of potential PC updates in total. */
2209	STAMCOUNTER StatNativePcUpdateTotal;
2210	/** Native recompiler: Number of PC updates which could be delayed. */
2211	STAMCOUNTER StatNativePcUpdateDelayed;
2212
2213	/** Native recompiler: Number of time we had complicated dirty shadow
2214	* register situations with the other branch in IEM_MC_ENDIF. */
2215	STAMCOUNTER StatNativeEndIfOtherBranchDirty;
2216
2217	/** Native recompiler: Number of calls to iemNativeSimdRegAllocFindFree. */
2218	STAMCOUNTER StatNativeSimdRegFindFree;
2219	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree needed
2220	* to free a variable. */
2221	STAMCOUNTER StatNativeSimdRegFindFreeVar;
2222	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree did
2223	* not need to free any variables. */
2224	STAMCOUNTER StatNativeSimdRegFindFreeNoVar;
2225	/** Native recompiler: Liveness info freed shadowed guest registers in
2226	* iemNativeSimdRegAllocFindFree. */
2227	STAMCOUNTER StatNativeSimdRegFindFreeLivenessUnshadowed;
2228	/** Native recompiler: Liveness info helped with the allocation in
2229	* iemNativeSimdRegAllocFindFree. */
2230	STAMCOUNTER StatNativeSimdRegFindFreeLivenessHelped;
2231
2232	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks. */
2233	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckPotential;
2234	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks. */
2235	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckPotential;
2236	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks. */
2237	STAMCOUNTER StatNativeMaybeSseXcptCheckPotential;
2238	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks. */
2239	STAMCOUNTER StatNativeMaybeAvxXcptCheckPotential;
2240
2241	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks omitted. */
2242	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckOmitted;
2243	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks omitted. */
2244	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckOmitted;
2245	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks omitted. */
2246	STAMCOUNTER StatNativeMaybeSseXcptCheckOmitted;
2247	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks omitted. */
2248	STAMCOUNTER StatNativeMaybeAvxXcptCheckOmitted;
2249
2250	/** Native recompiler: The TB finished executing completely without jumping to a an exit label.
2251	* Not availabe in release builds. */
2252	STAMCOUNTER StatNativeTbFinished;
2253	/** Native recompiler: The TB finished executing jumping to the ReturnBreak label. */
2254	STAMCOUNTER StatNativeTbExitReturnBreak;
2255	/** Native recompiler: The TB finished executing jumping to the ReturnBreakFF label. */
2256	STAMCOUNTER StatNativeTbExitReturnBreakFF;
2257	/** Native recompiler: The TB finished executing jumping to the ReturnWithFlags label. */
2258	STAMCOUNTER StatNativeTbExitReturnWithFlags;
2259	/** Native recompiler: The TB finished executing with other non-zero status. */
2260	STAMCOUNTER StatNativeTbExitReturnOtherStatus;
2261	/** Native recompiler: The TB finished executing via throw / long jump. */
2262	STAMCOUNTER StatNativeTbExitLongJump;
2263	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2264	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2265	STAMCOUNTER StatNativeTbExitDirectLinking1NoIrq;
2266	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2267	* label, but directly jumped to the next TB, scenario \#1 with IRQ checks. */
2268	STAMCOUNTER StatNativeTbExitDirectLinking1Irq;
2269	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2270	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2271	STAMCOUNTER StatNativeTbExitDirectLinking2NoIrq;
2272	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2273	* label, but directly jumped to the next TB, scenario \#2 with IRQ checks. */
2274	STAMCOUNTER StatNativeTbExitDirectLinking2Irq;
2275
2276	/** Native recompiler: The TB finished executing jumping to the RaiseDe label. */
2277	STAMCOUNTER StatNativeTbExitRaiseDe;
2278	/** Native recompiler: The TB finished executing jumping to the RaiseUd label. */
2279	STAMCOUNTER StatNativeTbExitRaiseUd;
2280	/** Native recompiler: The TB finished executing jumping to the RaiseSseRelated label. */
2281	STAMCOUNTER StatNativeTbExitRaiseSseRelated;
2282	/** Native recompiler: The TB finished executing jumping to the RaiseAvxRelated label. */
2283	STAMCOUNTER StatNativeTbExitRaiseAvxRelated;
2284	/** Native recompiler: The TB finished executing jumping to the RaiseSseAvxFpRelated label. */
2285	STAMCOUNTER StatNativeTbExitRaiseSseAvxFpRelated;
2286	/** Native recompiler: The TB finished executing jumping to the RaiseNm label. */
2287	STAMCOUNTER StatNativeTbExitRaiseNm;
2288	/** Native recompiler: The TB finished executing jumping to the RaiseGp0 label. */
2289	STAMCOUNTER StatNativeTbExitRaiseGp0;
2290	/** Native recompiler: The TB finished executing jumping to the RaiseMf label. */
2291	STAMCOUNTER StatNativeTbExitRaiseMf;
2292	/** Native recompiler: The TB finished executing jumping to the RaiseXf label. */
2293	STAMCOUNTER StatNativeTbExitRaiseXf;
2294	/** Native recompiler: The TB finished executing jumping to the ObsoleteTb label. */
2295	STAMCOUNTER StatNativeTbExitObsoleteTb;
2296
2297	/** Native recompiler: Number of full TB loops (jumps from end to start). */
2298	STAMCOUNTER StatNativeTbExitLoopFullTb;
2299
2300	/** Native recompiler: Failure situations with direct linking scenario \#1.
2301	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2302	* @{ */
2303	STAMCOUNTER StatNativeTbExitDirectLinking1NoTb;
2304	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchGCPhysPc;
2305	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchFlags;
2306	STAMCOUNTER StatNativeTbExitDirectLinking1PendingIrq;
2307	/** @} */
2308
2309	/** Native recompiler: Failure situations with direct linking scenario \#2.
2310	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2311	* @{ */
2312	STAMCOUNTER StatNativeTbExitDirectLinking2NoTb;
2313	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchGCPhysPc;
2314	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchFlags;
2315	STAMCOUNTER StatNativeTbExitDirectLinking2PendingIrq;
2316	/** @} */
2317
2318	/** iemMemMap and iemMemMapJmp statistics.
2319	* @{ */
2320	STAMCOUNTER StatMemMapJmp;
2321	STAMCOUNTER StatMemMapNoJmp;
2322	STAMCOUNTER StatMemBounceBufferCrossPage;
2323	STAMCOUNTER StatMemBounceBufferMapPhys;
2324	/** @} */
2325
2326	/** Timer polling statistics (debug only).
2327	* @{ */
2328	STAMPROFILE StatTimerPoll;
2329	STAMPROFILE StatTimerPollPoll;
2330	STAMPROFILE StatTimerPollRun;
2331	STAMCOUNTER StatTimerPollUnchanged;
2332	STAMCOUNTER StatTimerPollTiny;
2333	STAMCOUNTER StatTimerPollDefaultCalc;
2334	STAMCOUNTER StatTimerPollMax;
2335	STAMPROFILE StatTimerPollFactorDivision;
2336	STAMPROFILE StatTimerPollFactorMultiplication;
2337	/** @} */
2338
2339
2340	STAMCOUNTER aStatAdHoc[8];
2341
2342	#ifdef IEM_WITH_TLB_TRACE
2343	/uint64_t au64Padding[0];/
2344	#else
2345	uint64_t au64Padding[2];
2346	#endif
2347
2348	#ifdef IEM_WITH_TLB_TRACE
2349	/** The end (next) trace entry. */
2350	uint32_t idxTlbTraceEntry;
2351	/** Number of trace entries allocated expressed as a power of two. */
2352	uint32_t cTlbTraceEntriesShift;
2353	/** The trace entries. */
2354	PIEMTLBTRACEENTRY paTlbTraceEntries;
2355	#endif
2356
2357	/** Data TLB.
2358	* @remarks Must be 64-byte aligned. */
2359	IEMTLB DataTlb;
2360	/** Instruction TLB.
2361	* @remarks Must be 64-byte aligned. */
2362	IEMTLB CodeTlb;
2363
2364	/** Exception statistics. */
2365	STAMCOUNTER aStatXcpts[32];
2366	/** Interrupt statistics. */
2367	uint32_t aStatInts[256];
2368
2369	#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING) && !defined(IEM_WITHOUT_INSTRUCTION_STATS)
2370	/** Instruction statistics for ring-0/raw-mode. */
2371	IEMINSTRSTATS StatsRZ;
2372	/** Instruction statistics for ring-3. */
2373	IEMINSTRSTATS StatsR3;
2374	# ifdef VBOX_WITH_IEM_RECOMPILER
2375	/** Statistics per threaded function call.
2376	* Updated by both the threaded and native recompilers. */
2377	uint32_t acThreadedFuncStats[0x6000 /24576/];
2378	# endif
2379	#endif
2380	} IEMCPU;
2381	AssertCompileMemberOffset(IEMCPU, cActiveMappings, 0x4f);
2382	AssertCompileMemberAlignment(IEMCPU, aMemMappings, 16);
2383	AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
2384	AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
2385	AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
2386	AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
2387	AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
2388
2389	/** Pointer to the per-CPU IEM state. */
2390	typedef IEMCPU *PIEMCPU;
2391	/** Pointer to the const per-CPU IEM state. */
2392	typedef IEMCPU const *PCIEMCPU;
2393
2394	/** @def IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED
2395	* Value indicating the TB didn't modified the floating point control register.
2396	* @note Neither FPCR nor MXCSR accept this as a valid value (MXCSR is not fully populated,
2397	* FPCR has the upper 32-bit reserved), so this is safe. */
2398	#if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) \|\| defined(DOXYGEN_RUNNING)
2399	# ifdef RT_ARCH_AMD64
2400	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT32_MAX
2401	# elif defined(RT_ARCH_ARM64)
2402	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT64_MAX
2403	# else
2404	# error "Port me"
2405	# endif
2406	#endif
2407
2408	/** @def IEM_GET_CTX
2409	* Gets the guest CPU context for the calling EMT.
2410	* @returns PCPUMCTX
2411	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2412	*/
2413	#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
2414
2415	/** @def IEM_CTX_ASSERT
2416	* Asserts that the @a a_fExtrnMbz is present in the CPU context.
2417	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2418	* @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
2419	*/
2420	#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) \
2421	AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
2422	("fExtrn=%#RX64 & fExtrnMbz=%#RX64 -> %#RX64\n", \
2423	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz), (a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz) ))
2424
2425	/** @def IEM_CTX_IMPORT_RET
2426	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2427	*
2428	* Will call the keep to import the bits as needed.
2429	*
2430	* Returns on import failure.
2431	*
2432	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2433	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2434	*/
2435	#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
2436	do { \
2437	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2438	{ /* likely */ } \
2439	else \
2440	{ \
2441	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2442	AssertRCReturn(rcCtxImport, rcCtxImport); \
2443	} \
2444	} while (0)
2445
2446	/** @def IEM_CTX_IMPORT_NORET
2447	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2448	*
2449	* Will call the keep to import the bits as needed.
2450	*
2451	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2452	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2453	*/
2454	#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
2455	do { \
2456	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2457	{ /* likely */ } \
2458	else \
2459	{ \
2460	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2461	AssertLogRelRC(rcCtxImport); \
2462	} \
2463	} while (0)
2464
2465	/** @def IEM_CTX_IMPORT_JMP
2466	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2467	*
2468	* Will call the keep to import the bits as needed.
2469	*
2470	* Jumps on import failure.
2471	*
2472	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2473	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2474	*/
2475	#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
2476	do { \
2477	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2478	{ /* likely */ } \
2479	else \
2480	{ \
2481	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2482	AssertRCStmt(rcCtxImport, IEM_DO_LONGJMP(pVCpu, rcCtxImport)); \
2483	} \
2484	} while (0)
2485
2486
2487
2488	/** @def IEM_GET_TARGET_CPU
2489	* Gets the current IEMTARGETCPU value.
2490	* @returns IEMTARGETCPU value.
2491	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2492	*/
2493	#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
2494	# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
2495	#else
2496	# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
2497	#endif
2498
2499	/** @def IEM_GET_INSTR_LEN
2500	* Gets the instruction length.
2501	* @note x86 specific */
2502	#ifdef IEM_WITH_CODE_TLB
2503	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
2504	#else
2505	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
2506	#endif
2507
2508	/** @def IEM_TRY_SETJMP
2509	* Wrapper around setjmp / try, hiding all the ugly differences.
2510	*
2511	* @note Use with extreme care as this is a fragile macro.
2512	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2513	* @param a_rcTarget The variable that should receive the status code in case
2514	* of a longjmp/throw.
2515	*/
2516	/** @def IEM_TRY_SETJMP_AGAIN
2517	* For when setjmp / try is used again in the same variable scope as a previous
2518	* IEM_TRY_SETJMP invocation.
2519	*/
2520	/** @def IEM_CATCH_LONGJMP_BEGIN
2521	* Start wrapper for catch / setjmp-else.
2522	*
2523	* This will set up a scope.
2524	*
2525	* @note Use with extreme care as this is a fragile macro.
2526	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2527	* @param a_rcTarget The variable that should receive the status code in case
2528	* of a longjmp/throw.
2529	*/
2530	/** @def IEM_CATCH_LONGJMP_END
2531	* End wrapper for catch / setjmp-else.
2532	*
2533	* This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the
2534	* state.
2535	*
2536	* @note Use with extreme care as this is a fragile macro.
2537	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2538	*/
2539	#if defined(IEM_WITH_SETJMP) \|\| defined(DOXYGEN_RUNNING)
2540	# ifdef IEM_WITH_THROW_CATCH
2541	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2542	a_rcTarget = VINF_SUCCESS; \
2543	try
2544	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2545	IEM_TRY_SETJMP(a_pVCpu, a_rcTarget)
2546	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2547	catch (int rcThrown) \
2548	{ \
2549	a_rcTarget = rcThrown
2550	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2551	} \
2552	((void)0)
2553	# else /* !IEM_WITH_THROW_CATCH */
2554	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2555	jmp_buf JmpBuf; \
2556	jmp_buf * volatile pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2557	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2558	if ((rcStrict = setjmp(JmpBuf)) == 0)
2559	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2560	pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2561	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2562	if ((rcStrict = setjmp(JmpBuf)) == 0)
2563	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2564	else \
2565	{ \
2566	((void)0)
2567	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2568	} \
2569	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf
2570	# endif /* !IEM_WITH_THROW_CATCH */
2571	#endif /* IEM_WITH_SETJMP */
2572
2573
2574	/**
2575	* Shared per-VM IEM data.
2576	*/
2577	typedef struct IEM
2578	{
2579	/** The VMX APIC-access page handler type. */
2580	PGMPHYSHANDLERTYPE hVmxApicAccessPage;
2581	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
2582	/** Set if the CPUID host call functionality is enabled. */
2583	bool fCpuIdHostCall;
2584	#endif
2585	} IEM;
2586
2587
2588
2589	/** @name IEM_ACCESS_XXX - Access details.
2590	* @{ */
2591	#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
2592	#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
2593	#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
2594	#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
2595	#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
2596	#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
2597	#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
2598	#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
2599	#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
2600	#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
2601	/** The writes are partial, so if initialize the bounce buffer with the
2602	* orignal RAM content. */
2603	#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
2604	/** Used in aMemMappings to indicate that the entry is bounce buffered. */
2605	#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
2606	/** Bounce buffer with ring-3 write pending, first page. */
2607	#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
2608	/** Bounce buffer with ring-3 write pending, second page. */
2609	#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
2610	/** Not locked, accessed via the TLB. */
2611	#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
2612	/** Atomic access.
2613	* This enables special alignment checks and the VINF_EM_EMULATE_SPLIT_LOCK
2614	* fallback for misaligned stuff. See @bugref{10547}. */
2615	#define IEM_ACCESS_ATOMIC UINT32_C(0x00002000)
2616	/** Valid bit mask. */
2617	#define IEM_ACCESS_VALID_MASK UINT32_C(0x00003fff)
2618	/** Shift count for the TLB flags (upper word). */
2619	#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
2620
2621	/** Atomic read+write data alias. */
2622	#define IEM_ACCESS_DATA_ATOMIC (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_ATOMIC)
2623	/** Read+write data alias. */
2624	#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2625	/** Write data alias. */
2626	#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2627	/** Read data alias. */
2628	#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA)
2629	/** Instruction fetch alias. */
2630	#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_WHAT_CODE)
2631	/** Stack write alias. */
2632	#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2633	/** Stack read alias. */
2634	#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_STACK)
2635	/** Stack read+write alias. */
2636	#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2637	/** Read system table alias. */
2638	#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_SYS)
2639	/** Read+write system table alias. */
2640	#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_SYS)
2641	/** @} */
2642
2643
2644	/** @def IEM_DECL_MSC_GUARD_IGNORE
2645	* Disables control flow guards checks inside a method and any function pointers
2646	* referenced by it. */
2647	#if defined(_MSC_VER) && !defined(IN_RING0)
2648	# define IEM_DECL_MSC_GUARD_IGNORE __declspec(guard(ignore))
2649	#else
2650	# define IEM_DECL_MSC_GUARD_IGNORE
2651	#endif
2652
2653	/** @def IEM_DECL_MSC_GUARD_NONE
2654	* Disables control flow guards checks inside a method and but continue track
2655	* function pointers references by it. */
2656	#if defined(_MSC_VER) && !defined(IN_RING0)
2657	# define IEM_DECL_MSC_GUARD_NONE __declspec(guard(nocf))
2658	#else
2659	# define IEM_DECL_MSC_GUARD_NONE
2660	#endif
2661
2662
2663	/** @def IEM_DECL_IMPL_TYPE
2664	* For typedef'ing an instruction implementation function.
2665	*
2666	* @param a_RetType The return type.
2667	* @param a_Name The name of the type.
2668	* @param a_ArgList The argument list enclosed in parentheses.
2669	*/
2670
2671	/** @def IEM_DECL_IMPL_DEF
2672	* For defining an instruction implementation function.
2673	*
2674	* @param a_RetType The return type.
2675	* @param a_Name The name of the type.
2676	* @param a_ArgList The argument list enclosed in parentheses.
2677	*/
2678
2679	#if defined(__GNUC__) && defined(RT_ARCH_X86)
2680	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2681	__attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
2682	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2683	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2684	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2685	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2686
2687	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2688	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2689	a_RetType (__fastcall a_Name) a_ArgList
2690	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2691	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2692	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2693	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2694
2695	#elif __cplusplus >= 201700 /* P0012R1 support */
2696	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2697	a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
2698	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2699	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2700	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2701	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2702
2703	#else
2704	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2705	a_RetType (VBOXCALL a_Name) a_ArgList
2706	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2707	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2708	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2709	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2710
2711	#endif
2712
2713
2714	/** @name C instruction implementations for anything slightly complicated.
2715	* @{ */
2716
2717	/**
2718	* For typedef'ing or declaring a C instruction implementation function taking
2719	* no extra arguments.
2720	*
2721	* @param a_Name The name of the type.
2722	*/
2723	# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2724	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2725	/**
2726	* For defining a C instruction implementation function taking no extra
2727	* arguments.
2728	*
2729	* @param a_Name The name of the function
2730	*/
2731	# define IEM_CIMPL_DEF_0(a_Name) \
2732	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2733	/**
2734	* Prototype version of IEM_CIMPL_DEF_0.
2735	*/
2736	# define IEM_CIMPL_PROTO_0(a_Name) \
2737	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2738	/**
2739	* For calling a C instruction implementation function taking no extra
2740	* arguments.
2741	*
2742	* This special call macro adds default arguments to the call and allow us to
2743	* change these later.
2744	*
2745	* @param a_fn The name of the function.
2746	*/
2747	# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2748
2749	/** Type for a C instruction implementation function taking no extra
2750	* arguments. */
2751	typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
2752	/** Function pointer type for a C instruction implementation function taking
2753	* no extra arguments. */
2754	typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
2755
2756	/**
2757	* For typedef'ing or declaring a C instruction implementation function taking
2758	* one extra argument.
2759	*
2760	* @param a_Name The name of the type.
2761	* @param a_Type0 The argument type.
2762	* @param a_Arg0 The argument name.
2763	*/
2764	# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2765	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2766	/**
2767	* For defining a C instruction implementation function taking one extra
2768	* argument.
2769	*
2770	* @param a_Name The name of the function
2771	* @param a_Type0 The argument type.
2772	* @param a_Arg0 The argument name.
2773	*/
2774	# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2775	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2776	/**
2777	* Prototype version of IEM_CIMPL_DEF_1.
2778	*/
2779	# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2780	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2781	/**
2782	* For calling a C instruction implementation function taking one extra
2783	* argument.
2784	*
2785	* This special call macro adds default arguments to the call and allow us to
2786	* change these later.
2787	*
2788	* @param a_fn The name of the function.
2789	* @param a0 The name of the 1st argument.
2790	*/
2791	# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2792
2793	/**
2794	* For typedef'ing or declaring a C instruction implementation function taking
2795	* two extra arguments.
2796	*
2797	* @param a_Name The name of the type.
2798	* @param a_Type0 The type of the 1st argument
2799	* @param a_Arg0 The name of the 1st argument.
2800	* @param a_Type1 The type of the 2nd argument.
2801	* @param a_Arg1 The name of the 2nd argument.
2802	*/
2803	# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2804	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2805	/**
2806	* For defining a C instruction implementation function taking two extra
2807	* arguments.
2808	*
2809	* @param a_Name The name of the function.
2810	* @param a_Type0 The type of the 1st argument
2811	* @param a_Arg0 The name of the 1st argument.
2812	* @param a_Type1 The type of the 2nd argument.
2813	* @param a_Arg1 The name of the 2nd argument.
2814	*/
2815	# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2816	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2817	/**
2818	* Prototype version of IEM_CIMPL_DEF_2.
2819	*/
2820	# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2821	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2822	/**
2823	* For calling a C instruction implementation function taking two extra
2824	* arguments.
2825	*
2826	* This special call macro adds default arguments to the call and allow us to
2827	* change these later.
2828	*
2829	* @param a_fn The name of the function.
2830	* @param a0 The name of the 1st argument.
2831	* @param a1 The name of the 2nd argument.
2832	*/
2833	# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
2834
2835	/**
2836	* For typedef'ing or declaring a C instruction implementation function taking
2837	* three extra arguments.
2838	*
2839	* @param a_Name The name of the type.
2840	* @param a_Type0 The type of the 1st argument
2841	* @param a_Arg0 The name of the 1st argument.
2842	* @param a_Type1 The type of the 2nd argument.
2843	* @param a_Arg1 The name of the 2nd argument.
2844	* @param a_Type2 The type of the 3rd argument.
2845	* @param a_Arg2 The name of the 3rd argument.
2846	*/
2847	# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2848	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2849	/**
2850	* For defining a C instruction implementation function taking three extra
2851	* arguments.
2852	*
2853	* @param a_Name The name of the function.
2854	* @param a_Type0 The type of the 1st argument
2855	* @param a_Arg0 The name of the 1st argument.
2856	* @param a_Type1 The type of the 2nd argument.
2857	* @param a_Arg1 The name of the 2nd argument.
2858	* @param a_Type2 The type of the 3rd argument.
2859	* @param a_Arg2 The name of the 3rd argument.
2860	*/
2861	# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2862	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2863	/**
2864	* Prototype version of IEM_CIMPL_DEF_3.
2865	*/
2866	# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2867	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2868	/**
2869	* For calling a C instruction implementation function taking three extra
2870	* arguments.
2871	*
2872	* This special call macro adds default arguments to the call and allow us to
2873	* change these later.
2874	*
2875	* @param a_fn The name of the function.
2876	* @param a0 The name of the 1st argument.
2877	* @param a1 The name of the 2nd argument.
2878	* @param a2 The name of the 3rd argument.
2879	*/
2880	# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
2881
2882
2883	/**
2884	* For typedef'ing or declaring a C instruction implementation function taking
2885	* four extra arguments.
2886	*
2887	* @param a_Name The name of the type.
2888	* @param a_Type0 The type of the 1st argument
2889	* @param a_Arg0 The name of the 1st argument.
2890	* @param a_Type1 The type of the 2nd argument.
2891	* @param a_Arg1 The name of the 2nd argument.
2892	* @param a_Type2 The type of the 3rd argument.
2893	* @param a_Arg2 The name of the 3rd argument.
2894	* @param a_Type3 The type of the 4th argument.
2895	* @param a_Arg3 The name of the 4th argument.
2896	*/
2897	# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2898	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
2899	/**
2900	* For defining a C instruction implementation function taking four extra
2901	* arguments.
2902	*
2903	* @param a_Name The name of the function.
2904	* @param a_Type0 The type of the 1st argument
2905	* @param a_Arg0 The name of the 1st argument.
2906	* @param a_Type1 The type of the 2nd argument.
2907	* @param a_Arg1 The name of the 2nd argument.
2908	* @param a_Type2 The type of the 3rd argument.
2909	* @param a_Arg2 The name of the 3rd argument.
2910	* @param a_Type3 The type of the 4th argument.
2911	* @param a_Arg3 The name of the 4th argument.
2912	*/
2913	# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2914	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2915	a_Type2 a_Arg2, a_Type3 a_Arg3))
2916	/**
2917	* Prototype version of IEM_CIMPL_DEF_4.
2918	*/
2919	# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2920	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2921	a_Type2 a_Arg2, a_Type3 a_Arg3))
2922	/**
2923	* For calling a C instruction implementation function taking four extra
2924	* arguments.
2925	*
2926	* This special call macro adds default arguments to the call and allow us to
2927	* change these later.
2928	*
2929	* @param a_fn The name of the function.
2930	* @param a0 The name of the 1st argument.
2931	* @param a1 The name of the 2nd argument.
2932	* @param a2 The name of the 3rd argument.
2933	* @param a3 The name of the 4th argument.
2934	*/
2935	# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
2936
2937
2938	/**
2939	* For typedef'ing or declaring a C instruction implementation function taking
2940	* five extra arguments.
2941	*
2942	* @param a_Name The name of the type.
2943	* @param a_Type0 The type of the 1st argument
2944	* @param a_Arg0 The name of the 1st argument.
2945	* @param a_Type1 The type of the 2nd argument.
2946	* @param a_Arg1 The name of the 2nd argument.
2947	* @param a_Type2 The type of the 3rd argument.
2948	* @param a_Arg2 The name of the 3rd argument.
2949	* @param a_Type3 The type of the 4th argument.
2950	* @param a_Arg3 The name of the 4th argument.
2951	* @param a_Type4 The type of the 5th argument.
2952	* @param a_Arg4 The name of the 5th argument.
2953	*/
2954	# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2955	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
2956	a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
2957	a_Type3 a_Arg3, a_Type4 a_Arg4))
2958	/**
2959	* For defining a C instruction implementation function taking five extra
2960	* arguments.
2961	*
2962	* @param a_Name The name of the function.
2963	* @param a_Type0 The type of the 1st argument
2964	* @param a_Arg0 The name of the 1st argument.
2965	* @param a_Type1 The type of the 2nd argument.
2966	* @param a_Arg1 The name of the 2nd argument.
2967	* @param a_Type2 The type of the 3rd argument.
2968	* @param a_Arg2 The name of the 3rd argument.
2969	* @param a_Type3 The type of the 4th argument.
2970	* @param a_Arg3 The name of the 4th argument.
2971	* @param a_Type4 The type of the 5th argument.
2972	* @param a_Arg4 The name of the 5th argument.
2973	*/
2974	# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2975	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2976	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2977	/**
2978	* Prototype version of IEM_CIMPL_DEF_5.
2979	*/
2980	# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2981	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2982	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2983	/**
2984	* For calling a C instruction implementation function taking five extra
2985	* arguments.
2986	*
2987	* This special call macro adds default arguments to the call and allow us to
2988	* change these later.
2989	*
2990	* @param a_fn The name of the function.
2991	* @param a0 The name of the 1st argument.
2992	* @param a1 The name of the 2nd argument.
2993	* @param a2 The name of the 3rd argument.
2994	* @param a3 The name of the 4th argument.
2995	* @param a4 The name of the 5th argument.
2996	*/
2997	# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2998
2999	/** @} */
3000
3001
3002	/** @name Opcode Decoder Function Types.
3003	* @{ */
3004
3005	/** @typedef PFNIEMOP
3006	* Pointer to an opcode decoder function.
3007	*/
3008
3009	/** @def FNIEMOP_DEF
3010	* Define an opcode decoder function.
3011	*
3012	* We're using macors for this so that adding and removing parameters as well as
3013	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
3014	*
3015	* @param a_Name The function name.
3016	*/
3017
3018	/** @typedef PFNIEMOPRM
3019	* Pointer to an opcode decoder function with RM byte.
3020	*/
3021
3022	/** @def FNIEMOPRM_DEF
3023	* Define an opcode decoder function with RM byte.
3024	*
3025	* We're using macors for this so that adding and removing parameters as well as
3026	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
3027	*
3028	* @param a_Name The function name.
3029	*/
3030
3031	#if defined(__GNUC__) && defined(RT_ARCH_X86)
3032	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
3033	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3034	# define FNIEMOP_DEF(a_Name) \
3035	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
3036	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3037	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3038	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3039	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3040
3041	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
3042	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
3043	typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3044	# define FNIEMOP_DEF(a_Name) \
3045	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3046	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3047	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3048	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3049	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3050
3051	#elif defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3052	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3053	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3054	# define FNIEMOP_DEF(a_Name) \
3055	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
3056	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3057	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3058	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3059	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3060
3061	#else
3062	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3063	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3064	# define FNIEMOP_DEF(a_Name) \
3065	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3066	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3067	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3068	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3069	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3070
3071	#endif
3072	#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
3073
3074	/**
3075	* Call an opcode decoder function.
3076	*
3077	* We're using macors for this so that adding and removing parameters can be
3078	* done as we please. See FNIEMOP_DEF.
3079	*/
3080	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
3081
3082	/**
3083	* Call a common opcode decoder function taking one extra argument.
3084	*
3085	* We're using macors for this so that adding and removing parameters can be
3086	* done as we please. See FNIEMOP_DEF_1.
3087	*/
3088	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
3089
3090	/**
3091	* Call a common opcode decoder function taking one extra argument.
3092	*
3093	* We're using macors for this so that adding and removing parameters can be
3094	* done as we please. See FNIEMOP_DEF_1.
3095	*/
3096	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
3097	/** @} */
3098
3099
3100	/** @name Misc Helpers
3101	* @{ */
3102
3103	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
3104	* due to GCC lacking knowledge about the value range of a switch. */
3105	#if RT_CPLUSPLUS_PREREQ(202000)
3106	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: [[unlikely]] AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3107	#else
3108	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3109	#endif
3110
3111	/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
3112	#if RT_CPLUSPLUS_PREREQ(202000)
3113	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: [[unlikely]] AssertFailedReturn(a_RetValue)
3114	#else
3115	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
3116	#endif
3117
3118	/**
3119	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3120	* occation.
3121	*/
3122	#ifdef LOG_ENABLED
3123	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3124	do { \
3125	/Log/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
3126	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3127	} while (0)
3128	#else
3129	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3130	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3131	#endif
3132
3133	/**
3134	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3135	* occation using the supplied logger statement.
3136	*
3137	* @param a_LoggerArgs What to log on failure.
3138	*/
3139	#ifdef LOG_ENABLED
3140	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3141	do { \
3142	LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
3143	/LogFunc(a_LoggerArgs);/ \
3144	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3145	} while (0)
3146	#else
3147	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3148	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3149	#endif
3150
3151
3152	/** @} */
3153
3154	uint32_t iemCalcExecDbgFlagsSlow(PVMCPUCC pVCpu);
3155	VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu);
3156
3157	/** @} */
3158
3159
3160	/** @name Memory access.
3161	* @{ */
3162	VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void *ppvMem, uint8_t pbUnmapInfo,
3163	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) RT_NOEXCEPT;
3164	VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void *ppvMem, uint8_t pbUnmapInfo, size_t cbMem,
3165	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) RT_NOEXCEPT;
3166	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3167	#ifndef IN_RING3
3168	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3169	#endif
3170	void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3171	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3172
3173	#ifdef IEM_WITH_SETJMP
3174	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3175	void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3176	void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3177	void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3178	void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3179	void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3180	#endif
3181
3182	void iemTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3183	/** @} */
3184
3185	/*
3186	* Recompiler related stuff.
3187	*/
3188
3189	DECLHIDDEN(int) iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
3190
3191	DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
3192	uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
3193	void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree);
3194	DECLHIDDEN(void) iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
3195	void iemTbAllocatorProcessDelayedFrees(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator);
3196	void iemTbAllocatorFreeupNativeSpace(PVMCPUCC pVCpu, uint32_t cNeededInstrs);
3197	DECLHIDDEN(PIEMTBALLOCATOR) iemTbAllocatorFreeBulkStart(PVMCPUCC pVCpu);
3198	DECLHIDDEN(void) iemTbAllocatorFreeBulk(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator, PIEMTB pTb);
3199	DECLHIDDEN(const char ) iemTbFlagsToString(uint32_t fFlags, char pszBuf, size_t cbBuf) RT_NOEXCEPT;
3200	DECLHIDDEN(void) iemThreadedDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3201	#if defined(VBOX_WITH_IEM_NATIVE_RECOMPILER) && defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
3202	DECLHIDDEN(void) iemThreadedSaveTbForProfilingCleanup(PVMCPU pVCpu);
3203	#endif
3204
3205
3206	/** @todo FNIEMTHREADEDFUNC and friends may need more work... */
3207	#if defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3208	typedef VBOXSTRICTRC /__attribute__((__nothrow__))/ FNIEMTHREADEDFUNC(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3209	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3210	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3211	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3212	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3213	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3214
3215	#else
3216	typedef VBOXSTRICTRC (FNIEMTHREADEDFUNC)(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3217	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3218	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3219	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3220	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3221	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3222	#endif
3223
3224
3225	/* Native recompiler public bits: */
3226
3227	DECLHIDDEN(PIEMTB) iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb) RT_NOEXCEPT;
3228	DECLHIDDEN(void) iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3229	int iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT;
3230	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, PIEMNATIVEINSTR *ppaExec,
3231	struct IEMNATIVEPERCHUNKCTX const **ppChunkCtx) RT_NOEXCEPT;
3232	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq,
3233	PIEMNATIVEINSTR *ppaExec);
3234	DECLHIDDEN(void) iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3235	void iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3236	DECLASM(DECL_NO_RETURN(void)) iemNativeTbLongJmp(void *pvFramePointer, int rc) RT_NOEXCEPT;
3237	DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb);
3238	DECLHIDDEN(int) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk, struct IEMNATIVEPERCHUNKCTX const **ppCtx);
3239
3240	# ifdef VBOX_VMM_TARGET_X86
3241	# include "VMMAll/target-x86/IEMInternal-x86.h"
3242	# elif defined(VBOX_VMM_TARGET_ARMV8)
3243	//# include "VMMAll/target-armv8/IEMInternal-armv8.h"
3244	# endif
3245
3246	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3247
3248
3249	/** @} */
3250
3251	RT_C_DECLS_END
3252
3253	/* ASM-INC: %include "IEMInternalStruct.mac" */
3254
3255	#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3256

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 108250

Download in other formats: