IEMInternal.h@ 108296

Last change on this file since 108296 was 108278, checked in by vboxsync, 3 months ago
VMM/IEM: Removed the #ifndef IEM_WITH_SETJMP code. We've had IEM_WITH_SETJMP defined unconditionally since 7.0 and the code probably doesn't even compile w/o it, so best remove the unused code. jiraref:VBP-1531
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 147.0 KB

Line
1	/* $Id: IEMInternal.h 108278 2025-02-18 15:46:53Z vboxsync $ */
2	/** @file
3	* IEM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
29	#define VMM_INCLUDED_SRC_include_IEMInternal_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#ifndef RT_IN_ASSEMBLER
35	# include <VBox/vmm/cpum.h>
36	# include <VBox/vmm/iem.h>
37	# include <VBox/vmm/pgm.h>
38	# include <VBox/vmm/stam.h>
39	# include <VBox/param.h>
40
41	# include <iprt/setjmp-without-sigmask.h>
42	# include <iprt/list.h>
43	#endif /* !RT_IN_ASSEMBLER */
44
45
46	RT_C_DECLS_BEGIN
47
48
49	/** @defgroup grp_iem_int Internals
50	* @ingroup grp_iem
51	* @internal
52	* @{
53	*/
54
55	/* Make doxygen happy w/o overcomplicating the #if checks. */
56	#ifdef DOXYGEN_RUNNING
57	# define IEM_WITH_THROW_CATCH
58	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
59	#endif
60
61	/** For expanding symbol in slickedit and other products tagging and
62	* crossreferencing IEM symbols. */
63	#ifndef IEM_STATIC
64	# define IEM_STATIC static
65	#endif
66
67	/** @def IEM_WITH_THROW_CATCH
68	* Enables using C++ throw/catch as an alternative to setjmp/longjmp in user
69	* mode code.
70	*
71	* With GCC 11.3.1 and code TLB on linux, using throw/catch instead of
72	* setjmp/long resulted in bs2-test-1 running 3.00% faster and all but on test
73	* result value improving by more than 1%. (Best out of three.)
74	*
75	* With Visual C++ 2019 and code TLB on windows, using throw/catch instead of
76	* setjmp/long resulted in bs2-test-1 running 3.68% faster and all but some of
77	* the MMIO and CPUID tests ran noticeably faster. Variation is greater than on
78	* Linux, but it should be quite a bit faster for normal code.
79	*/
80	#if defined(__cplusplus) && defined(IN_RING3) && (defined(__GNUC__) \|\| defined(_MSC_VER)) /* ASM-NOINC-START */
81	# define IEM_WITH_THROW_CATCH
82	#endif /ASM-NOINC-END/
83
84	/** @def IEM_WITH_ADAPTIVE_TIMER_POLLING
85	* Enables the adaptive timer polling code.
86	*/
87	#if defined(DOXYGEN_RUNNING) \|\| 1
88	# define IEM_WITH_ADAPTIVE_TIMER_POLLING
89	#endif
90
91	/** @def IEM_WITH_INTRA_TB_JUMPS
92	* Enables loop-jumps within a TB (currently only to the first call).
93	*/
94	#if defined(DOXYGEN_RUNNING) \|\| 1
95	# define IEM_WITH_INTRA_TB_JUMPS
96	#endif
97
98	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
99	* Enables the delayed PC updating optimization (see @bugref{10373}).
100	*/
101	#if defined(DOXYGEN_RUNNING) \|\| 1
102	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING
103	#endif
104	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
105	* Enabled delayed PC updating debugging code.
106	* This is an alternative to the ARM64-only IEMNATIVE_REG_FIXED_PC_DBG. */
107	#if defined(DOXYGEN_RUNNING) \|\| 0
108	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
109	#endif
110
111	/** Enables access to even callee saved registers. */
112	/# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS/
113
114	#if defined(DOXYGEN_RUNNING) \|\| 1
115	/** @def IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
116	* Delay the writeback or dirty registers as long as possible. */
117	# define IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
118	#endif
119
120	/** @def IEM_WITH_TLB_STATISTICS
121	* Enables all TLB statistics. */
122	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
123	# define IEM_WITH_TLB_STATISTICS
124	#endif
125
126	/** @def IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
127	* Enable this to use native emitters for certain SIMD FP operations. */
128	#if 1 \|\| defined(DOXYGEN_RUNNING)
129	# define IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
130	#endif
131
132	/** @def VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
133	* Enable this to create a saved state file with the threaded translation
134	* blocks fed to the native recompiler on VCPU \#0. The resulting file can
135	* then be fed into the native recompiler for code profiling purposes.
136	* This is not a feature that should be normally be enabled! */
137	#if 0 \|\| defined(DOXYGEN_RUNNING)
138	# define VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
139	#endif
140
141	/** @def VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
142	* Enables a quicker alternative to throw/longjmp for IEM_DO_LONGJMP when
143	* executing native translation blocks.
144	*
145	* This exploits the fact that we save all non-volatile registers in the TB
146	* prologue and thus just need to do the same as the TB epilogue to get the
147	* effect of a longjmp/throw. Since MSC marks XMM6 thru XMM15 as
148	* non-volatile (and does something even more crazy for ARM), this probably
149	* won't work reliably on Windows. */
150	#ifdef RT_ARCH_ARM64
151	# ifndef RT_OS_WINDOWS
152	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
153	# endif
154	#endif
155	/* ASM-NOINC-START */
156	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
157	# if !defined(IN_RING3) \
158	\|\| !defined(VBOX_WITH_IEM_RECOMPILER) \
159	\|\| !defined(VBOX_WITH_IEM_NATIVE_RECOMPILER)
160	# undef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
161	# elif defined(RT_OS_WINDOWS)
162	# pragma message("VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP is not safe to use on windows")
163	# endif
164	#endif
165
166
167	/** @def IEM_DO_LONGJMP
168	*
169	* Wrapper around longjmp / throw.
170	*
171	* @param a_pVCpu The CPU handle.
172	* @param a_rc The status code jump back with / throw.
173	*/
174	#ifdef IEM_WITH_THROW_CATCH
175	# ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
176	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) do { \
177	if ((a_pVCpu)->iem.s.pvTbFramePointerR3) \
178	iemNativeTbLongJmp((a_pVCpu)->iem.s.pvTbFramePointerR3, (a_rc)); \
179	throw int(a_rc); \
180	} while (0)
181	# else
182	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) throw int(a_rc)
183	# endif
184	#else
185	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) longjmp(*(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf), (a_rc))
186	#endif
187
188	/** For use with IEM function that may do a longjmp (when enabled).
189	*
190	* Visual C++ has trouble longjmp'ing from/over functions with the noexcept
191	* attribute. So, we indicate that function that may be part of a longjmp may
192	* throw "exceptions" and that the compiler should definitely not generate and
193	* std::terminate calling unwind code.
194	*
195	* Here is one example of this ending in std::terminate:
196	* @code{.txt}
197	00 00000041`cadfda10 00007ffc`5d5a1f9f ucrtbase!abort+0x4e
198	01 00000041`cadfda40 00007ffc`57af229a ucrtbase!terminate+0x1f
199	02 00000041`cadfda70 00007ffb`eec91030 VCRUNTIME140!__std_terminate+0xa [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\ehhelpers.cpp @ 192]
200	03 00000041`cadfdaa0 00007ffb`eec92c6d VCRUNTIME140_1!_CallSettingFrame+0x20 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\handlers.asm @ 50]
201	04 00000041`cadfdad0 00007ffb`eec93ae5 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToState+0x241 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 1085]
202	05 00000041`cadfdc00 00007ffb`eec92258 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToEmptyState+0x2d [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 218]
203	06 00000041`cadfdc30 00007ffb`eec940e9 VCRUNTIME140_1!__InternalCxxFrameHandler<__FrameHandler4>+0x194 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 304]
204	07 00000041`cadfdcd0 00007ffc`5f9f249f VCRUNTIME140_1!__CxxFrameHandler4+0xa9 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 290]
205	08 00000041`cadfdd40 00007ffc`5f980939 ntdll!RtlpExecuteHandlerForUnwind+0xf
206	09 00000041`cadfdd70 00007ffc`5f9a0edd ntdll!RtlUnwindEx+0x339
207	0a 00000041`cadfe490 00007ffc`57aff976 ntdll!RtlUnwind+0xcd
208	0b 00000041`cadfea00 00007ffb`e1b5de01 VCRUNTIME140!__longjmp_internal+0xe6 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\longjmp.asm @ 140]
209	0c (Inline Function) --------`-------- VBoxVMM!iemOpcodeGetNextU8SlowJmp+0x95 [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 1155]
210	0d 00000041`cadfea50 00007ffb`e1b60f6b VBoxVMM!iemOpcodeGetNextU8Jmp+0xc1 [L:\vbox-intern\src\VBox\VMM\include\IEMInline.h @ 402]
211	0e 00000041`cadfea90 00007ffb`e1cc6201 VBoxVMM!IEMExecForExits+0xdb [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 10185]
212	0f 00000041`cadfec70 00007ffb`e1d0df8d VBoxVMM!EMHistoryExec+0x4f1 [L:\vbox-intern\src\VBox\VMM\VMMAll\EMAll.cpp @ 452]
213	10 00000041`cadfed60 00007ffb`e1d0d4c0 VBoxVMM!nemR3WinHandleExitCpuId+0x79d [L:\vbox-intern\src\VBox\VMM\VMMAll\NEMAllNativeTemplate-win.cpp.h @ 1829] @encode
214	@endcode
215	*
216	* @see https://developercommunity.visualstudio.com/t/fragile-behavior-of-longjmp-called-from-noexcept-f/1532859
217	*/
218	#if defined(_MSC_VER) \|\| defined(IEM_WITH_THROW_CATCH)
219	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT_EX(false)
220	#else
221	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT
222	#endif
223	/* ASM-NOINC-END */
224
225
226	//#define IEM_WITH_CODE_TLB // - work in progress
227	//#define IEM_WITH_DATA_TLB // - work in progress
228
229
230	/** @def IEM_USE_UNALIGNED_DATA_ACCESS
231	* Use unaligned accesses instead of elaborate byte assembly. */
232	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86) \|\| defined(DOXYGEN_RUNNING) /ASM-NOINC/
233	# define IEM_USE_UNALIGNED_DATA_ACCESS
234	#endif /ASM-NOINC/
235
236	//#define IEM_LOG_MEMORY_WRITES
237
238
239	/** @def IEM_CFG_TARGET_CPU
240	* The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
241	*
242	* By default we allow this to be configured by the user via the
243	* CPUM/GuestCpuName config string, but this comes at a slight cost during
244	* decoding. So, for applications of this code where there is no need to
245	* be dynamic wrt target CPU, just modify this define.
246	*/
247	#if !defined(IEM_CFG_TARGET_CPU) \|\| defined(DOXYGEN_RUNNING)
248	# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
249	#endif
250
251
252	/*
253	* X86 config.
254	*/
255
256	#define IEM_IMPLEMENTS_TASKSWITCH
257
258	/** @def IEM_WITH_3DNOW
259	* Includes the 3DNow decoding. */
260	#if !defined(IEM_WITH_3DNOW) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
261	# ifndef IEM_WITHOUT_3DNOW
262	# define IEM_WITH_3DNOW
263	# endif
264	#endif
265
266	/** @def IEM_WITH_THREE_0F_38
267	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
268	#if !defined(IEM_WITH_THREE_0F_38) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
269	# ifndef IEM_WITHOUT_THREE_0F_38
270	# define IEM_WITH_THREE_0F_38
271	# endif
272	#endif
273
274	/** @def IEM_WITH_THREE_0F_3A
275	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
276	#if !defined(IEM_WITH_THREE_0F_3A) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
277	# ifndef IEM_WITHOUT_THREE_0F_3A
278	# define IEM_WITH_THREE_0F_3A
279	# endif
280	#endif
281
282	/** @def IEM_WITH_VEX
283	* Includes the VEX decoding. */
284	#if !defined(IEM_WITH_VEX) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
285	# ifndef IEM_WITHOUT_VEX
286	# define IEM_WITH_VEX
287	# endif
288	#endif
289
290
291	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
292
293	# if !defined(IEM_WITHOUT_INSTRUCTION_STATS) && !defined(DOXYGEN_RUNNING)
294	/** Instruction statistics. */
295	typedef struct IEMINSTRSTATS
296	{
297	# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
298	# include "IEMInstructionStatisticsTmpl.h"
299	# undef IEM_DO_INSTR_STAT
300	} IEMINSTRSTATS;
301	#else
302	struct IEMINSTRSTATS;
303	typedef struct IEMINSTRSTATS IEMINSTRSTATS;
304	#endif
305	/** Pointer to IEM instruction statistics. */
306	typedef IEMINSTRSTATS *PIEMINSTRSTATS;
307
308
309	/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
310	* @{ */
311	#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /*< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. /
312	#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /*< Intel EFLAGS result. /
313	#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /*< AMD EFLAGS result /
314	#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /*< Reserved/dummy entry slot that's the same as 0. /
315	#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /*< For masking the index before use. /
316	/** Selects the right variant from a_aArray.
317	* pVCpu is implicit in the caller context. */
318	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
319	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
320	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
321	* be used because the host CPU does not support the operation. */
322	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
323	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
324	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
325	* array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
326	* into the two.
327	* @sa IEM_SELECT_NATIVE_OR_FALLBACK */
328	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
329	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
330	(a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
331	#else
332	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
333	(a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
334	#endif
335	/** @} */
336
337	/**
338	* Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
339	* indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
340	*
341	* On non-x86 hosts, this will shortcut to the fallback w/o checking the
342	* indicator.
343	*
344	* @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
345	*/
346	#if (defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
347	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
348	(g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
349	#else
350	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
351	#endif
352
353	/** @name Helpers for passing C++ template arguments to an
354	* IEM_MC_NATIVE_EMIT_3/4/5 style macro.
355	* @{
356	*/
357	#define IEM_TEMPL_ARG_1(a1) <a1>
358	#define IEM_TEMPL_ARG_2(a1, a2) <a1,a2>
359	#define IEM_TEMPL_ARG_3(a1, a2, a3) <a1,a2,a3>
360	/** @} */
361
362
363	/**
364	* IEM TLB entry.
365	*
366	* Lookup assembly:
367	* @code{.asm}
368	; Calculate tag.
369	mov rax, [VA]
370	shl rax, 16
371	shr rax, 16 + X86_PAGE_SHIFT
372	or rax, [uTlbRevision]
373
374	; Do indexing.
375	movzx ecx, al
376	lea rcx, [pTlbEntries + rcx]
377
378	; Check tag.
379	cmp [rcx + IEMTLBENTRY.uTag], rax
380	jne .TlbMiss
381
382	; Check access.
383	mov rax, ACCESS_FLAGS \| MAPPING_R3_NOT_VALID \| 0xffffff00
384	and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
385	cmp rax, [uTlbPhysRev]
386	jne .TlbMiss
387
388	; Calc address and we're done.
389	mov eax, X86_PAGE_OFFSET_MASK
390	and eax, [VA]
391	or rax, [rcx + IEMTLBENTRY.pMappingR3]
392	%ifdef VBOX_WITH_STATISTICS
393	inc qword [cTlbHits]
394	%endif
395	jmp .Done
396
397	.TlbMiss:
398	mov r8d, ACCESS_FLAGS
399	mov rdx, [VA]
400	mov rcx, [pVCpu]
401	call iemTlbTypeMiss
402	.Done:
403
404	@endcode
405	*
406	*/
407	typedef struct IEMTLBENTRY
408	{
409	/** The TLB entry tag.
410	* Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
411	* is ASSUMING a virtual address width of 48 bits.
412	*
413	* Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
414	*
415	* The TLB lookup code uses the current TLB revision, which won't ever be zero,
416	* enabling an extremely cheap TLB invalidation most of the time. When the TLB
417	* revision wraps around though, the tags needs to be zeroed.
418	*
419	* @note Try use SHRD instruction? After seeing
420	* https://gmplib.org/~tege/x86-timing.pdf, maybe not.
421	*
422	* @todo This will need to be reorganized for 57-bit wide virtual address and
423	* PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
424	* have to move the TLB entry versioning entirely to the
425	* fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
426	* 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
427	* consumed by PCID and ASID (12 + 6 = 18).
428	*/
429	uint64_t uTag;
430	/** Access flags and physical TLB revision.
431	*
432	* - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
433	* - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
434	* - Bit 2 - page tables - not user (complemented X86_PTE_US).
435	* - Bit 3 - pgm phys/virt - not directly writable.
436	* - Bit 4 - pgm phys page - not directly readable.
437	* - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
438	* - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
439	* - Bit 7 - tlb entry - pMappingR3 member not valid.
440	* - Bits 63 thru 8 are used for the physical TLB revision number.
441	*
442	* We're using complemented bit meanings here because it makes it easy to check
443	* whether special action is required. For instance a user mode write access
444	* would do a "TEST fFlags, (X86_PTE_RW \| X86_PTE_US \| X86_PTE_D)" and a
445	* non-zero result would mean special handling needed because either it wasn't
446	* writable, or it wasn't user, or the page wasn't dirty. A user mode read
447	* access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
448	* need to check any PTE flag.
449	*/
450	uint64_t fFlagsAndPhysRev;
451	/** The guest physical page address. */
452	uint64_t GCPhys;
453	/** Pointer to the ring-3 mapping. */
454	R3PTRTYPE(uint8_t *) pbMappingR3;
455	#if HC_ARCH_BITS == 32
456	uint32_t u32Padding1;
457	#endif
458	} IEMTLBENTRY;
459	AssertCompileSize(IEMTLBENTRY, 32);
460	/** Pointer to an IEM TLB entry. */
461	typedef IEMTLBENTRY *PIEMTLBENTRY;
462	/** Pointer to a const IEM TLB entry. */
463	typedef IEMTLBENTRY const *PCIEMTLBENTRY;
464
465	/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
466	* @{ */
467	#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /*< Page tables: Not executable. /
468	#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /*< Page tables: Not writable. /
469	#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /*< Page tables: Not user accessible (supervisor only). /
470	#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /*< Phys page: Not writable (access handler, ROM, whatever). /
471	#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /*< Phys page: Not readable (MMIO / access handler, ROM) /
472	#define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /*< Phys tables: Not accessed (need to be marked accessed). /
473	#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /*< Page tables: Not dirty (needs to be made dirty on write). /
474	#define IEMTLBE_F_PT_LARGE_PAGE RT_BIT_64(7) /*< Page tables: Large 2 or 4 MiB page (for flushing). /
475	#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(8) /*< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. /
476	#define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(9) /*< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). /
477	#define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(10) /*< Phys page: Code page. /
478	#define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffff800) /*< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR /
479	/** @} */
480	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
481	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
482	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
483	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
484	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
485	AssertCompile(PGM_WALKINFO_BIG_PAGE == IEMTLBE_F_PT_LARGE_PAGE);
486	/** The bits set by PGMPhysIemGCPhys2PtrNoLock. */
487	#define IEMTLBE_GCPHYS2PTR_MASK ( PGMIEMGCPHYS2PTR_F_NO_WRITE \
488	\| PGMIEMGCPHYS2PTR_F_NO_READ \
489	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \
490	\| PGMIEMGCPHYS2PTR_F_UNASSIGNED \
491	\| PGMIEMGCPHYS2PTR_F_CODE_PAGE \
492	\| IEMTLBE_F_PHYS_REV )
493
494
495	/** The TLB size (power of two).
496	* We initially chose 256 because that way we can obtain the result directly
497	* from a 8-bit register without an additional AND instruction.
498	* See also @bugref{10687}. */
499	#if defined(RT_ARCH_AMD64)
500	# define IEMTLB_ENTRY_COUNT 256
501	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 8
502	#else
503	# define IEMTLB_ENTRY_COUNT 8192
504	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 13
505	#endif
506	AssertCompile(RT_BIT_32(IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO) == IEMTLB_ENTRY_COUNT);
507
508	/** TLB slot format spec (assumes uint32_t or unsigned value). */
509	#if IEMTLB_ENTRY_COUNT <= 0x100 / 2
510	# define IEMTLB_SLOT_FMT "%02x"
511	#elif IEMTLB_ENTRY_COUNT <= 0x1000 / 2
512	# define IEMTLB_SLOT_FMT "%03x"
513	#elif IEMTLB_ENTRY_COUNT <= 0x10000 / 2
514	# define IEMTLB_SLOT_FMT "%04x"
515	#else
516	# define IEMTLB_SLOT_FMT "%05x"
517	#endif
518
519	/** Enable the large page bitmap TLB optimization.
520	*
521	* The idea here is to avoid scanning the full 32 KB (2MB pages, 2*512 TLB
522	* entries) or 64 KB (4MB pages, 2*1024 TLB entries) worth of TLB entries during
523	* invlpg when large pages are used, and instead just scan 128 or 256 bytes of
524	* the bmLargePage bitmap to determin which TLB entires that might be containing
525	* large pages and actually require checking.
526	*
527	* There is a good posibility of false positives since we currently don't clear
528	* the bitmap when flushing the TLB, but it should help reduce the workload when
529	* the large pages aren't fully loaded into the TLB in their entirity...
530	*/
531	#define IEMTLB_WITH_LARGE_PAGE_BITMAP
532
533	/**
534	* An IEM TLB.
535	*
536	* We've got two of these, one for data and one for instructions.
537	*/
538	typedef struct IEMTLB
539	{
540	/** The non-global TLB revision.
541	* This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
542	* by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
543	* the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
544	* (The revision zero indicates an invalid TLB entry.)
545	*
546	* The initial value is choosen to cause an early wraparound. */
547	uint64_t uTlbRevision;
548	/** The TLB physical address revision - shadow of PGM variable.
549	*
550	* This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
551	* incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
552	* a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
553	* as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
554	*
555	* The initial value is choosen to cause an early wraparound.
556	*
557	* @note This is placed between the two TLB revisions because we
558	* load it in pair with one or the other on arm64. */
559	uint64_t volatile uTlbPhysRev;
560	/** The global TLB revision.
561	* Same as uTlbRevision, but only increased for global flushes. */
562	uint64_t uTlbRevisionGlobal;
563
564	/** Large page tag range.
565	*
566	* This is used to avoid scanning a large page's worth of TLB entries for each
567	* INVLPG instruction, and only to do so iff we've loaded any and when the
568	* address is in this range. This is kept up to date when we loading new TLB
569	* entries.
570	*/
571	struct LARGEPAGERANGE
572	{
573	/** The lowest large page address tag, UINT64_MAX if none. */
574	uint64_t uFirstTag;
575	/** The highest large page address tag (with offset mask part set), 0 if none. */
576	uint64_t uLastTag;
577	}
578	/** Large page range for non-global pages. */
579	NonGlobalLargePageRange,
580	/** Large page range for global pages. */
581	GlobalLargePageRange;
582	/** Number of non-global entries for large pages loaded since last TLB flush. */
583	uint32_t cTlbNonGlobalLargePageCurLoads;
584	/** Number of global entries for large pages loaded since last TLB flush. */
585	uint32_t cTlbGlobalLargePageCurLoads;
586
587	/* Statistics: */
588
589	/** TLB hits in IEMAll.cpp code (IEM_WITH_TLB_STATISTICS only; both).
590	* @note For the data TLB this is only used in iemMemMap and and for direct (i.e.
591	* not via safe read/write path) calls to iemMemMapJmp. */
592	uint64_t cTlbCoreHits;
593	/** Safe read/write TLB hits in iemMemMapJmp (IEM_WITH_TLB_STATISTICS
594	* only; data tlb only). */
595	uint64_t cTlbSafeHits;
596	/** TLB hits in IEMAllMemRWTmplInline.cpp.h (data + IEM_WITH_TLB_STATISTICS only). */
597	uint64_t cTlbInlineCodeHits;
598
599	/** TLB misses in IEMAll.cpp code (both).
600	* @note For the data TLB this is only used in iemMemMap and for direct (i.e.
601	* not via safe read/write path) calls to iemMemMapJmp. So,
602	* for the data TLB this more like 'other misses', while for the code
603	* TLB is all misses. */
604	uint64_t cTlbCoreMisses;
605	/** Subset of cTlbCoreMisses that results in PTE.G=1 loads (odd entries). */
606	uint64_t cTlbCoreGlobalLoads;
607	/** Safe read/write TLB misses in iemMemMapJmp (so data only). */
608	uint64_t cTlbSafeMisses;
609	/** Subset of cTlbSafeMisses that results in PTE.G=1 loads (odd entries). */
610	uint64_t cTlbSafeGlobalLoads;
611	/** Safe read path taken (data only). */
612	uint64_t cTlbSafeReadPath;
613	/** Safe write path taken (data only). */
614	uint64_t cTlbSafeWritePath;
615
616	/** @name Details for native code TLB misses.
617	* @note These counts are included in the above counters (cTlbSafeReadPath,
618	* cTlbSafeWritePath, cTlbInlineCodeHits).
619	* @{ */
620	/** TLB misses in native code due to tag mismatch. */
621	STAMCOUNTER cTlbNativeMissTag;
622	/** TLB misses in native code due to flags or physical revision mismatch. */
623	STAMCOUNTER cTlbNativeMissFlagsAndPhysRev;
624	/** TLB misses in native code due to misaligned access. */
625	STAMCOUNTER cTlbNativeMissAlignment;
626	/** TLB misses in native code due to cross page access. */
627	uint32_t cTlbNativeMissCrossPage;
628	/** TLB misses in native code due to non-canonical address. */
629	uint32_t cTlbNativeMissNonCanonical;
630	/** @} */
631
632	/** Slow read path (code only). */
633	uint32_t cTlbSlowCodeReadPath;
634
635	/** Regular TLB flush count. */
636	uint32_t cTlsFlushes;
637	/** Global TLB flush count. */
638	uint32_t cTlsGlobalFlushes;
639	/** Revision rollovers. */
640	uint32_t cTlbRevisionRollovers;
641	/** Physical revision flushes. */
642	uint32_t cTlbPhysRevFlushes;
643	/** Physical revision rollovers. */
644	uint32_t cTlbPhysRevRollovers;
645
646	/** Number of INVLPG (and similar) operations. */
647	uint32_t cTlbInvlPg;
648	/** Subset of cTlbInvlPg that involved non-global large pages. */
649	uint32_t cTlbInvlPgLargeNonGlobal;
650	/** Subset of cTlbInvlPg that involved global large pages. */
651	uint32_t cTlbInvlPgLargeGlobal;
652
653	uint32_t au32Padding[13];
654
655	/** The TLB entries.
656	* Even entries are for PTE.G=0 and uses uTlbRevision.
657	* Odd entries are for PTE.G=1 and uses uTlbRevisionGlobal. */
658	IEMTLBENTRY aEntries[IEMTLB_ENTRY_COUNT * 2];
659	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
660	/** Bitmap tracking TLB entries for large pages.
661	* This duplicates IEMTLBE_F_PT_LARGE_PAGE for each TLB entry. */
662	uint64_t bmLargePage[IEMTLB_ENTRY_COUNT * 2 / 64];
663	#endif
664	} IEMTLB;
665	AssertCompileSizeAlignment(IEMTLB, 64);
666	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
667	AssertCompile(IEMTLB_ENTRY_COUNT >= 32 /* bmLargePage ASSUMPTION */);
668	#endif
669	/** The width (in bits) of the address portion of the TLB tag. */
670	#define IEMTLB_TAG_ADDR_WIDTH 36
671	/** IEMTLB::uTlbRevision increment. */
672	#define IEMTLB_REVISION_INCR RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH)
673	/** IEMTLB::uTlbRevision mask. */
674	#define IEMTLB_REVISION_MASK (~(RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH) - 1))
675
676	/** IEMTLB::uTlbPhysRev increment.
677	* @sa IEMTLBE_F_PHYS_REV */
678	#define IEMTLB_PHYS_REV_INCR RT_BIT_64(11)
679	AssertCompile(IEMTLBE_F_PHYS_REV == ~(IEMTLB_PHYS_REV_INCR - 1U));
680
681	/**
682	* Calculates the TLB tag for a virtual address but without TLB revision.
683	* @returns Tag value for indexing and comparing with IEMTLB::uTag.
684	* @param a_GCPtr The virtual address. Must be RTGCPTR or same size or
685	* the clearing of the top 16 bits won't work (if 32-bit
686	* we'll end up with mostly zeros).
687	*/
688	#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
689	/**
690	* Converts a TLB tag value into a even TLB index.
691	* @returns Index into IEMTLB::aEntries.
692	* @param a_uTag Value returned by IEMTLB_CALC_TAG.
693	*/
694	#if IEMTLB_ENTRY_COUNT == 256
695	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( (uint8_t)(a_uTag) * 2U )
696	#else
697	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( ((a_uTag) & (IEMTLB_ENTRY_COUNT - 1U)) * 2U )
698	AssertCompile(RT_IS_POWER_OF_TWO(IEMTLB_ENTRY_COUNT));
699	#endif
700	/**
701	* Converts a TLB tag value into an even TLB index.
702	* @returns Pointer into IEMTLB::aEntries corresponding to .
703	* @param a_pTlb The TLB.
704	* @param a_uTag Value returned by IEMTLB_CALC_TAG or
705	* IEMTLB_CALC_TAG_NO_REV.
706	*/
707	#define IEMTLB_TAG_TO_EVEN_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_EVEN_INDEX(a_uTag)] )
708
709	/** Converts a GC address to an even TLB index. */
710	#define IEMTLB_ADDR_TO_EVEN_INDEX(a_GCPtr) IEMTLB_TAG_TO_EVEN_INDEX(IEMTLB_CALC_TAG_NO_REV(a_GCPtr))
711
712
713	/** @def IEM_WITH_TLB_TRACE
714	* Enables the TLB tracing.
715	* Adjust buffer size in IEMR3Init. */
716	#if defined(DOXYGEN_RUNNING) \|\| 0
717	# define IEM_WITH_TLB_TRACE
718	#endif
719
720	#ifdef IEM_WITH_TLB_TRACE
721
722	/** TLB trace entry types. */
723	typedef enum : uint8_t
724	{
725	kIemTlbTraceType_Invalid,
726	kIemTlbTraceType_InvlPg,
727	kIemTlbTraceType_EvictSlot,
728	kIemTlbTraceType_LargeEvictSlot,
729	kIemTlbTraceType_LargeScan,
730	kIemTlbTraceType_Flush,
731	kIemTlbTraceType_FlushGlobal,
732	kIemTlbTraceType_Load,
733	kIemTlbTraceType_LoadGlobal,
734	kIemTlbTraceType_Load_Cr0, /*< x86 specific /
735	kIemTlbTraceType_Load_Cr3, /*< x86 specific /
736	kIemTlbTraceType_Load_Cr4, /*< x86 specific /
737	kIemTlbTraceType_Load_Efer, /*< x86 specific /
738	kIemTlbTraceType_Irq,
739	kIemTlbTraceType_Xcpt,
740	kIemTlbTraceType_IRet, /*< x86 specific /
741	kIemTlbTraceType_Tb_Compile,
742	kIemTlbTraceType_Tb_Exec_Threaded,
743	kIemTlbTraceType_Tb_Exec_Native,
744	kIemTlbTraceType_User0,
745	kIemTlbTraceType_User1,
746	kIemTlbTraceType_User2,
747	kIemTlbTraceType_User3,
748	} IEMTLBTRACETYPE;
749
750	/** TLB trace entry. */
751	typedef struct IEMTLBTRACEENTRY
752	{
753	/** The flattened RIP for the event. */
754	uint64_t rip;
755	/** The event type. */
756	IEMTLBTRACETYPE enmType;
757	/** Byte parameter - typically used as 'bool fDataTlb'. */
758	uint8_t bParam;
759	/** 16-bit parameter value. */
760	uint16_t u16Param;
761	/** 32-bit parameter value. */
762	uint32_t u32Param;
763	/** 64-bit parameter value. */
764	uint64_t u64Param;
765	/** 64-bit parameter value. */
766	uint64_t u64Param2;
767	} IEMTLBTRACEENTRY;
768	AssertCompileSize(IEMTLBTRACEENTRY, 32);
769	/** Pointer to a TLB trace entry. */
770	typedef IEMTLBTRACEENTRY *PIEMTLBTRACEENTRY;
771	/** Pointer to a const TLB trace entry. */
772	typedef IEMTLBTRACEENTRY const *PCIEMTLBTRACEENTRY;
773	#endif /* !IEM_WITH_TLB_TRACE */
774
775	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
776	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) \
777	iemTlbTrace(a_pVCpu, kIemTlbTraceType_InvlPg, a_GCPtr)
778	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
779	iemTlbTrace(a_pVCpu, kIemTlbTraceType_EvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
780	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
781	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeEvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
782	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) \
783	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeScan, 0, 0, a_fDataTlb, (uint8_t)a_fGlobal \| ((uint8_t)a_fNonGlobal << 1))
784	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) \
785	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Flush, a_uRev, 0, a_fDataTlb)
786	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) \
787	iemTlbTrace(a_pVCpu, kIemTlbTraceType_FlushGlobal, a_uRev, a_uGRev, a_fDataTlb)
788	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
789	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
790	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
791	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LoadGlobal, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
792	#else
793	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) do { } while (0)
794	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
795	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
796	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) do { } while (0)
797	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) do { } while (0)
798	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) do { } while (0)
799	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
800	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
801	#endif
802
803	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
804	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr0, a_uNew, a_uOld)
805	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr3, a_uNew, a_uOld)
806	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr4, a_uNew, a_uOld)
807	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Efer, a_uNew, a_uOld)
808	#else
809	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) do { } while (0)
810	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) do { } while (0)
811	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) do { } while (0)
812	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) do { } while (0)
813	#endif
814
815	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
816	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) \
817	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Irq, a_fEFlags, 0, a_uVector, a_fFlags)
818	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) \
819	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Xcpt, a_uErr, a_uCr2, a_uVector, a_fFlags)
820	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) \
821	iemTlbTrace(a_pVCpu, kIemTlbTraceType_IRet, a_uRetRip, a_fEFlags, 0, a_uRetCs)
822	#else
823	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) do { } while (0)
824	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) do { } while (0)
825	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) do { } while (0)
826	#endif
827
828	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
829	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) \
830	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Compile, a_GCPhysPc)
831	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) \
832	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Threaded, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
833	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) \
834	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Native, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
835	#else
836	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) do { } while (0)
837	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) do { } while (0)
838	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) do { } while (0)
839	#endif
840
841	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
842	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
843	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User0, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
844	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
845	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User1, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
846	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
847	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User2, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
848	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
849	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User3, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
850	#else
851	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
852	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
853	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
854	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
855	#endif
856
857
858	/** @name IEM_MC_F_XXX - MC block flags/clues.
859	* @note x86 specific
860	* @todo Merge with IEM_CIMPL_F_XXX
861	* @{ */
862	#define IEM_MC_F_ONLY_8086 RT_BIT_32(0)
863	#define IEM_MC_F_MIN_186 RT_BIT_32(1)
864	#define IEM_MC_F_MIN_286 RT_BIT_32(2)
865	#define IEM_MC_F_NOT_286_OR_OLDER IEM_MC_F_MIN_386
866	#define IEM_MC_F_MIN_386 RT_BIT_32(3)
867	#define IEM_MC_F_MIN_486 RT_BIT_32(4)
868	#define IEM_MC_F_MIN_PENTIUM RT_BIT_32(5)
869	#define IEM_MC_F_MIN_PENTIUM_II IEM_MC_F_MIN_PENTIUM
870	#define IEM_MC_F_MIN_CORE IEM_MC_F_MIN_PENTIUM
871	#define IEM_MC_F_64BIT RT_BIT_32(6)
872	#define IEM_MC_F_NOT_64BIT RT_BIT_32(7)
873	/** This is set by IEMAllN8vePython.py to indicate a variation with the
874	* flags-clearing-and-checking. */
875	#define IEM_MC_F_WITH_FLAGS RT_BIT_32(8)
876	/** This is set by IEMAllN8vePython.py to indicate a variation without the
877	* flags-clearing-and-checking, when there is also a variation with that.
878	* @note Do not set this manully, it's only for python and for testing in
879	* the native recompiler! */
880	#define IEM_MC_F_WITHOUT_FLAGS RT_BIT_32(9)
881	/** @} */
882
883	/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
884	*
885	* These clues are mainly for the recompiler, so that it can emit correct code.
886	*
887	* They are processed by the python script and which also automatically
888	* calculates flags for MC blocks based on the statements, extending the use of
889	* these flags to describe MC block behavior to the recompiler core. The python
890	* script pass the flags to the IEM_MC2_END_EMIT_CALLS macro, but mainly for
891	* error checking purposes. The script emits the necessary fEndTb = true and
892	* similar statements as this reduces compile time a tiny bit.
893	*
894	* @{ */
895	/** Flag set if direct branch, clear if absolute or indirect. */
896	#define IEM_CIMPL_F_BRANCH_DIRECT RT_BIT_32(0)
897	/** Flag set if indirect branch, clear if direct or relative.
898	* This is also used for all system control transfers (SYSCALL, SYSRET, INT, ++)
899	* as well as for return instructions (RET, IRET, RETF). */
900	#define IEM_CIMPL_F_BRANCH_INDIRECT RT_BIT_32(1)
901	/** Flag set if relative branch, clear if absolute or indirect. */
902	#define IEM_CIMPL_F_BRANCH_RELATIVE RT_BIT_32(2)
903	/** Flag set if conditional branch, clear if unconditional. */
904	#define IEM_CIMPL_F_BRANCH_CONDITIONAL RT_BIT_32(3)
905	/** Flag set if it's a far branch (changes CS).
906	* @note x86 specific */
907	#define IEM_CIMPL_F_BRANCH_FAR RT_BIT_32(4)
908	/** Convenience: Testing any kind of branch. */
909	#define IEM_CIMPL_F_BRANCH_ANY (IEM_CIMPL_F_BRANCH_DIRECT \| IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_RELATIVE)
910
911	/** Execution flags may change (IEMCPU::fExec). */
912	#define IEM_CIMPL_F_MODE RT_BIT_32(5)
913	/** May change significant portions of RFLAGS.
914	* @note x86 specific */
915	#define IEM_CIMPL_F_RFLAGS RT_BIT_32(6)
916	/** May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS.
917	* @note x86 specific */
918	#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(7)
919	/** May trigger interrupt shadowing.
920	* @note x86 specific */
921	#define IEM_CIMPL_F_INHIBIT_SHADOW RT_BIT_32(8)
922	/** May enable interrupts, so recheck IRQ immediately afterwards executing
923	* the instruction. */
924	#define IEM_CIMPL_F_CHECK_IRQ_AFTER RT_BIT_32(9)
925	/** May disable interrupts, so recheck IRQ immediately before executing the
926	* instruction. */
927	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE RT_BIT_32(10)
928	/** Convenience: Check for IRQ both before and after an instruction. */
929	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE_AND_AFTER (IEM_CIMPL_F_CHECK_IRQ_BEFORE \| IEM_CIMPL_F_CHECK_IRQ_AFTER)
930	/** May trigger a VM exit (treated like IEM_CIMPL_F_MODE atm). */
931	#define IEM_CIMPL_F_VMEXIT RT_BIT_32(11)
932	/** May modify FPU state.
933	* @todo Not sure if this is useful yet. */
934	#define IEM_CIMPL_F_FPU RT_BIT_32(12)
935	/** REP prefixed instruction which may yield before updating PC.
936	* @todo Not sure if this is useful, REP functions now return non-zero
937	* status if they don't update the PC.
938	* @note x86 specific */
939	#define IEM_CIMPL_F_REP RT_BIT_32(13)
940	/** I/O instruction.
941	* @todo Not sure if this is useful yet.
942	* @note x86 specific */
943	#define IEM_CIMPL_F_IO RT_BIT_32(14)
944	/** Force end of TB after the instruction. */
945	#define IEM_CIMPL_F_END_TB RT_BIT_32(15)
946	/** Flag set if a branch may also modify the stack (push/pop return address). */
947	#define IEM_CIMPL_F_BRANCH_STACK RT_BIT_32(16)
948	/** Flag set if a branch may also modify the stack (push/pop return address)
949	* and switch it (load/restore SS:RSP).
950	* @note x86 specific */
951	#define IEM_CIMPL_F_BRANCH_STACK_FAR RT_BIT_32(17)
952	/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
953	#define IEM_CIMPL_F_XCPT \
954	(IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_FAR \| IEM_CIMPL_F_BRANCH_STACK_FAR \
955	\| IEM_CIMPL_F_MODE \| IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_VMEXIT)
956
957	/** The block calls a C-implementation instruction function with two implicit arguments.
958	* Mutually exclusive with IEM_CIMPL_F_CALLS_AIMPL and
959	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
960	* @note The python scripts will add this if missing. */
961	#define IEM_CIMPL_F_CALLS_CIMPL RT_BIT_32(18)
962	/** The block calls an ASM-implementation instruction function.
963	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and
964	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
965	* @note The python scripts will add this if missing. */
966	#define IEM_CIMPL_F_CALLS_AIMPL RT_BIT_32(19)
967	/** The block calls an ASM-implementation instruction function with an implicit
968	* X86FXSTATE pointer argument.
969	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
970	* IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE.
971	* @note The python scripts will add this if missing.
972	* @note x86 specific */
973	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE RT_BIT_32(20)
974	/** The block calls an ASM-implementation instruction function with an implicit
975	* X86XSAVEAREA pointer argument.
976	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
977	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
978	* @note No different from IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE, so same value.
979	* @note The python scripts will add this if missing.
980	* @note x86 specific */
981	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE
982	/** @} */
983
984
985	/** @name IEM_F_XXX - Execution mode flags (IEMCPU::fExec, IEMTB::fFlags).
986	*
987	* These flags are set when entering IEM and adjusted as code is executed, such
988	* that they will always contain the current values as instructions are
989	* finished.
990	*
991	* In recompiled execution mode, (most of) these flags are included in the
992	* translation block selection key and stored in IEMTB::fFlags alongside the
993	* IEMTB_F_XXX flags. The latter flags uses bits 31 thru 24, which are all zero
994	* in IEMCPU::fExec.
995	*
996	* @{ */
997	/** Mode: The block target mode mask. */
998	#define IEM_F_MODE_MASK UINT32_C(0x0000001f)
999	/** Mode: The IEMMODE part of the IEMTB_F_MODE_MASK value. */
1000	#define IEM_F_MODE_CPUMODE_MASK UINT32_C(0x00000003)
1001	/** X86 Mode: Bit used to indicating pre-386 CPU in 16-bit mode (for eliminating
1002	* conditional in EIP/IP updating), and flat wide open CS, SS, DS, and ES in
1003	* 32-bit mode (for simplifying most memory accesses). */
1004	#define IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK UINT32_C(0x00000004)
1005	/** X86 Mode: Bit indicating protected mode, real mode (or SMM) when not set. */
1006	#define IEM_F_MODE_X86_PROT_MASK UINT32_C(0x00000008)
1007	/** X86 Mode: Bit used to indicate virtual 8086 mode (only 16-bit). */
1008	#define IEM_F_MODE_X86_V86_MASK UINT32_C(0x00000010)
1009
1010	/** X86 Mode: 16-bit on 386 or later. */
1011	#define IEM_F_MODE_X86_16BIT UINT32_C(0x00000000)
1012	/** X86 Mode: 80286, 80186 and 8086/88 targetting blocks (EIP update opt). */
1013	#define IEM_F_MODE_X86_16BIT_PRE_386 UINT32_C(0x00000004)
1014	/** X86 Mode: 16-bit protected mode on 386 or later. */
1015	#define IEM_F_MODE_X86_16BIT_PROT UINT32_C(0x00000008)
1016	/** X86 Mode: 16-bit protected mode on 386 or later. */
1017	#define IEM_F_MODE_X86_16BIT_PROT_PRE_386 UINT32_C(0x0000000c)
1018	/** X86 Mode: 16-bit virtual 8086 protected mode (on 386 or later). */
1019	#define IEM_F_MODE_X86_16BIT_PROT_V86 UINT32_C(0x00000018)
1020
1021	/** X86 Mode: 32-bit on 386 or later. */
1022	#define IEM_F_MODE_X86_32BIT UINT32_C(0x00000001)
1023	/** X86 Mode: 32-bit mode with wide open flat CS, SS, DS and ES. */
1024	#define IEM_F_MODE_X86_32BIT_FLAT UINT32_C(0x00000005)
1025	/** X86 Mode: 32-bit protected mode. */
1026	#define IEM_F_MODE_X86_32BIT_PROT UINT32_C(0x00000009)
1027	/** X86 Mode: 32-bit protected mode with wide open flat CS, SS, DS and ES. */
1028	#define IEM_F_MODE_X86_32BIT_PROT_FLAT UINT32_C(0x0000000d)
1029
1030	/** X86 Mode: 64-bit (includes protected, but not the flat bit). */
1031	#define IEM_F_MODE_X86_64BIT UINT32_C(0x0000000a)
1032
1033	/** X86 Mode: Checks if @a a_fExec represent a FLAT mode. */
1034	#define IEM_F_MODE_X86_IS_FLAT(a_fExec) ( ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT \
1035	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT \
1036	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT)
1037
1038	/** Bypass access handlers when set. */
1039	#define IEM_F_BYPASS_HANDLERS UINT32_C(0x00010000)
1040	/** Have pending hardware instruction breakpoints. */
1041	#define IEM_F_PENDING_BRK_INSTR UINT32_C(0x00020000)
1042	/** Have pending hardware data breakpoints. */
1043	#define IEM_F_PENDING_BRK_DATA UINT32_C(0x00040000)
1044
1045	/** X86: Have pending hardware I/O breakpoints. */
1046	#define IEM_F_PENDING_BRK_X86_IO UINT32_C(0x00000400)
1047	/** X86: Disregard the lock prefix (implied or not) when set. */
1048	#define IEM_F_X86_DISREGARD_LOCK UINT32_C(0x00000800)
1049
1050	/** Pending breakpoint mask (what iemCalcExecDbgFlags works out). */
1051	#define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR \| IEM_F_PENDING_BRK_DATA \| IEM_F_PENDING_BRK_X86_IO)
1052
1053	/** Caller configurable options. */
1054	#define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS \| IEM_F_X86_DISREGARD_LOCK)
1055
1056	/** X86: The current protection level (CPL) shift factor. */
1057	#define IEM_F_X86_CPL_SHIFT 8
1058	/** X86: The current protection level (CPL) mask. */
1059	#define IEM_F_X86_CPL_MASK UINT32_C(0x00000300)
1060	/** X86: The current protection level (CPL) shifted mask. */
1061	#define IEM_F_X86_CPL_SMASK UINT32_C(0x00000003)
1062
1063	/** X86: Alignment checks enabled (CR0.AM=1 & EFLAGS.AC=1). */
1064	#define IEM_F_X86_AC UINT32_C(0x00080000)
1065
1066	/** X86 execution context.
1067	* The IEM_F_X86_CTX_XXX values are individual flags that can be combined (with
1068	* the exception of IEM_F_X86_CTX_NORMAL). This allows running VMs from SMM
1069	* mode. */
1070	#define IEM_F_X86_CTX_MASK UINT32_C(0x0000f000)
1071	/** X86 context: Plain regular execution context. */
1072	#define IEM_F_X86_CTX_NORMAL UINT32_C(0x00000000)
1073	/** X86 context: VT-x enabled. */
1074	#define IEM_F_X86_CTX_VMX UINT32_C(0x00001000)
1075	/** X86 context: AMD-V enabled. */
1076	#define IEM_F_X86_CTX_SVM UINT32_C(0x00002000)
1077	/** X86 context: In AMD-V or VT-x guest mode. */
1078	#define IEM_F_X86_CTX_IN_GUEST UINT32_C(0x00004000)
1079	/** X86 context: System management mode (SMM). */
1080	#define IEM_F_X86_CTX_SMM UINT32_C(0x00008000)
1081
1082	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1083	* iemRegFinishClearingRF() most for most situations (CPUMCTX_DBG_HIT_DRX_MASK
1084	* and CPUMCTX_DBG_DBGF_MASK are covered by the IEM_F_PENDING_BRK_XXX bits
1085	* alread). */
1086
1087	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1088	* iemRegFinishClearingRF() most for most situations
1089	* (CPUMCTX_DBG_HIT_DRX_MASK and CPUMCTX_DBG_DBGF_MASK are covered by
1090	* the IEM_F_PENDING_BRK_XXX bits alread). */
1091
1092	/** @} */
1093
1094
1095	/** @name IEMTB_F_XXX - Translation block flags (IEMTB::fFlags).
1096	*
1097	* Extends the IEM_F_XXX flags (subject to IEMTB_F_IEM_F_MASK) to make up the
1098	* translation block flags. The combined flag mask (subject to
1099	* IEMTB_F_KEY_MASK) is used as part of the lookup key for translation blocks.
1100	*
1101	* @{ */
1102	/** Mask of IEM_F_XXX flags included in IEMTB_F_XXX. */
1103	#define IEMTB_F_IEM_F_MASK UINT32_C(0x00ffffff)
1104
1105	/** Type: The block type mask. */
1106	#define IEMTB_F_TYPE_MASK UINT32_C(0x03000000)
1107	/** Type: Purly threaded recompiler (via tables). */
1108	#define IEMTB_F_TYPE_THREADED UINT32_C(0x01000000)
1109	/** Type: Native recompilation. */
1110	#define IEMTB_F_TYPE_NATIVE UINT32_C(0x02000000)
1111
1112	/** Set when we're starting the block in an "interrupt shadow".
1113	* We don't need to distingish between the two types of this mask, thus the one.
1114	* @see CPUMCTX_INHIBIT_SHADOW, CPUMIsInInterruptShadow() */
1115	#define IEMTB_F_INHIBIT_SHADOW UINT32_C(0x04000000)
1116	/** Set when we're currently inhibiting NMIs
1117	* @see CPUMCTX_INHIBIT_NMI, CPUMAreInterruptsInhibitedByNmi() */
1118	#define IEMTB_F_INHIBIT_NMI UINT32_C(0x08000000)
1119
1120	/** Checks that EIP/IP is wihin CS.LIM before each instruction. Used when
1121	* we're close the limit before starting a TB, as determined by
1122	* iemGetTbFlagsForCurrentPc().
1123	* @note x86 specific */
1124	#define IEMTB_F_CS_LIM_CHECKS UINT32_C(0x10000000)
1125
1126	/** Mask of the IEMTB_F_XXX flags that are part of the TB lookup key.
1127	*
1128	* @note We skip all of IEM_F_X86_CTX_MASK, with the exception of SMM (which we
1129	* don't implement), because we don't currently generate any context
1130	* specific code - that's all handled in CIMPL functions.
1131	*
1132	* For the threaded recompiler we don't generate any CPL specific code
1133	* either, but the native recompiler does for memory access (saves getting
1134	* the CPL from fExec and turning it into IEMTLBE_F_PT_NO_USER).
1135	* Since most OSes will not share code between rings, this shouldn't
1136	* have any real effect on TB/memory/recompiling load.
1137	*/
1138	#define IEMTB_F_KEY_MASK ((UINT32_MAX & ~(IEM_F_X86_CTX_MASK \| IEMTB_F_TYPE_MASK)) \| IEM_F_X86_CTX_SMM)
1139	/** @} */
1140
1141	AssertCompile( (IEM_F_MODE_X86_16BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1142	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1143	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_PROT_MASK));
1144	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_V86_MASK));
1145	AssertCompile( (IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1146	AssertCompile( IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1147	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_PROT_MASK));
1148	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1149	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1150	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1151	AssertCompile( IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1152	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_V86_MASK));
1153	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_16BIT);
1154	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1155	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_PROT_MASK);
1156	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1157	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_PROT_MASK);
1158	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1159	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_V86_MASK);
1160
1161	AssertCompile( (IEM_F_MODE_X86_32BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1162	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1163	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_PROT_MASK));
1164	AssertCompile( (IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1165	AssertCompile( IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1166	AssertCompile(!(IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_PROT_MASK));
1167	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1168	AssertCompile(!(IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1169	AssertCompile( IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1170	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT);
1171	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1172	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_PROT_MASK);
1173
1174	AssertCompile( (IEM_F_MODE_X86_64BIT & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT);
1175	AssertCompile( IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_PROT_MASK);
1176	AssertCompile(!(IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1177
1178	/** Native instruction type for use with the native code generator.
1179	* This is a byte (uint8_t) for x86 and amd64 and uint32_t for the other(s). */
1180	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
1181	typedef uint8_t IEMNATIVEINSTR;
1182	#else
1183	typedef uint32_t IEMNATIVEINSTR;
1184	#endif
1185	/** Pointer to a native instruction unit. */
1186	typedef IEMNATIVEINSTR *PIEMNATIVEINSTR;
1187	/** Pointer to a const native instruction unit. */
1188	typedef IEMNATIVEINSTR const *PCIEMNATIVEINSTR;
1189
1190	/**
1191	* A call for the threaded call table.
1192	*/
1193	typedef struct IEMTHRDEDCALLENTRY
1194	{
1195	/** The function to call (IEMTHREADEDFUNCS). */
1196	uint16_t enmFunction;
1197
1198	/** Instruction number in the TB (for statistics). */
1199	uint8_t idxInstr;
1200	/** The opcode length. */
1201	uint8_t cbOpcode;
1202	/** Offset into IEMTB::pabOpcodes. */
1203	uint16_t offOpcode;
1204
1205	/** TB lookup table index (7 bits) and large size (1 bits).
1206	*
1207	* The default size is 1 entry, but for indirect calls and returns we set the
1208	* top bit and allocate 4 (IEM_TB_LOOKUP_TAB_LARGE_SIZE) entries. The large
1209	* tables uses RIP for selecting the entry to use, as it is assumed a hash table
1210	* lookup isn't that slow compared to sequentially trying out 4 TBs.
1211	*
1212	* By default lookup table entry 0 for a TB is reserved as a fallback for
1213	* calltable entries w/o explicit entreis, so this member will be non-zero if
1214	* there is a lookup entry associated with this call.
1215	*
1216	* @sa IEM_TB_LOOKUP_TAB_GET_SIZE, IEM_TB_LOOKUP_TAB_GET_IDX
1217	*/
1218	uint8_t uTbLookup;
1219
1220	/** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
1221	uint8_t fFlags;
1222
1223	/** Generic parameters. */
1224	uint64_t auParams[3];
1225	} IEMTHRDEDCALLENTRY;
1226	AssertCompileSize(IEMTHRDEDCALLENTRY, sizeof(uint64_t) * 4);
1227	/** Pointer to a threaded call entry. */
1228	typedef struct IEMTHRDEDCALLENTRY *PIEMTHRDEDCALLENTRY;
1229	/** Pointer to a const threaded call entry. */
1230	typedef IEMTHRDEDCALLENTRY const *PCIEMTHRDEDCALLENTRY;
1231
1232	/** The number of TB lookup table entries for a large allocation
1233	* (IEMTHRDEDCALLENTRY::uTbLookup bit 7 set). */
1234	#define IEM_TB_LOOKUP_TAB_LARGE_SIZE 4
1235	/** Get the lookup table size from IEMTHRDEDCALLENTRY::uTbLookup. */
1236	#define IEM_TB_LOOKUP_TAB_GET_SIZE(a_uTbLookup) (!((a_uTbLookup) & 0x80) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE)
1237	/** Get the first lookup table index from IEMTHRDEDCALLENTRY::uTbLookup. */
1238	#define IEM_TB_LOOKUP_TAB_GET_IDX(a_uTbLookup) ((a_uTbLookup) & 0x7f)
1239	/** Get the lookup table index from IEMTHRDEDCALLENTRY::uTbLookup and RIP. */
1240	#define IEM_TB_LOOKUP_TAB_GET_IDX_WITH_RIP(a_uTbLookup, a_Rip) \
1241	(!((a_uTbLookup) & 0x80) ? (a_uTbLookup) & 0x7f : ((a_uTbLookup) & 0x7f) + ((a_Rip) & (IEM_TB_LOOKUP_TAB_LARGE_SIZE - 1)) )
1242
1243	/** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
1244	#define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge) ((a_idxTable) \| ((a_fLarge) ? 0x80 : 0))
1245
1246
1247	/** The call entry is a jump target. */
1248	#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET UINT8_C(0x01)
1249
1250
1251	/**
1252	* Native IEM TB 'function' typedef.
1253	*
1254	* This will throw/longjmp on occation.
1255	*
1256	* @note AMD64 doesn't have that many non-volatile registers and does sport
1257	* 32-bit address displacments, so we don't need pCtx.
1258	*
1259	* On ARM64 pCtx allows us to directly address the whole register
1260	* context without requiring a separate indexing register holding the
1261	* offset. This saves an instruction loading the offset for each guest
1262	* CPU context access, at the cost of a non-volatile register.
1263	* Fortunately, ARM64 has quite a lot more registers.
1264	*/
1265	typedef
1266	#ifdef RT_ARCH_AMD64
1267	int FNIEMTBNATIVE(PVMCPUCC pVCpu)
1268	#else
1269	int FNIEMTBNATIVE(PVMCPUCC pVCpu, PCPUMCTX pCtx)
1270	#endif
1271	#if RT_CPLUSPLUS_PREREQ(201700)
1272	IEM_NOEXCEPT_MAY_LONGJMP
1273	#endif
1274	;
1275	/** Pointer to a native IEM TB entry point function.
1276	* This will throw/longjmp on occation. */
1277	typedef FNIEMTBNATIVE *PFNIEMTBNATIVE;
1278
1279
1280	/**
1281	* Translation block.
1282	*
1283	* The current plan is to just keep TBs and associated lookup hash table private
1284	* to each VCpu as that simplifies TB removal greatly (no races) and generally
1285	* avoids using expensive atomic primitives for updating lists and stuff.
1286	*/
1287	#pragma pack(2) /* to prevent the Thrd structure from being padded unnecessarily */
1288	typedef struct IEMTB
1289	{
1290	/** Next block with the same hash table entry. */
1291	struct IEMTB *pNext;
1292	/** Usage counter. */
1293	uint32_t cUsed;
1294	/** The IEMCPU::msRecompilerPollNow last time it was used. */
1295	uint32_t msLastUsed;
1296
1297	/** @name What uniquely identifies the block.
1298	* @{ */
1299	RTGCPHYS GCPhysPc;
1300	/** IEMTB_F_XXX (i.e. IEM_F_XXX ++). */
1301	uint32_t fFlags;
1302	union
1303	{
1304	struct
1305	{
1306	/*< Relevant CS X86DESCATTR_XXX bits. /
1307	uint16_t fAttr;
1308	} x86;
1309	};
1310	/** @} */
1311
1312	/** Number of opcode ranges. */
1313	uint8_t cRanges;
1314	/** Statistics: Number of instructions in the block. */
1315	uint8_t cInstructions;
1316
1317	/** Type specific info. */
1318	union
1319	{
1320	struct
1321	{
1322	/** The call sequence table. */
1323	PIEMTHRDEDCALLENTRY paCalls;
1324	/** Number of calls in paCalls. */
1325	uint16_t cCalls;
1326	/** Number of calls allocated. */
1327	uint16_t cAllocated;
1328	} Thrd;
1329	struct
1330	{
1331	/** The native instructions (PFNIEMTBNATIVE). */
1332	PIEMNATIVEINSTR paInstructions;
1333	/** Number of instructions pointed to by paInstructions. */
1334	uint32_t cInstructions;
1335	} Native;
1336	/** Generic view for zeroing when freeing. */
1337	struct
1338	{
1339	uintptr_t uPtr;
1340	uint32_t uData;
1341	} Gen;
1342	};
1343
1344	/** The allocation chunk this TB belongs to. */
1345	uint8_t idxAllocChunk;
1346	/** The number of entries in the lookup table.
1347	* Because we're out of space, the TB lookup table is located before the
1348	* opcodes pointed to by pabOpcodes. */
1349	uint8_t cTbLookupEntries;
1350
1351	/** Number of bytes of opcodes stored in pabOpcodes.
1352	* @todo this field isn't really needed, aRanges keeps the actual info. */
1353	uint16_t cbOpcodes;
1354	/** Pointer to the opcode bytes this block was recompiled from.
1355	* This also points to the TB lookup table, which starts cTbLookupEntries
1356	* entries before the opcodes (we don't have room atm for another point). */
1357	uint8_t *pabOpcodes;
1358
1359	union
1360	{
1361	/** Native recompilation debug info if enabled.
1362	* This is only generated by the native recompiler. */
1363	struct IEMTBDBG *pDbgInfo;
1364	/** For threaded TBs and natives when debug info is disabled, this is the flat
1365	* PC corresponding to GCPhysPc. */
1366	RTGCPTR FlatPc;
1367	};
1368
1369	/* --- 64 byte cache line end --- */
1370
1371	/** Opcode ranges.
1372	*
1373	* The opcode checkers and maybe TLB loading functions will use this to figure
1374	* out what to do. The parameter will specify an entry and the opcode offset to
1375	* start at and the minimum number of bytes to verify (instruction length).
1376	*
1377	* When VT-x and AMD-V looks up the opcode bytes for an exitting instruction,
1378	* they'll first translate RIP (+ cbInstr - 1) to a physical address using the
1379	* code TLB (must have a valid entry for that address) and scan the ranges to
1380	* locate the corresponding opcodes. Probably.
1381	*/
1382	struct IEMTBOPCODERANGE
1383	{
1384	/** Offset within pabOpcodes. */
1385	uint16_t offOpcodes;
1386	/** Number of bytes. */
1387	uint16_t cbOpcodes;
1388	/** The page offset. */
1389	RT_GCC_EXTENSION
1390	uint16_t offPhysPage : 12;
1391	/** Unused bits. */
1392	RT_GCC_EXTENSION
1393	uint16_t u2Unused : 2;
1394	/** Index into GCPhysPc + aGCPhysPages for the physical page address. */
1395	RT_GCC_EXTENSION
1396	uint16_t idxPhysPage : 2;
1397	} aRanges[8];
1398
1399	/** Physical pages that this TB covers.
1400	* The GCPhysPc w/o page offset is element zero, so starting here with 1. */
1401	RTGCPHYS aGCPhysPages[2];
1402	} IEMTB;
1403	#pragma pack()
1404	AssertCompileMemberAlignment(IEMTB, GCPhysPc, sizeof(RTGCPHYS));
1405	AssertCompileMemberAlignment(IEMTB, Thrd, sizeof(void *));
1406	AssertCompileMemberAlignment(IEMTB, pabOpcodes, sizeof(void *));
1407	AssertCompileMemberAlignment(IEMTB, pDbgInfo, sizeof(void *));
1408	AssertCompileMemberAlignment(IEMTB, aGCPhysPages, sizeof(RTGCPHYS));
1409	AssertCompileMemberOffset(IEMTB, aRanges, 64);
1410	AssertCompileMemberSize(IEMTB, aRanges[0], 6);
1411	#if 1
1412	AssertCompileSize(IEMTB, 128);
1413	# define IEMTB_SIZE_IS_POWER_OF_TWO /*< The IEMTB size is a power of two. /
1414	#else
1415	AssertCompileSize(IEMTB, 168);
1416	# undef IEMTB_SIZE_IS_POWER_OF_TWO
1417	#endif
1418
1419	/** Pointer to a translation block. */
1420	typedef IEMTB *PIEMTB;
1421	/** Pointer to a const translation block. */
1422	typedef IEMTB const *PCIEMTB;
1423
1424	/** Gets address of the given TB lookup table entry. */
1425	#define IEMTB_GET_TB_LOOKUP_TAB_ENTRY(a_pTb, a_idx) \
1426	((PIEMTB )&(a_pTb)->pabOpcodes[-(int)((a_pTb)->cTbLookupEntries - (a_idx)) sizeof(PIEMTB)])
1427
1428	/**
1429	* Gets the physical address for a TB opcode range.
1430	*/
1431	DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1432	{
1433	Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1434	uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1435	Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1436	if (idxPage == 0)
1437	return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1438	Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1439	return pTb->aGCPhysPages[idxPage - 1];
1440	}
1441
1442
1443	/**
1444	* A chunk of memory in the TB allocator.
1445	*/
1446	typedef struct IEMTBCHUNK
1447	{
1448	/** Pointer to the translation blocks in this chunk. */
1449	PIEMTB paTbs;
1450	#ifdef IN_RING0
1451	/** Allocation handle. */
1452	RTR0MEMOBJ hMemObj;
1453	#endif
1454	} IEMTBCHUNK;
1455
1456	/**
1457	* A per-CPU translation block allocator.
1458	*
1459	* Because of how the IEMTBCACHE uses the lower 6 bits of the TB address to keep
1460	* the length of the collision list, and of course also for cache line alignment
1461	* reasons, the TBs must be allocated with at least 64-byte alignment.
1462	* Memory is there therefore allocated using one of the page aligned allocators.
1463	*
1464	*
1465	* To avoid wasting too much memory, it is allocated piecemeal as needed,
1466	* in chunks (IEMTBCHUNK) of 2 MiB or more. The TB has an 8-bit chunk index
1467	* that enables us to quickly calculate the allocation bitmap position when
1468	* freeing the translation block.
1469	*/
1470	typedef struct IEMTBALLOCATOR
1471	{
1472	/** Magic value (IEMTBALLOCATOR_MAGIC). */
1473	uint32_t uMagic;
1474
1475	#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
1476	/** Mask corresponding to cTbsPerChunk - 1. */
1477	uint32_t fChunkMask;
1478	/** Shift count corresponding to cTbsPerChunk. */
1479	uint8_t cChunkShift;
1480	#else
1481	uint32_t uUnused;
1482	uint8_t bUnused;
1483	#endif
1484	/** Number of chunks we're allowed to allocate. */
1485	uint8_t cMaxChunks;
1486	/** Number of chunks currently populated. */
1487	uint16_t cAllocatedChunks;
1488	/** Number of translation blocks per chunk. */
1489	uint32_t cTbsPerChunk;
1490	/** Chunk size. */
1491	uint32_t cbPerChunk;
1492
1493	/** The maximum number of TBs. */
1494	uint32_t cMaxTbs;
1495	/** Total number of TBs in the populated chunks.
1496	* (cAllocatedChunks * cTbsPerChunk) */
1497	uint32_t cTotalTbs;
1498	/** The current number of TBs in use.
1499	* The number of free TBs: cAllocatedTbs - cInUseTbs; */
1500	uint32_t cInUseTbs;
1501	/** Statistics: Number of the cInUseTbs that are native ones. */
1502	uint32_t cNativeTbs;
1503	/** Statistics: Number of the cInUseTbs that are threaded ones. */
1504	uint32_t cThreadedTbs;
1505
1506	/** Where to start pruning TBs from when we're out.
1507	* See iemTbAllocatorAllocSlow for details. */
1508	uint32_t iPruneFrom;
1509	/** Where to start pruning native TBs from when we're out of executable memory.
1510	* See iemTbAllocatorFreeupNativeSpace for details. */
1511	uint32_t iPruneNativeFrom;
1512	uint64_t u64Padding;
1513
1514	/** Statistics: Number of TB allocation calls. */
1515	STAMCOUNTER StatAllocs;
1516	/** Statistics: Number of TB free calls. */
1517	STAMCOUNTER StatFrees;
1518	/** Statistics: Time spend pruning. */
1519	STAMPROFILE StatPrune;
1520	/** Statistics: Time spend pruning native TBs. */
1521	STAMPROFILE StatPruneNative;
1522
1523	/** The delayed free list (see iemTbAlloctorScheduleForFree). */
1524	PIEMTB pDelayedFreeHead;
1525	/* Head of the list of free TBs. */
1526	PIEMTB pTbsFreeHead;
1527
1528	/** Allocation chunks. */
1529	IEMTBCHUNK aChunks[256];
1530	} IEMTBALLOCATOR;
1531	/** Pointer to a TB allocator. */
1532	typedef struct IEMTBALLOCATOR *PIEMTBALLOCATOR;
1533
1534	/** Magic value for the TB allocator (Emmet Harley Cohen). */
1535	#define IEMTBALLOCATOR_MAGIC UINT32_C(0x19900525)
1536
1537
1538	/**
1539	* A per-CPU translation block cache (hash table).
1540	*
1541	* The hash table is allocated once during IEM initialization and size double
1542	* the max TB count, rounded up to the nearest power of two (so we can use and
1543	* AND mask rather than a rest division when hashing).
1544	*/
1545	typedef struct IEMTBCACHE
1546	{
1547	/** Magic value (IEMTBCACHE_MAGIC). */
1548	uint32_t uMagic;
1549	/** Size of the hash table. This is a power of two. */
1550	uint32_t cHash;
1551	/** The mask corresponding to cHash. */
1552	uint32_t uHashMask;
1553	uint32_t uPadding;
1554
1555	/** @name Statistics
1556	* @{ */
1557	/** Number of collisions ever. */
1558	STAMCOUNTER cCollisions;
1559
1560	/** Statistics: Number of TB lookup misses. */
1561	STAMCOUNTER cLookupMisses;
1562	/** Statistics: Number of TB lookup hits via hash table (debug only). */
1563	STAMCOUNTER cLookupHits;
1564	/** Statistics: Number of TB lookup hits via TB associated lookup table (debug only). */
1565	STAMCOUNTER cLookupHitsViaTbLookupTable;
1566	STAMCOUNTER auPadding2[2];
1567	/** Statistics: Collision list length pruning. */
1568	STAMPROFILE StatPrune;
1569	/** @} */
1570
1571	/** The hash table itself.
1572	* @note The lower 6 bits of the pointer is used for keeping the collision
1573	* list length, so we can take action when it grows too long.
1574	* This works because TBs are allocated using a 64 byte (or
1575	* higher) alignment from page aligned chunks of memory, so the lower
1576	* 6 bits of the address will always be zero.
1577	* See IEMTBCACHE_PTR_COUNT_MASK, IEMTBCACHE_PTR_MAKE and friends.
1578	*/
1579	RT_FLEXIBLE_ARRAY_EXTENSION
1580	PIEMTB apHash[RT_FLEXIBLE_ARRAY];
1581	} IEMTBCACHE;
1582	/** Pointer to a per-CPU translation block cache. */
1583	typedef IEMTBCACHE *PIEMTBCACHE;
1584
1585	/** Magic value for IEMTBCACHE (Johnny O'Neal). */
1586	#define IEMTBCACHE_MAGIC UINT32_C(0x19561010)
1587
1588	/** The collision count mask for IEMTBCACHE::apHash entries. */
1589	#define IEMTBCACHE_PTR_COUNT_MASK ((uintptr_t)0x3f)
1590	/** The max collision count for IEMTBCACHE::apHash entries before pruning. */
1591	#define IEMTBCACHE_PTR_MAX_COUNT ((uintptr_t)0x30)
1592	/** Combine a TB pointer and a collision list length into a value for an
1593	* IEMTBCACHE::apHash entry. */
1594	#define IEMTBCACHE_PTR_MAKE(a_pTb, a_cCount) (PIEMTB)((uintptr_t)(a_pTb) \| (a_cCount))
1595	/** Combine a TB pointer and a collision list length into a value for an
1596	* IEMTBCACHE::apHash entry. */
1597	#define IEMTBCACHE_PTR_GET_TB(a_pHashEntry) (PIEMTB)((uintptr_t)(a_pHashEntry) & ~IEMTBCACHE_PTR_COUNT_MASK)
1598	/** Combine a TB pointer and a collision list length into a value for an
1599	* IEMTBCACHE::apHash entry. */
1600	#define IEMTBCACHE_PTR_GET_COUNT(a_pHashEntry) ((uintptr_t)(a_pHashEntry) & IEMTBCACHE_PTR_COUNT_MASK)
1601
1602	/**
1603	* Calculates the hash table slot for a TB from physical PC address and TB flags.
1604	*/
1605	#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
1606	IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, (a_fTbFlags) & IEMTB_F_KEY_MASK, a_GCPhysPc)
1607
1608	/**
1609	* Calculates the hash table slot for a TB from physical PC address and TB
1610	* flags, ASSUMING the caller has applied IEMTB_F_KEY_MASK to @a a_fTbFlags.
1611	*/
1612	#define IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, a_fTbFlags, a_GCPhysPc) \
1613	(((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
1614
1615
1616	/** @name IEMBRANCHED_F_XXX - Branched indicator (IEMCPU::fTbBranched).
1617	*
1618	* These flags parallels the main IEM_CIMPL_F_BRANCH_XXX flags.
1619	*
1620	* @{ */
1621	/** Value if no branching happened recently. */
1622	#define IEMBRANCHED_F_NO UINT8_C(0x00)
1623	/** Flag set if direct branch, clear if absolute or indirect. */
1624	#define IEMBRANCHED_F_DIRECT UINT8_C(0x01)
1625	/** Flag set if indirect branch, clear if direct or relative. */
1626	#define IEMBRANCHED_F_INDIRECT UINT8_C(0x02)
1627	/** Flag set if relative branch, clear if absolute or indirect. */
1628	#define IEMBRANCHED_F_RELATIVE UINT8_C(0x04)
1629	/** Flag set if conditional branch, clear if unconditional. */
1630	#define IEMBRANCHED_F_CONDITIONAL UINT8_C(0x08)
1631	/** Flag set if it's a far branch.
1632	* @note x86 specific */
1633	#define IEMBRANCHED_F_FAR UINT8_C(0x10)
1634	/** Flag set if the stack pointer is modified. */
1635	#define IEMBRANCHED_F_STACK UINT8_C(0x20)
1636	/** Flag set if the stack pointer and (maybe) the stack segment are modified.
1637	* @note x86 specific */
1638	#define IEMBRANCHED_F_STACK_FAR UINT8_C(0x40)
1639	/** Flag set (by IEM_MC_REL_JMP_XXX) if it's a zero bytes relative jump. */
1640	#define IEMBRANCHED_F_ZERO UINT8_C(0x80)
1641	/** @} */
1642
1643
1644	/**
1645	* The per-CPU IEM state.
1646	*/
1647	typedef struct IEMCPU
1648	{
1649	/** Info status code that needs to be propagated to the IEM caller.
1650	* This cannot be passed internally, as it would complicate all success
1651	* checks within the interpreter making the code larger and almost impossible
1652	* to get right. Instead, we'll store status codes to pass on here. Each
1653	* source of these codes will perform appropriate sanity checks. */
1654	int32_t rcPassUp; /* 0x00 */
1655	/** Execution flag, IEM_F_XXX. */
1656	uint32_t fExec; /* 0x04 */
1657
1658	/** @name Decoder state.
1659	* @{ */
1660	#ifdef IEM_WITH_CODE_TLB
1661	/** The offset of the next instruction byte. */
1662	uint32_t offInstrNextByte; /* 0x08 */
1663	/** The number of bytes available at pbInstrBuf for the current instruction.
1664	* This takes the max opcode length into account so that doesn't need to be
1665	* checked separately. */
1666	uint32_t cbInstrBuf; /* 0x0c */
1667	/** Pointer to the page containing RIP, user specified buffer or abOpcode.
1668	* This can be NULL if the page isn't mappable for some reason, in which
1669	* case we'll do fallback stuff.
1670	*
1671	* If we're executing an instruction from a user specified buffer,
1672	* IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
1673	* aligned pointer but pointer to the user data.
1674	*
1675	* For instructions crossing pages, this will start on the first page and be
1676	* advanced to the next page by the time we've decoded the instruction. This
1677	* therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
1678	*/
1679	uint8_t const pbInstrBuf; / 0x10 */
1680	# if ARCH_BITS == 32
1681	uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
1682	# endif
1683	/** The program counter corresponding to pbInstrBuf.
1684	* This is set to a non-canonical address when we need to invalidate it. */
1685	uint64_t uInstrBufPc; /* 0x18 */
1686	/** The guest physical address corresponding to pbInstrBuf. */
1687	RTGCPHYS GCPhysInstrBuf; /* 0x20 */
1688	/** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1689	* This takes the CS segment limit into account.
1690	* @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1691	uint16_t cbInstrBufTotal; /* 0x28 */
1692	/** Offset into pbInstrBuf of the first byte of the current instruction.
1693	* Can be negative to efficiently handle cross page instructions. */
1694	int16_t offCurInstrStart; /* 0x2a */
1695
1696	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1697	/** The prefix mask (IEM_OP_PRF_XXX). */
1698	uint32_t fPrefixes; /* 0x2c */
1699	/** The extra REX ModR/M register field bit (REX.R << 3). */
1700	uint8_t uRexReg; /* 0x30 */
1701	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
1702	* (REX.B << 3). */
1703	uint8_t uRexB; /* 0x31 */
1704	/** The extra REX SIB index field bit (REX.X << 3). */
1705	uint8_t uRexIndex; /* 0x32 */
1706
1707	/** The effective segment register (X86_SREG_XXX). */
1708	uint8_t iEffSeg; /* 0x33 */
1709
1710	/** The offset of the ModR/M byte relative to the start of the instruction. */
1711	uint8_t offModRm; /* 0x34 */
1712
1713	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1714	/** The current offset into abOpcode. */
1715	uint8_t offOpcode; /* 0x35 */
1716	# else
1717	uint8_t bUnused; /* 0x35 */
1718	# endif
1719	# else /* IEM_WITH_OPAQUE_DECODER_STATE */
1720	uint8_t abOpaqueDecoderPart1[0x36 - 0x2c];
1721	# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1722
1723	#else /* !IEM_WITH_CODE_TLB */
1724	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1725	/** The size of what has currently been fetched into abOpcode. */
1726	uint8_t cbOpcode; /* 0x08 */
1727	/** The current offset into abOpcode. */
1728	uint8_t offOpcode; /* 0x09 */
1729	/** The offset of the ModR/M byte relative to the start of the instruction. */
1730	uint8_t offModRm; /* 0x0a */
1731
1732	/** The effective segment register (X86_SREG_XXX). */
1733	uint8_t iEffSeg; /* 0x0b */
1734
1735	/** The prefix mask (IEM_OP_PRF_XXX). */
1736	uint32_t fPrefixes; /* 0x0c */
1737	/** The extra REX ModR/M register field bit (REX.R << 3). */
1738	uint8_t uRexReg; /* 0x10 */
1739	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
1740	* (REX.B << 3). */
1741	uint8_t uRexB; /* 0x11 */
1742	/** The extra REX SIB index field bit (REX.X << 3). */
1743	uint8_t uRexIndex; /* 0x12 */
1744
1745	# else /* IEM_WITH_OPAQUE_DECODER_STATE */
1746	uint8_t abOpaqueDecoderPart1[0x13 - 0x08];
1747	# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1748	#endif /* !IEM_WITH_CODE_TLB */
1749
1750	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1751	/** The effective operand mode. */
1752	IEMMODE enmEffOpSize; /* 0x36, 0x13 */
1753	/** The default addressing mode. */
1754	IEMMODE enmDefAddrMode; /* 0x37, 0x14 */
1755	/** The effective addressing mode. */
1756	IEMMODE enmEffAddrMode; /* 0x38, 0x15 */
1757	/** The default operand mode. */
1758	IEMMODE enmDefOpSize; /* 0x39, 0x16 */
1759
1760	/** Prefix index (VEX.pp) for two byte and three byte tables. */
1761	uint8_t idxPrefix; /* 0x3a, 0x17 */
1762	/** 3rd VEX/EVEX/XOP register.
1763	* Please use IEM_GET_EFFECTIVE_VVVV to access. */
1764	uint8_t uVex3rdReg; /* 0x3b, 0x18 */
1765	/** The VEX/EVEX/XOP length field. */
1766	uint8_t uVexLength; /* 0x3c, 0x19 */
1767	/** Additional EVEX stuff. */
1768	uint8_t fEvexStuff; /* 0x3d, 0x1a */
1769
1770	# ifndef IEM_WITH_CODE_TLB
1771	/** Explicit alignment padding. */
1772	uint8_t abAlignment2a[1]; /* 0x1b */
1773	# endif
1774	/** The FPU opcode (FOP). */
1775	uint16_t uFpuOpcode; /* 0x3e, 0x1c */
1776	# ifndef IEM_WITH_CODE_TLB
1777	/** Explicit alignment padding. */
1778	uint8_t abAlignment2b[2]; /* 0x1e */
1779	# endif
1780
1781	/** The opcode bytes. */
1782	uint8_t abOpcode[15]; /* 0x40, 0x20 */
1783	/** Explicit alignment padding. */
1784	# ifdef IEM_WITH_CODE_TLB
1785	//uint8_t abAlignment2c[0x4f - 0x4f]; /* 0x4f */
1786	# else
1787	uint8_t abAlignment2c[0x4f - 0x2f]; /* 0x2f */
1788	# endif
1789
1790	#else /* IEM_WITH_OPAQUE_DECODER_STATE */
1791	# ifdef IEM_WITH_CODE_TLB
1792	uint8_t abOpaqueDecoderPart2[0x4f - 0x36];
1793	# else
1794	uint8_t abOpaqueDecoderPart2[0x4f - 0x13];
1795	# endif
1796	#endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1797	/** @} */
1798
1799
1800	/** The number of active guest memory mappings. */
1801	uint8_t cActiveMappings; /* 0x4f, 0x4f */
1802
1803	/** Records for tracking guest memory mappings. */
1804	struct
1805	{
1806	/** The address of the mapped bytes. */
1807	R3R0PTRTYPE(void *) pv;
1808	/** The access flags (IEM_ACCESS_XXX).
1809	* IEM_ACCESS_INVALID if the entry is unused. */
1810	uint32_t fAccess;
1811	#if HC_ARCH_BITS == 64
1812	uint32_t u32Alignment4; /*< Alignment padding. /
1813	#endif
1814	} aMemMappings[3]; /* 0x50 LB 0x30 */
1815
1816	/** Locking records for the mapped memory. */
1817	union
1818	{
1819	PGMPAGEMAPLOCK Lock;
1820	uint64_t au64Padding[2];
1821	} aMemMappingLocks[3]; /* 0x80 LB 0x30 */
1822
1823	/** Bounce buffer info.
1824	* This runs in parallel to aMemMappings. */
1825	struct
1826	{
1827	/** The physical address of the first byte. */
1828	RTGCPHYS GCPhysFirst;
1829	/** The physical address of the second page. */
1830	RTGCPHYS GCPhysSecond;
1831	/** The number of bytes in the first page. */
1832	uint16_t cbFirst;
1833	/** The number of bytes in the second page. */
1834	uint16_t cbSecond;
1835	/** Whether it's unassigned memory. */
1836	bool fUnassigned;
1837	/** Explicit alignment padding. */
1838	bool afAlignment5[3];
1839	} aMemBbMappings[3]; /* 0xb0 LB 0x48 */
1840
1841	/** The flags of the current exception / interrupt. */
1842	uint32_t fCurXcpt; /* 0xf8 */
1843	/** The current exception / interrupt. */
1844	uint8_t uCurXcpt; /* 0xfc */
1845	/** Exception / interrupt recursion depth. */
1846	int8_t cXcptRecursions; /* 0xfb */
1847
1848	/** The next unused mapping index.
1849	* @todo try find room for this up with cActiveMappings. */
1850	uint8_t iNextMapping; /* 0xfd */
1851	uint8_t abAlignment7[1];
1852
1853	/** Bounce buffer storage.
1854	* This runs in parallel to aMemMappings and aMemBbMappings. */
1855	struct
1856	{
1857	uint8_t ab[512];
1858	} aBounceBuffers[3]; /* 0x100 LB 0x600 */
1859
1860
1861	/** Pointer set jump buffer - ring-3 context. */
1862	R3PTRTYPE(jmp_buf *) pJmpBufR3;
1863	/** Pointer set jump buffer - ring-0 context. */
1864	R0PTRTYPE(jmp_buf *) pJmpBufR0;
1865
1866	/** @todo Should move this near @a fCurXcpt later. */
1867	/** The CR2 for the current exception / interrupt. */
1868	uint64_t uCurXcptCr2;
1869	/** The error code for the current exception / interrupt. */
1870	uint32_t uCurXcptErr;
1871
1872	/** @name Statistics
1873	* @{ */
1874	/** The number of instructions we've executed. */
1875	uint32_t cInstructions;
1876	/** The number of potential exits. */
1877	uint32_t cPotentialExits;
1878	/** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
1879	uint32_t cRetInstrNotImplemented;
1880	/** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
1881	uint32_t cRetAspectNotImplemented;
1882	/** Counts informational statuses returned (other than VINF_SUCCESS). */
1883	uint32_t cRetInfStatuses;
1884	/** Counts other error statuses returned. */
1885	uint32_t cRetErrStatuses;
1886	/** Number of times rcPassUp has been used. */
1887	uint32_t cRetPassUpStatus;
1888	/** Number of times RZ left with instruction commit pending for ring-3. */
1889	uint32_t cPendingCommit;
1890	/** Number of misaligned (host sense) atomic instruction accesses. */
1891	uint32_t cMisalignedAtomics;
1892	/** Number of long jumps. */
1893	uint32_t cLongJumps;
1894	/** @} */
1895
1896	/** @name Target CPU information.
1897	* @{ */
1898	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1899	/** The target CPU. */
1900	uint8_t uTargetCpu;
1901	#else
1902	uint8_t bTargetCpuPadding;
1903	#endif
1904	/** For selecting assembly works matching the target CPU EFLAGS behaviour, see
1905	* IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
1906	* native host support and the 2nd for when there is.
1907	*
1908	* The two values are typically indexed by a g_CpumHostFeatures bit.
1909	*
1910	* This is for instance used for the BSF & BSR instructions where AMD and
1911	* Intel CPUs produce different EFLAGS. */
1912	uint8_t aidxTargetCpuEflFlavour[2];
1913
1914	/** The CPU vendor. */
1915	CPUMCPUVENDOR enmCpuVendor;
1916	/** @} */
1917
1918	/** Counts RDMSR \#GP(0) LogRel(). */
1919	uint8_t cLogRelRdMsr;
1920	/** Counts WRMSR \#GP(0) LogRel(). */
1921	uint8_t cLogRelWrMsr;
1922	/** Alignment padding. */
1923	uint8_t abAlignment9[50];
1924
1925
1926	/** @name Recompiled Exection
1927	* @{ */
1928	/** Pointer to the current translation block.
1929	* This can either be one being executed or one being compiled. */
1930	R3PTRTYPE(PIEMTB) pCurTbR3;
1931	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
1932	/** Frame pointer for the last native TB to execute. */
1933	R3PTRTYPE(void *) pvTbFramePointerR3;
1934	#else
1935	R3PTRTYPE(void *) pvUnusedR3;
1936	#endif
1937	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1938	/** The saved host floating point control register (MXCSR on x86, FPCR on arm64)
1939	* needing restore when the TB finished, IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED indicates the TB
1940	* didn't modify it so we don't need to restore it. */
1941	# ifdef RT_ARCH_AMD64
1942	uint32_t uRegFpCtrl;
1943	/** Temporary copy of MXCSR for stmxcsr/ldmxcsr (so we don't have to fiddle with stack pointers). */
1944	uint32_t uRegMxcsrTmp;
1945	# elif defined(RT_ARCH_ARM64)
1946	uint64_t uRegFpCtrl;
1947	# else
1948	# error "Port me"
1949	# endif
1950	#else
1951	uint64_t u64Unused;
1952	#endif
1953	/** Pointer to the ring-3 TB cache for this EMT. */
1954	R3PTRTYPE(PIEMTBCACHE) pTbCacheR3;
1955	/** Pointer to the ring-3 TB lookup entry.
1956	* This either points to pTbLookupEntryDummyR3 or an actually lookuptable
1957	* entry, thus it can always safely be used w/o NULL checking. */
1958	R3PTRTYPE(PIEMTB *) ppTbLookupEntryR3;
1959	#if 0 /* unused */
1960	/** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
1961	* The TBs are based on physical addresses, so this is needed to correleated
1962	* RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
1963	uint64_t uCurTbStartPc;
1964	#endif
1965
1966	/** Number of threaded TBs executed. */
1967	uint64_t cTbExecThreaded;
1968	/** Number of native TBs executed. */
1969	uint64_t cTbExecNative;
1970
1971	/** The number of IRQ/FF checks till the next timer poll call. */
1972	uint32_t cTbsTillNextTimerPoll;
1973	/** The virtual sync time at the last timer poll call in milliseconds. */
1974	uint32_t msRecompilerPollNow;
1975	/** The virtual sync time at the last timer poll call in nanoseconds. */
1976	uint64_t nsRecompilerPollNow;
1977	/** The previous cTbsTillNextTimerPoll value. */
1978	uint32_t cTbsTillNextTimerPollPrev;
1979
1980	/** The current instruction number in a native TB.
1981	* This is set by code that may trigger an unexpected TB exit (throw/longjmp)
1982	* and will be picked up by the TB execution loop. Only used when
1983	* IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
1984	uint8_t idxTbCurInstr;
1985	/** @} */
1986
1987	/** @name Recompilation
1988	* @{ */
1989	/** Whether we need to check the opcode bytes for the current instruction.
1990	* This is set by a previous instruction if it modified memory or similar. */
1991	bool fTbCheckOpcodes;
1992	/** Indicates whether and how we just branched - IEMBRANCHED_F_XXX. */
1993	uint8_t fTbBranched;
1994	/** Set when GCPhysInstrBuf is updated because of a page crossing. */
1995	bool fTbCrossedPage;
1996	/** Whether to end the current TB. */
1997	bool fEndTb;
1998	/** Indicates that the current instruction is an STI. This is set by the
1999	* iemCImpl_sti code and subsequently cleared by the recompiler. */
2000	bool fTbCurInstrIsSti;
2001	/** Spaced reserved for recompiler data / alignment. */
2002	bool afRecompilerStuff1[1];
2003	/** Number of instructions before we need emit an IRQ check call again.
2004	* This helps making sure we don't execute too long w/o checking for
2005	* interrupts and immediately following instructions that may enable
2006	* interrupts (e.g. POPF, IRET, STI). With STI an additional hack is
2007	* required to make sure we check following the next instruction as well, see
2008	* fTbCurInstrIsSti. */
2009	uint8_t cInstrTillIrqCheck;
2010	/** The index of the last CheckIrq call during threaded recompilation. */
2011	uint16_t idxLastCheckIrqCallNo;
2012	/** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
2013	uint16_t cbOpcodesAllocated;
2014	/** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
2015	uint32_t uTbNativeRecompileAtUsedCount;
2016	/** The IEM_CIMPL_F_XXX mask for the current instruction. */
2017	uint32_t fTbCurInstr;
2018	/** The IEM_CIMPL_F_XXX mask for the previous instruction. */
2019	uint32_t fTbPrevInstr;
2020	/** Strict: Tracking skipped EFLAGS calculations. Any bits set here are
2021	* currently not up to date in EFLAGS. */
2022	uint32_t fSkippingEFlags;
2023	#if 0 /* unused */
2024	/** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set. */
2025	RTGCPHYS GCPhysInstrBufPrev;
2026	#endif
2027
2028	/** Fixed TB used for threaded recompilation.
2029	* This is allocated once with maxed-out sizes and re-used afterwards. */
2030	R3PTRTYPE(PIEMTB) pThrdCompileTbR3;
2031	/** Pointer to the ring-3 TB allocator for this EMT. */
2032	R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
2033	/** Pointer to the ring-3 executable memory allocator for this EMT. */
2034	R3PTRTYPE(struct IEMEXECMEMALLOCATOR *) pExecMemAllocatorR3;
2035	/** Pointer to the native recompiler state for ring-3. */
2036	R3PTRTYPE(struct IEMRECOMPILERSTATE *) pNativeRecompilerStateR3;
2037	/** Dummy entry for ppTbLookupEntryR3. */
2038	R3PTRTYPE(PIEMTB) pTbLookupEntryDummyR3;
2039	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2040	/** The debug code advances this register as if it was CPUMCTX::rip and we
2041	* didn't do delayed PC updating. When CPUMCTX::rip is finally updated,
2042	* the result is compared with this value. */
2043	uint64_t uPcUpdatingDebug;
2044	#elif defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
2045	/** The SSM handle used for saving threaded TBs for recompiler profiling. */
2046	R3PTRTYPE(PSSMHANDLE) pSsmThreadedTbsForProfiling;
2047	#else
2048	uint64_t u64Placeholder;
2049	#endif
2050	/**
2051	* Whether we should use the host instruction invalidation APIs of the
2052	* host OS or our own version of it (macOS). */
2053	uint8_t fHostICacheInvalidation;
2054	#define IEMNATIVE_ICACHE_F_USE_HOST_API UINT8_C(0x01) /*< Use the host API (macOS) instead of our code. /
2055	#define IEMNATIVE_ICACHE_F_END_WITH_ISH UINT8_C(0x02) /*< Whether to end with a ISH barrier (arm). /
2056	bool afRecompilerStuff2[7];
2057	/** @} */
2058
2059	/** Dummy TLB entry used for accesses to pages with databreakpoints. */
2060	IEMTLBENTRY DataBreakpointTlbe;
2061
2062	/** Threaded TB statistics: Times TB execution was broken off before reaching the end. */
2063	STAMCOUNTER StatTbThreadedExecBreaks;
2064	/** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
2065	STAMCOUNTER StatCheckIrqBreaks;
2066	/** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
2067	STAMCOUNTER StatCheckTimersBreaks;
2068	/** Statistics: Times BltIn_CheckMode breaks out of the TB. */
2069	STAMCOUNTER StatCheckModeBreaks;
2070	/** Threaded TB statistics: Times execution break on call with lookup entries. */
2071	STAMCOUNTER StatTbThreadedExecBreaksWithLookup;
2072	/** Threaded TB statistics: Times execution break on call without lookup entries. */
2073	STAMCOUNTER StatTbThreadedExecBreaksWithoutLookup;
2074	/** Statistics: Times a post jump target check missed and had to find new TB. */
2075	STAMCOUNTER StatCheckBranchMisses;
2076	/** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
2077	STAMCOUNTER StatCheckNeedCsLimChecking;
2078	/** Statistics: Times a loop was detected within a TB. */
2079	STAMCOUNTER StatTbLoopInTbDetected;
2080	/** Statistics: Times a loop back to the start of the TB was detected. */
2081	STAMCOUNTER StatTbLoopFullTbDetected;
2082	/** Statistics: Times a loop back to the start of the TB was detected, var 2. */
2083	STAMCOUNTER StatTbLoopFullTbDetected2;
2084	/** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
2085	STAMCOUNTER StatNativeExecMemInstrBufAllocFailed;
2086	/** Native TB statistics: Number of fully recompiled TBs. */
2087	STAMCOUNTER StatNativeFullyRecompiledTbs;
2088	/** TB statistics: Number of instructions per TB. */
2089	STAMPROFILE StatTbInstr;
2090	/** TB statistics: Number of TB lookup table entries per TB. */
2091	STAMPROFILE StatTbLookupEntries;
2092	/** Threaded TB statistics: Number of calls per TB. */
2093	STAMPROFILE StatTbThreadedCalls;
2094	/** Native TB statistics: Native code size per TB. */
2095	STAMPROFILE StatTbNativeCode;
2096	/** Native TB statistics: Profiling native recompilation. */
2097	STAMPROFILE StatNativeRecompilation;
2098	/** Native TB statistics: Number of calls per TB that were recompiled properly. */
2099	STAMPROFILE StatNativeCallsRecompiled;
2100	/** Native TB statistics: Number of threaded calls per TB that weren't recompiled. */
2101	STAMPROFILE StatNativeCallsThreaded;
2102	/** Native recompiled execution: TLB hits for data fetches. */
2103	STAMCOUNTER StatNativeTlbHitsForFetch;
2104	/** Native recompiled execution: TLB hits for data stores. */
2105	STAMCOUNTER StatNativeTlbHitsForStore;
2106	/** Native recompiled execution: TLB hits for stack accesses. */
2107	STAMCOUNTER StatNativeTlbHitsForStack;
2108	/** Native recompiled execution: TLB hits for mapped accesses. */
2109	STAMCOUNTER StatNativeTlbHitsForMapped;
2110	/** Native recompiled execution: Code TLB misses for new page. */
2111	STAMCOUNTER StatNativeCodeTlbMissesNewPage;
2112	/** Native recompiled execution: Code TLB hits for new page. */
2113	STAMCOUNTER StatNativeCodeTlbHitsForNewPage;
2114	/** Native recompiled execution: Code TLB misses for new page with offset. */
2115	STAMCOUNTER StatNativeCodeTlbMissesNewPageWithOffset;
2116	/** Native recompiled execution: Code TLB hits for new page with offset. */
2117	STAMCOUNTER StatNativeCodeTlbHitsForNewPageWithOffset;
2118
2119	/** Native recompiler: Number of calls to iemNativeRegAllocFindFree. */
2120	STAMCOUNTER StatNativeRegFindFree;
2121	/** Native recompiler: Number of times iemNativeRegAllocFindFree needed
2122	* to free a variable. */
2123	STAMCOUNTER StatNativeRegFindFreeVar;
2124	/** Native recompiler: Number of times iemNativeRegAllocFindFree did
2125	* not need to free any variables. */
2126	STAMCOUNTER StatNativeRegFindFreeNoVar;
2127	/** Native recompiler: Liveness info freed shadowed guest registers in
2128	* iemNativeRegAllocFindFree. */
2129	STAMCOUNTER StatNativeRegFindFreeLivenessUnshadowed;
2130	/** Native recompiler: Liveness info helped with the allocation in
2131	* iemNativeRegAllocFindFree. */
2132	STAMCOUNTER StatNativeRegFindFreeLivenessHelped;
2133
2134	/** Native recompiler: Number of times status flags calc has been skipped. */
2135	STAMCOUNTER StatNativeEflSkippedArithmetic;
2136	/** Native recompiler: Number of times status flags calc has been postponed. */
2137	STAMCOUNTER StatNativeEflPostponedArithmetic;
2138	/** Native recompiler: Total number instructions in this category. */
2139	STAMCOUNTER StatNativeEflTotalArithmetic;
2140
2141	/** Native recompiler: Number of times status flags calc has been skipped. */
2142	STAMCOUNTER StatNativeEflSkippedLogical;
2143	/** Native recompiler: Number of times status flags calc has been postponed. */
2144	STAMCOUNTER StatNativeEflPostponedLogical;
2145	/** Native recompiler: Total number instructions in this category. */
2146	STAMCOUNTER StatNativeEflTotalLogical;
2147
2148	/** Native recompiler: Number of times status flags calc has been skipped. */
2149	STAMCOUNTER StatNativeEflSkippedShift;
2150	/** Native recompiler: Number of times status flags calc has been postponed. */
2151	STAMCOUNTER StatNativeEflPostponedShift;
2152	/** Native recompiler: Total number instructions in this category. */
2153	STAMCOUNTER StatNativeEflTotalShift;
2154
2155	/** Native recompiler: Number of emits per postponement. */
2156	STAMPROFILE StatNativeEflPostponedEmits;
2157
2158	/** Native recompiler: Number of opportunities to skip EFLAGS.CF updating. */
2159	STAMCOUNTER StatNativeLivenessEflCfSkippable;
2160	/** Native recompiler: Number of opportunities to skip EFLAGS.PF updating. */
2161	STAMCOUNTER StatNativeLivenessEflPfSkippable;
2162	/** Native recompiler: Number of opportunities to skip EFLAGS.AF updating. */
2163	STAMCOUNTER StatNativeLivenessEflAfSkippable;
2164	/** Native recompiler: Number of opportunities to skip EFLAGS.ZF updating. */
2165	STAMCOUNTER StatNativeLivenessEflZfSkippable;
2166	/** Native recompiler: Number of opportunities to skip EFLAGS.SF updating. */
2167	STAMCOUNTER StatNativeLivenessEflSfSkippable;
2168	/** Native recompiler: Number of opportunities to skip EFLAGS.OF updating. */
2169	STAMCOUNTER StatNativeLivenessEflOfSkippable;
2170	/** Native recompiler: Number of required EFLAGS.CF updates. */
2171	STAMCOUNTER StatNativeLivenessEflCfRequired;
2172	/** Native recompiler: Number of required EFLAGS.PF updates. */
2173	STAMCOUNTER StatNativeLivenessEflPfRequired;
2174	/** Native recompiler: Number of required EFLAGS.AF updates. */
2175	STAMCOUNTER StatNativeLivenessEflAfRequired;
2176	/** Native recompiler: Number of required EFLAGS.ZF updates. */
2177	STAMCOUNTER StatNativeLivenessEflZfRequired;
2178	/** Native recompiler: Number of required EFLAGS.SF updates. */
2179	STAMCOUNTER StatNativeLivenessEflSfRequired;
2180	/** Native recompiler: Number of required EFLAGS.OF updates. */
2181	STAMCOUNTER StatNativeLivenessEflOfRequired;
2182	/** Native recompiler: Number of potentially delayable EFLAGS.CF updates. */
2183	STAMCOUNTER StatNativeLivenessEflCfDelayable;
2184	/** Native recompiler: Number of potentially delayable EFLAGS.PF updates. */
2185	STAMCOUNTER StatNativeLivenessEflPfDelayable;
2186	/** Native recompiler: Number of potentially delayable EFLAGS.AF updates. */
2187	STAMCOUNTER StatNativeLivenessEflAfDelayable;
2188	/** Native recompiler: Number of potentially delayable EFLAGS.ZF updates. */
2189	STAMCOUNTER StatNativeLivenessEflZfDelayable;
2190	/** Native recompiler: Number of potentially delayable EFLAGS.SF updates. */
2191	STAMCOUNTER StatNativeLivenessEflSfDelayable;
2192	/** Native recompiler: Number of potentially delayable EFLAGS.OF updates. */
2193	STAMCOUNTER StatNativeLivenessEflOfDelayable;
2194
2195	/** Native recompiler: Number of potential PC updates in total. */
2196	STAMCOUNTER StatNativePcUpdateTotal;
2197	/** Native recompiler: Number of PC updates which could be delayed. */
2198	STAMCOUNTER StatNativePcUpdateDelayed;
2199
2200	/** Native recompiler: Number of time we had complicated dirty shadow
2201	* register situations with the other branch in IEM_MC_ENDIF. */
2202	STAMCOUNTER StatNativeEndIfOtherBranchDirty;
2203
2204	/** Native recompiler: Number of calls to iemNativeSimdRegAllocFindFree. */
2205	STAMCOUNTER StatNativeSimdRegFindFree;
2206	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree needed
2207	* to free a variable. */
2208	STAMCOUNTER StatNativeSimdRegFindFreeVar;
2209	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree did
2210	* not need to free any variables. */
2211	STAMCOUNTER StatNativeSimdRegFindFreeNoVar;
2212	/** Native recompiler: Liveness info freed shadowed guest registers in
2213	* iemNativeSimdRegAllocFindFree. */
2214	STAMCOUNTER StatNativeSimdRegFindFreeLivenessUnshadowed;
2215	/** Native recompiler: Liveness info helped with the allocation in
2216	* iemNativeSimdRegAllocFindFree. */
2217	STAMCOUNTER StatNativeSimdRegFindFreeLivenessHelped;
2218
2219	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks. */
2220	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckPotential;
2221	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks. */
2222	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckPotential;
2223	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks. */
2224	STAMCOUNTER StatNativeMaybeSseXcptCheckPotential;
2225	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks. */
2226	STAMCOUNTER StatNativeMaybeAvxXcptCheckPotential;
2227
2228	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks omitted. */
2229	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckOmitted;
2230	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks omitted. */
2231	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckOmitted;
2232	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks omitted. */
2233	STAMCOUNTER StatNativeMaybeSseXcptCheckOmitted;
2234	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks omitted. */
2235	STAMCOUNTER StatNativeMaybeAvxXcptCheckOmitted;
2236
2237	/** Native recompiler: The TB finished executing completely without jumping to a an exit label.
2238	* Not availabe in release builds. */
2239	STAMCOUNTER StatNativeTbFinished;
2240	/** Native recompiler: The TB finished executing jumping to the ReturnBreak label. */
2241	STAMCOUNTER StatNativeTbExitReturnBreak;
2242	/** Native recompiler: The TB finished executing jumping to the ReturnBreakFF label. */
2243	STAMCOUNTER StatNativeTbExitReturnBreakFF;
2244	/** Native recompiler: The TB finished executing jumping to the ReturnWithFlags label. */
2245	STAMCOUNTER StatNativeTbExitReturnWithFlags;
2246	/** Native recompiler: The TB finished executing with other non-zero status. */
2247	STAMCOUNTER StatNativeTbExitReturnOtherStatus;
2248	/** Native recompiler: The TB finished executing via throw / long jump. */
2249	STAMCOUNTER StatNativeTbExitLongJump;
2250	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2251	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2252	STAMCOUNTER StatNativeTbExitDirectLinking1NoIrq;
2253	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2254	* label, but directly jumped to the next TB, scenario \#1 with IRQ checks. */
2255	STAMCOUNTER StatNativeTbExitDirectLinking1Irq;
2256	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2257	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2258	STAMCOUNTER StatNativeTbExitDirectLinking2NoIrq;
2259	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2260	* label, but directly jumped to the next TB, scenario \#2 with IRQ checks. */
2261	STAMCOUNTER StatNativeTbExitDirectLinking2Irq;
2262
2263	/** Native recompiler: The TB finished executing jumping to the RaiseDe label. */
2264	STAMCOUNTER StatNativeTbExitRaiseDe;
2265	/** Native recompiler: The TB finished executing jumping to the RaiseUd label. */
2266	STAMCOUNTER StatNativeTbExitRaiseUd;
2267	/** Native recompiler: The TB finished executing jumping to the RaiseSseRelated label. */
2268	STAMCOUNTER StatNativeTbExitRaiseSseRelated;
2269	/** Native recompiler: The TB finished executing jumping to the RaiseAvxRelated label. */
2270	STAMCOUNTER StatNativeTbExitRaiseAvxRelated;
2271	/** Native recompiler: The TB finished executing jumping to the RaiseSseAvxFpRelated label. */
2272	STAMCOUNTER StatNativeTbExitRaiseSseAvxFpRelated;
2273	/** Native recompiler: The TB finished executing jumping to the RaiseNm label. */
2274	STAMCOUNTER StatNativeTbExitRaiseNm;
2275	/** Native recompiler: The TB finished executing jumping to the RaiseGp0 label. */
2276	STAMCOUNTER StatNativeTbExitRaiseGp0;
2277	/** Native recompiler: The TB finished executing jumping to the RaiseMf label. */
2278	STAMCOUNTER StatNativeTbExitRaiseMf;
2279	/** Native recompiler: The TB finished executing jumping to the RaiseXf label. */
2280	STAMCOUNTER StatNativeTbExitRaiseXf;
2281	/** Native recompiler: The TB finished executing jumping to the ObsoleteTb label. */
2282	STAMCOUNTER StatNativeTbExitObsoleteTb;
2283
2284	/** Native recompiler: Number of full TB loops (jumps from end to start). */
2285	STAMCOUNTER StatNativeTbExitLoopFullTb;
2286
2287	/** Native recompiler: Failure situations with direct linking scenario \#1.
2288	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2289	* @{ */
2290	STAMCOUNTER StatNativeTbExitDirectLinking1NoTb;
2291	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchGCPhysPc;
2292	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchFlags;
2293	STAMCOUNTER StatNativeTbExitDirectLinking1PendingIrq;
2294	/** @} */
2295
2296	/** Native recompiler: Failure situations with direct linking scenario \#2.
2297	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2298	* @{ */
2299	STAMCOUNTER StatNativeTbExitDirectLinking2NoTb;
2300	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchGCPhysPc;
2301	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchFlags;
2302	STAMCOUNTER StatNativeTbExitDirectLinking2PendingIrq;
2303	/** @} */
2304
2305	/** iemMemMap and iemMemMapJmp statistics.
2306	* @{ */
2307	STAMCOUNTER StatMemMapJmp;
2308	STAMCOUNTER StatMemMapNoJmp;
2309	STAMCOUNTER StatMemBounceBufferCrossPage;
2310	STAMCOUNTER StatMemBounceBufferMapPhys;
2311	/** @} */
2312
2313	/** Timer polling statistics (debug only).
2314	* @{ */
2315	STAMPROFILE StatTimerPoll;
2316	STAMPROFILE StatTimerPollPoll;
2317	STAMPROFILE StatTimerPollRun;
2318	STAMCOUNTER StatTimerPollUnchanged;
2319	STAMCOUNTER StatTimerPollTiny;
2320	STAMCOUNTER StatTimerPollDefaultCalc;
2321	STAMCOUNTER StatTimerPollMax;
2322	STAMPROFILE StatTimerPollFactorDivision;
2323	STAMPROFILE StatTimerPollFactorMultiplication;
2324	/** @} */
2325
2326
2327	STAMCOUNTER aStatAdHoc[8];
2328
2329	#ifdef IEM_WITH_TLB_TRACE
2330	/uint64_t au64Padding[0];/
2331	#else
2332	uint64_t au64Padding[2];
2333	#endif
2334
2335	#ifdef IEM_WITH_TLB_TRACE
2336	/** The end (next) trace entry. */
2337	uint32_t idxTlbTraceEntry;
2338	/** Number of trace entries allocated expressed as a power of two. */
2339	uint32_t cTlbTraceEntriesShift;
2340	/** The trace entries. */
2341	PIEMTLBTRACEENTRY paTlbTraceEntries;
2342	#endif
2343
2344	/** Data TLB.
2345	* @remarks Must be 64-byte aligned. */
2346	IEMTLB DataTlb;
2347	/** Instruction TLB.
2348	* @remarks Must be 64-byte aligned. */
2349	IEMTLB CodeTlb;
2350
2351	/** Exception statistics. */
2352	STAMCOUNTER aStatXcpts[32];
2353	/** Interrupt statistics. */
2354	uint32_t aStatInts[256];
2355
2356	#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING) && !defined(IEM_WITHOUT_INSTRUCTION_STATS)
2357	/** Instruction statistics for ring-0/raw-mode. */
2358	IEMINSTRSTATS StatsRZ;
2359	/** Instruction statistics for ring-3. */
2360	IEMINSTRSTATS StatsR3;
2361	# ifdef VBOX_WITH_IEM_RECOMPILER
2362	/** Statistics per threaded function call.
2363	* Updated by both the threaded and native recompilers. */
2364	uint32_t acThreadedFuncStats[0x6000 /24576/];
2365	# endif
2366	#endif
2367	} IEMCPU;
2368	AssertCompileMemberOffset(IEMCPU, cActiveMappings, 0x4f);
2369	AssertCompileMemberAlignment(IEMCPU, aMemMappings, 16);
2370	AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
2371	AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
2372	AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
2373	AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
2374	AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
2375
2376	/** Pointer to the per-CPU IEM state. */
2377	typedef IEMCPU *PIEMCPU;
2378	/** Pointer to the const per-CPU IEM state. */
2379	typedef IEMCPU const *PCIEMCPU;
2380
2381	/** @def IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED
2382	* Value indicating the TB didn't modified the floating point control register.
2383	* @note Neither FPCR nor MXCSR accept this as a valid value (MXCSR is not fully populated,
2384	* FPCR has the upper 32-bit reserved), so this is safe. */
2385	#if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) \|\| defined(DOXYGEN_RUNNING)
2386	# ifdef RT_ARCH_AMD64
2387	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT32_MAX
2388	# elif defined(RT_ARCH_ARM64)
2389	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT64_MAX
2390	# else
2391	# error "Port me"
2392	# endif
2393	#endif
2394
2395	/** @def IEM_GET_CTX
2396	* Gets the guest CPU context for the calling EMT.
2397	* @returns PCPUMCTX
2398	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2399	*/
2400	#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
2401
2402	/** @def IEM_CTX_ASSERT
2403	* Asserts that the @a a_fExtrnMbz is present in the CPU context.
2404	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2405	* @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
2406	*/
2407	#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) \
2408	AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
2409	("fExtrn=%#RX64 & fExtrnMbz=%#RX64 -> %#RX64\n", \
2410	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz), (a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz) ))
2411
2412	/** @def IEM_CTX_IMPORT_RET
2413	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2414	*
2415	* Will call the keep to import the bits as needed.
2416	*
2417	* Returns on import failure.
2418	*
2419	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2420	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2421	*/
2422	#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
2423	do { \
2424	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2425	{ /* likely */ } \
2426	else \
2427	{ \
2428	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2429	AssertRCReturn(rcCtxImport, rcCtxImport); \
2430	} \
2431	} while (0)
2432
2433	/** @def IEM_CTX_IMPORT_NORET
2434	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2435	*
2436	* Will call the keep to import the bits as needed.
2437	*
2438	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2439	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2440	*/
2441	#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
2442	do { \
2443	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2444	{ /* likely */ } \
2445	else \
2446	{ \
2447	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2448	AssertLogRelRC(rcCtxImport); \
2449	} \
2450	} while (0)
2451
2452	/** @def IEM_CTX_IMPORT_JMP
2453	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2454	*
2455	* Will call the keep to import the bits as needed.
2456	*
2457	* Jumps on import failure.
2458	*
2459	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2460	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2461	*/
2462	#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
2463	do { \
2464	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2465	{ /* likely */ } \
2466	else \
2467	{ \
2468	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2469	AssertRCStmt(rcCtxImport, IEM_DO_LONGJMP(pVCpu, rcCtxImport)); \
2470	} \
2471	} while (0)
2472
2473
2474
2475	/** @def IEM_GET_TARGET_CPU
2476	* Gets the current IEMTARGETCPU value.
2477	* @returns IEMTARGETCPU value.
2478	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2479	*/
2480	#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
2481	# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
2482	#else
2483	# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
2484	#endif
2485
2486	/** @def IEM_GET_INSTR_LEN
2487	* Gets the instruction length.
2488	* @note x86 specific */
2489	#ifdef IEM_WITH_CODE_TLB
2490	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
2491	#else
2492	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
2493	#endif
2494
2495	/** @def IEM_TRY_SETJMP
2496	* Wrapper around setjmp / try, hiding all the ugly differences.
2497	*
2498	* @note Use with extreme care as this is a fragile macro.
2499	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2500	* @param a_rcTarget The variable that should receive the status code in case
2501	* of a longjmp/throw.
2502	*/
2503	/** @def IEM_TRY_SETJMP_AGAIN
2504	* For when setjmp / try is used again in the same variable scope as a previous
2505	* IEM_TRY_SETJMP invocation.
2506	*/
2507	/** @def IEM_CATCH_LONGJMP_BEGIN
2508	* Start wrapper for catch / setjmp-else.
2509	*
2510	* This will set up a scope.
2511	*
2512	* @note Use with extreme care as this is a fragile macro.
2513	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2514	* @param a_rcTarget The variable that should receive the status code in case
2515	* of a longjmp/throw.
2516	*/
2517	/** @def IEM_CATCH_LONGJMP_END
2518	* End wrapper for catch / setjmp-else.
2519	*
2520	* This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the
2521	* state.
2522	*
2523	* @note Use with extreme care as this is a fragile macro.
2524	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2525	*/
2526	#ifdef IEM_WITH_THROW_CATCH
2527	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2528	a_rcTarget = VINF_SUCCESS; \
2529	try
2530	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2531	IEM_TRY_SETJMP(a_pVCpu, a_rcTarget)
2532	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2533	catch (int rcThrown) \
2534	{ \
2535	a_rcTarget = rcThrown
2536	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2537	} \
2538	((void)0)
2539	#else /* !IEM_WITH_THROW_CATCH */
2540	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2541	jmp_buf JmpBuf; \
2542	jmp_buf * volatile pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2543	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2544	if ((rcStrict = setjmp(JmpBuf)) == 0)
2545	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2546	pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2547	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2548	if ((rcStrict = setjmp(JmpBuf)) == 0)
2549	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2550	else \
2551	{ \
2552	((void)0)
2553	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2554	} \
2555	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf
2556	#endif /* !IEM_WITH_THROW_CATCH */
2557
2558
2559	/**
2560	* Shared per-VM IEM data.
2561	*/
2562	typedef struct IEM
2563	{
2564	/** The VMX APIC-access page handler type. */
2565	PGMPHYSHANDLERTYPE hVmxApicAccessPage;
2566	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
2567	/** Set if the CPUID host call functionality is enabled. */
2568	bool fCpuIdHostCall;
2569	#endif
2570	} IEM;
2571
2572
2573
2574	/** @name IEM_ACCESS_XXX - Access details.
2575	* @{ */
2576	#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
2577	#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
2578	#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
2579	#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
2580	#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
2581	#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
2582	#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
2583	#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
2584	#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
2585	#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
2586	/** The writes are partial, so if initialize the bounce buffer with the
2587	* orignal RAM content. */
2588	#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
2589	/** Used in aMemMappings to indicate that the entry is bounce buffered. */
2590	#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
2591	/** Bounce buffer with ring-3 write pending, first page. */
2592	#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
2593	/** Bounce buffer with ring-3 write pending, second page. */
2594	#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
2595	/** Not locked, accessed via the TLB. */
2596	#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
2597	/** Atomic access.
2598	* This enables special alignment checks and the VINF_EM_EMULATE_SPLIT_LOCK
2599	* fallback for misaligned stuff. See @bugref{10547}. */
2600	#define IEM_ACCESS_ATOMIC UINT32_C(0x00002000)
2601	/** Valid bit mask. */
2602	#define IEM_ACCESS_VALID_MASK UINT32_C(0x00003fff)
2603	/** Shift count for the TLB flags (upper word). */
2604	#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
2605
2606	/** Atomic read+write data alias. */
2607	#define IEM_ACCESS_DATA_ATOMIC (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_ATOMIC)
2608	/** Read+write data alias. */
2609	#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2610	/** Write data alias. */
2611	#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2612	/** Read data alias. */
2613	#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA)
2614	/** Instruction fetch alias. */
2615	#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_WHAT_CODE)
2616	/** Stack write alias. */
2617	#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2618	/** Stack read alias. */
2619	#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_STACK)
2620	/** Stack read+write alias. */
2621	#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2622	/** Read system table alias. */
2623	#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_SYS)
2624	/** Read+write system table alias. */
2625	#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_SYS)
2626	/** @} */
2627
2628
2629	/** @def IEM_DECL_MSC_GUARD_IGNORE
2630	* Disables control flow guards checks inside a method and any function pointers
2631	* referenced by it. */
2632	#if defined(_MSC_VER) && !defined(IN_RING0)
2633	# define IEM_DECL_MSC_GUARD_IGNORE __declspec(guard(ignore))
2634	#else
2635	# define IEM_DECL_MSC_GUARD_IGNORE
2636	#endif
2637
2638	/** @def IEM_DECL_MSC_GUARD_NONE
2639	* Disables control flow guards checks inside a method and but continue track
2640	* function pointers references by it. */
2641	#if defined(_MSC_VER) && !defined(IN_RING0)
2642	# define IEM_DECL_MSC_GUARD_NONE __declspec(guard(nocf))
2643	#else
2644	# define IEM_DECL_MSC_GUARD_NONE
2645	#endif
2646
2647
2648	/** @def IEM_DECL_IMPL_TYPE
2649	* For typedef'ing an instruction implementation function.
2650	*
2651	* @param a_RetType The return type.
2652	* @param a_Name The name of the type.
2653	* @param a_ArgList The argument list enclosed in parentheses.
2654	*/
2655
2656	/** @def IEM_DECL_IMPL_DEF
2657	* For defining an instruction implementation function.
2658	*
2659	* @param a_RetType The return type.
2660	* @param a_Name The name of the type.
2661	* @param a_ArgList The argument list enclosed in parentheses.
2662	*/
2663
2664	#if defined(__GNUC__) && defined(RT_ARCH_X86)
2665	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2666	__attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
2667	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2668	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2669	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2670	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2671
2672	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2673	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2674	a_RetType (__fastcall a_Name) a_ArgList
2675	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2676	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2677	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2678	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2679
2680	#elif __cplusplus >= 201700 /* P0012R1 support */
2681	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2682	a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
2683	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2684	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2685	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2686	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2687
2688	#else
2689	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2690	a_RetType (VBOXCALL a_Name) a_ArgList
2691	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2692	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2693	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2694	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2695
2696	#endif
2697
2698
2699	/** @name C instruction implementations for anything slightly complicated.
2700	* @{ */
2701
2702	/**
2703	* For typedef'ing or declaring a C instruction implementation function taking
2704	* no extra arguments.
2705	*
2706	* @param a_Name The name of the type.
2707	*/
2708	# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2709	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2710	/**
2711	* For defining a C instruction implementation function taking no extra
2712	* arguments.
2713	*
2714	* @param a_Name The name of the function
2715	*/
2716	# define IEM_CIMPL_DEF_0(a_Name) \
2717	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2718	/**
2719	* Prototype version of IEM_CIMPL_DEF_0.
2720	*/
2721	# define IEM_CIMPL_PROTO_0(a_Name) \
2722	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2723	/**
2724	* For calling a C instruction implementation function taking no extra
2725	* arguments.
2726	*
2727	* This special call macro adds default arguments to the call and allow us to
2728	* change these later.
2729	*
2730	* @param a_fn The name of the function.
2731	*/
2732	# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2733
2734	/** Type for a C instruction implementation function taking no extra
2735	* arguments. */
2736	typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
2737	/** Function pointer type for a C instruction implementation function taking
2738	* no extra arguments. */
2739	typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
2740
2741	/**
2742	* For typedef'ing or declaring a C instruction implementation function taking
2743	* one extra argument.
2744	*
2745	* @param a_Name The name of the type.
2746	* @param a_Type0 The argument type.
2747	* @param a_Arg0 The argument name.
2748	*/
2749	# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2750	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2751	/**
2752	* For defining a C instruction implementation function taking one extra
2753	* argument.
2754	*
2755	* @param a_Name The name of the function
2756	* @param a_Type0 The argument type.
2757	* @param a_Arg0 The argument name.
2758	*/
2759	# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2760	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2761	/**
2762	* Prototype version of IEM_CIMPL_DEF_1.
2763	*/
2764	# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2765	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2766	/**
2767	* For calling a C instruction implementation function taking one extra
2768	* argument.
2769	*
2770	* This special call macro adds default arguments to the call and allow us to
2771	* change these later.
2772	*
2773	* @param a_fn The name of the function.
2774	* @param a0 The name of the 1st argument.
2775	*/
2776	# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2777
2778	/**
2779	* For typedef'ing or declaring a C instruction implementation function taking
2780	* two extra arguments.
2781	*
2782	* @param a_Name The name of the type.
2783	* @param a_Type0 The type of the 1st argument
2784	* @param a_Arg0 The name of the 1st argument.
2785	* @param a_Type1 The type of the 2nd argument.
2786	* @param a_Arg1 The name of the 2nd argument.
2787	*/
2788	# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2789	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2790	/**
2791	* For defining a C instruction implementation function taking two extra
2792	* arguments.
2793	*
2794	* @param a_Name The name of the function.
2795	* @param a_Type0 The type of the 1st argument
2796	* @param a_Arg0 The name of the 1st argument.
2797	* @param a_Type1 The type of the 2nd argument.
2798	* @param a_Arg1 The name of the 2nd argument.
2799	*/
2800	# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2801	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2802	/**
2803	* Prototype version of IEM_CIMPL_DEF_2.
2804	*/
2805	# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2806	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2807	/**
2808	* For calling a C instruction implementation function taking two extra
2809	* arguments.
2810	*
2811	* This special call macro adds default arguments to the call and allow us to
2812	* change these later.
2813	*
2814	* @param a_fn The name of the function.
2815	* @param a0 The name of the 1st argument.
2816	* @param a1 The name of the 2nd argument.
2817	*/
2818	# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
2819
2820	/**
2821	* For typedef'ing or declaring a C instruction implementation function taking
2822	* three extra arguments.
2823	*
2824	* @param a_Name The name of the type.
2825	* @param a_Type0 The type of the 1st argument
2826	* @param a_Arg0 The name of the 1st argument.
2827	* @param a_Type1 The type of the 2nd argument.
2828	* @param a_Arg1 The name of the 2nd argument.
2829	* @param a_Type2 The type of the 3rd argument.
2830	* @param a_Arg2 The name of the 3rd argument.
2831	*/
2832	# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2833	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2834	/**
2835	* For defining a C instruction implementation function taking three extra
2836	* arguments.
2837	*
2838	* @param a_Name The name of the function.
2839	* @param a_Type0 The type of the 1st argument
2840	* @param a_Arg0 The name of the 1st argument.
2841	* @param a_Type1 The type of the 2nd argument.
2842	* @param a_Arg1 The name of the 2nd argument.
2843	* @param a_Type2 The type of the 3rd argument.
2844	* @param a_Arg2 The name of the 3rd argument.
2845	*/
2846	# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2847	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2848	/**
2849	* Prototype version of IEM_CIMPL_DEF_3.
2850	*/
2851	# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2852	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2853	/**
2854	* For calling a C instruction implementation function taking three extra
2855	* arguments.
2856	*
2857	* This special call macro adds default arguments to the call and allow us to
2858	* change these later.
2859	*
2860	* @param a_fn The name of the function.
2861	* @param a0 The name of the 1st argument.
2862	* @param a1 The name of the 2nd argument.
2863	* @param a2 The name of the 3rd argument.
2864	*/
2865	# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
2866
2867
2868	/**
2869	* For typedef'ing or declaring a C instruction implementation function taking
2870	* four extra arguments.
2871	*
2872	* @param a_Name The name of the type.
2873	* @param a_Type0 The type of the 1st argument
2874	* @param a_Arg0 The name of the 1st argument.
2875	* @param a_Type1 The type of the 2nd argument.
2876	* @param a_Arg1 The name of the 2nd argument.
2877	* @param a_Type2 The type of the 3rd argument.
2878	* @param a_Arg2 The name of the 3rd argument.
2879	* @param a_Type3 The type of the 4th argument.
2880	* @param a_Arg3 The name of the 4th argument.
2881	*/
2882	# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2883	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
2884	/**
2885	* For defining a C instruction implementation function taking four extra
2886	* arguments.
2887	*
2888	* @param a_Name The name of the function.
2889	* @param a_Type0 The type of the 1st argument
2890	* @param a_Arg0 The name of the 1st argument.
2891	* @param a_Type1 The type of the 2nd argument.
2892	* @param a_Arg1 The name of the 2nd argument.
2893	* @param a_Type2 The type of the 3rd argument.
2894	* @param a_Arg2 The name of the 3rd argument.
2895	* @param a_Type3 The type of the 4th argument.
2896	* @param a_Arg3 The name of the 4th argument.
2897	*/
2898	# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2899	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2900	a_Type2 a_Arg2, a_Type3 a_Arg3))
2901	/**
2902	* Prototype version of IEM_CIMPL_DEF_4.
2903	*/
2904	# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2905	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2906	a_Type2 a_Arg2, a_Type3 a_Arg3))
2907	/**
2908	* For calling a C instruction implementation function taking four extra
2909	* arguments.
2910	*
2911	* This special call macro adds default arguments to the call and allow us to
2912	* change these later.
2913	*
2914	* @param a_fn The name of the function.
2915	* @param a0 The name of the 1st argument.
2916	* @param a1 The name of the 2nd argument.
2917	* @param a2 The name of the 3rd argument.
2918	* @param a3 The name of the 4th argument.
2919	*/
2920	# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
2921
2922
2923	/**
2924	* For typedef'ing or declaring a C instruction implementation function taking
2925	* five extra arguments.
2926	*
2927	* @param a_Name The name of the type.
2928	* @param a_Type0 The type of the 1st argument
2929	* @param a_Arg0 The name of the 1st argument.
2930	* @param a_Type1 The type of the 2nd argument.
2931	* @param a_Arg1 The name of the 2nd argument.
2932	* @param a_Type2 The type of the 3rd argument.
2933	* @param a_Arg2 The name of the 3rd argument.
2934	* @param a_Type3 The type of the 4th argument.
2935	* @param a_Arg3 The name of the 4th argument.
2936	* @param a_Type4 The type of the 5th argument.
2937	* @param a_Arg4 The name of the 5th argument.
2938	*/
2939	# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2940	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
2941	a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
2942	a_Type3 a_Arg3, a_Type4 a_Arg4))
2943	/**
2944	* For defining a C instruction implementation function taking five extra
2945	* arguments.
2946	*
2947	* @param a_Name The name of the function.
2948	* @param a_Type0 The type of the 1st argument
2949	* @param a_Arg0 The name of the 1st argument.
2950	* @param a_Type1 The type of the 2nd argument.
2951	* @param a_Arg1 The name of the 2nd argument.
2952	* @param a_Type2 The type of the 3rd argument.
2953	* @param a_Arg2 The name of the 3rd argument.
2954	* @param a_Type3 The type of the 4th argument.
2955	* @param a_Arg3 The name of the 4th argument.
2956	* @param a_Type4 The type of the 5th argument.
2957	* @param a_Arg4 The name of the 5th argument.
2958	*/
2959	# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2960	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2961	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2962	/**
2963	* Prototype version of IEM_CIMPL_DEF_5.
2964	*/
2965	# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2966	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2967	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2968	/**
2969	* For calling a C instruction implementation function taking five extra
2970	* arguments.
2971	*
2972	* This special call macro adds default arguments to the call and allow us to
2973	* change these later.
2974	*
2975	* @param a_fn The name of the function.
2976	* @param a0 The name of the 1st argument.
2977	* @param a1 The name of the 2nd argument.
2978	* @param a2 The name of the 3rd argument.
2979	* @param a3 The name of the 4th argument.
2980	* @param a4 The name of the 5th argument.
2981	*/
2982	# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2983
2984	/** @} */
2985
2986
2987	/** @name Opcode Decoder Function Types.
2988	* @{ */
2989
2990	/** @typedef PFNIEMOP
2991	* Pointer to an opcode decoder function.
2992	*/
2993
2994	/** @def FNIEMOP_DEF
2995	* Define an opcode decoder function.
2996	*
2997	* We're using macors for this so that adding and removing parameters as well as
2998	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
2999	*
3000	* @param a_Name The function name.
3001	*/
3002
3003	/** @typedef PFNIEMOPRM
3004	* Pointer to an opcode decoder function with RM byte.
3005	*/
3006
3007	/** @def FNIEMOPRM_DEF
3008	* Define an opcode decoder function with RM byte.
3009	*
3010	* We're using macors for this so that adding and removing parameters as well as
3011	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
3012	*
3013	* @param a_Name The function name.
3014	*/
3015
3016	#if defined(__GNUC__) && defined(RT_ARCH_X86)
3017	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
3018	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3019	# define FNIEMOP_DEF(a_Name) \
3020	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
3021	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3022	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3023	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3024	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3025
3026	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
3027	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
3028	typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3029	# define FNIEMOP_DEF(a_Name) \
3030	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3031	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3032	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3033	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3034	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3035
3036	#elif defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3037	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3038	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3039	# define FNIEMOP_DEF(a_Name) \
3040	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
3041	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3042	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3043	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3044	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3045
3046	#else
3047	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3048	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3049	# define FNIEMOP_DEF(a_Name) \
3050	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3051	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3052	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3053	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3054	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3055
3056	#endif
3057	#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
3058
3059	/**
3060	* Call an opcode decoder function.
3061	*
3062	* We're using macors for this so that adding and removing parameters can be
3063	* done as we please. See FNIEMOP_DEF.
3064	*/
3065	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
3066
3067	/**
3068	* Call a common opcode decoder function taking one extra argument.
3069	*
3070	* We're using macors for this so that adding and removing parameters can be
3071	* done as we please. See FNIEMOP_DEF_1.
3072	*/
3073	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
3074
3075	/**
3076	* Call a common opcode decoder function taking one extra argument.
3077	*
3078	* We're using macors for this so that adding and removing parameters can be
3079	* done as we please. See FNIEMOP_DEF_1.
3080	*/
3081	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
3082	/** @} */
3083
3084
3085	/** @name Misc Helpers
3086	* @{ */
3087
3088	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
3089	* due to GCC lacking knowledge about the value range of a switch. */
3090	#if RT_CPLUSPLUS_PREREQ(202000)
3091	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: [[unlikely]] AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3092	#else
3093	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3094	#endif
3095
3096	/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
3097	#if RT_CPLUSPLUS_PREREQ(202000)
3098	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: [[unlikely]] AssertFailedReturn(a_RetValue)
3099	#else
3100	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
3101	#endif
3102
3103	/**
3104	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3105	* occation.
3106	*/
3107	#ifdef LOG_ENABLED
3108	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3109	do { \
3110	/Log/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
3111	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3112	} while (0)
3113	#else
3114	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3115	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3116	#endif
3117
3118	/**
3119	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3120	* occation using the supplied logger statement.
3121	*
3122	* @param a_LoggerArgs What to log on failure.
3123	*/
3124	#ifdef LOG_ENABLED
3125	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3126	do { \
3127	LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
3128	/LogFunc(a_LoggerArgs);/ \
3129	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3130	} while (0)
3131	#else
3132	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3133	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3134	#endif
3135
3136
3137	/** @} */
3138
3139	uint32_t iemCalcExecDbgFlagsSlow(PVMCPUCC pVCpu);
3140	VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu);
3141
3142	/** @} */
3143
3144
3145	/** @name Memory access.
3146	* @{ */
3147	VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void *ppvMem, uint8_t pbUnmapInfo,
3148	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) RT_NOEXCEPT;
3149	VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void *ppvMem, uint8_t pbUnmapInfo, size_t cbMem,
3150	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) RT_NOEXCEPT;
3151	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3152	#ifndef IN_RING3
3153	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3154	#endif
3155	void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3156	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3157
3158	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3159	void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3160	void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3161	void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3162	void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3163	void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3164
3165	void iemTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3166	/** @} */
3167
3168	/*
3169	* Recompiler related stuff.
3170	*/
3171
3172	DECLHIDDEN(int) iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
3173
3174	DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
3175	uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
3176	void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree);
3177	DECLHIDDEN(void) iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
3178	void iemTbAllocatorProcessDelayedFrees(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator);
3179	void iemTbAllocatorFreeupNativeSpace(PVMCPUCC pVCpu, uint32_t cNeededInstrs);
3180	DECLHIDDEN(PIEMTBALLOCATOR) iemTbAllocatorFreeBulkStart(PVMCPUCC pVCpu);
3181	DECLHIDDEN(void) iemTbAllocatorFreeBulk(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator, PIEMTB pTb);
3182	DECLHIDDEN(const char ) iemTbFlagsToString(uint32_t fFlags, char pszBuf, size_t cbBuf) RT_NOEXCEPT;
3183	DECLHIDDEN(void) iemThreadedDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3184	#if defined(VBOX_WITH_IEM_NATIVE_RECOMPILER) && defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
3185	DECLHIDDEN(void) iemThreadedSaveTbForProfilingCleanup(PVMCPU pVCpu);
3186	#endif
3187
3188
3189	/** @todo FNIEMTHREADEDFUNC and friends may need more work... */
3190	#if defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3191	typedef VBOXSTRICTRC /__attribute__((__nothrow__))/ FNIEMTHREADEDFUNC(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3192	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3193	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3194	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3195	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3196	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3197
3198	#else
3199	typedef VBOXSTRICTRC (FNIEMTHREADEDFUNC)(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3200	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3201	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3202	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3203	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3204	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3205	#endif
3206
3207
3208	/* Native recompiler public bits: */
3209
3210	DECLHIDDEN(PIEMTB) iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb) RT_NOEXCEPT;
3211	DECLHIDDEN(void) iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3212	int iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT;
3213	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, PIEMNATIVEINSTR *ppaExec,
3214	struct IEMNATIVEPERCHUNKCTX const **ppChunkCtx) RT_NOEXCEPT;
3215	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq,
3216	PIEMNATIVEINSTR *ppaExec);
3217	DECLHIDDEN(void) iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3218	void iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3219	DECLASM(DECL_NO_RETURN(void)) iemNativeTbLongJmp(void *pvFramePointer, int rc) RT_NOEXCEPT;
3220	DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb);
3221	DECLHIDDEN(int) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk, struct IEMNATIVEPERCHUNKCTX const **ppCtx);
3222
3223	# ifdef VBOX_VMM_TARGET_X86
3224	# include "VMMAll/target-x86/IEMInternal-x86.h"
3225	# elif defined(VBOX_VMM_TARGET_ARMV8)
3226	//# include "VMMAll/target-armv8/IEMInternal-armv8.h"
3227	# endif
3228
3229	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3230
3231
3232	/** @} */
3233
3234	RT_C_DECLS_END
3235
3236	/* ASM-INC: %include "IEMInternalStruct.mac" */
3237
3238	#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3239

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 108296

Download in other formats: