IEMInternal.h@ 108702

Last change on this file since 108702 was 108702, checked in by vboxsync, 4 weeks ago
VMM/IEM: More ARM target work. jiraref:VBP-1598
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 157.5 KB

Line
1	/* $Id: IEMInternal.h 108702 2025-03-21 23:27:14Z vboxsync $ */
2	/** @file
3	* IEM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
29	#define VMM_INCLUDED_SRC_include_IEMInternal_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#ifndef RT_IN_ASSEMBLER
35	# include <VBox/vmm/cpum.h>
36	# include <VBox/vmm/iem.h>
37	# include <VBox/vmm/pgm.h>
38	# include <VBox/vmm/stam.h>
39	# include <VBox/param.h>
40
41	# include <iprt/setjmp-without-sigmask.h>
42	# include <iprt/list.h>
43	#endif /* !RT_IN_ASSEMBLER */
44
45
46	RT_C_DECLS_BEGIN
47
48
49	/** @defgroup grp_iem_int Internals
50	* @ingroup grp_iem
51	* @internal
52	* @{
53	*/
54
55	/* Make doxygen happy w/o overcomplicating the #if checks. */
56	#ifdef DOXYGEN_RUNNING
57	# define IEM_WITH_THROW_CATCH
58	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
59	#endif
60
61	/** For expanding symbol in slickedit and other products tagging and
62	* crossreferencing IEM symbols. */
63	#ifndef IEM_STATIC
64	# define IEM_STATIC static
65	#endif
66
67	/** @def IEM_WITH_THROW_CATCH
68	* Enables using C++ throw/catch as an alternative to setjmp/longjmp in user
69	* mode code.
70	*
71	* With GCC 11.3.1 and code TLB on linux, using throw/catch instead of
72	* setjmp/long resulted in bs2-test-1 running 3.00% faster and all but on test
73	* result value improving by more than 1%. (Best out of three.)
74	*
75	* With Visual C++ 2019 and code TLB on windows, using throw/catch instead of
76	* setjmp/long resulted in bs2-test-1 running 3.68% faster and all but some of
77	* the MMIO and CPUID tests ran noticeably faster. Variation is greater than on
78	* Linux, but it should be quite a bit faster for normal code.
79	*/
80	#if defined(__cplusplus) && defined(IN_RING3) && (defined(__GNUC__) \|\| defined(_MSC_VER)) /* ASM-NOINC-START */
81	# define IEM_WITH_THROW_CATCH
82	#endif /ASM-NOINC-END/
83
84	/** @def IEM_WITH_ADAPTIVE_TIMER_POLLING
85	* Enables the adaptive timer polling code.
86	*/
87	#if defined(DOXYGEN_RUNNING) \|\| 1
88	# define IEM_WITH_ADAPTIVE_TIMER_POLLING
89	#endif
90
91	/** @def IEM_WITH_INTRA_TB_JUMPS
92	* Enables loop-jumps within a TB (currently only to the first call).
93	*/
94	#if defined(DOXYGEN_RUNNING) \|\| 1
95	# define IEM_WITH_INTRA_TB_JUMPS
96	#endif
97
98	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
99	* Enables the delayed PC updating optimization (see @bugref{10373}).
100	*/
101	#if defined(DOXYGEN_RUNNING) \|\| 1
102	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING
103	#endif
104	/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
105	* Enabled delayed PC updating debugging code.
106	* This is an alternative to the ARM64-only IEMNATIVE_REG_FIXED_PC_DBG. */
107	#if defined(DOXYGEN_RUNNING) \|\| 0
108	# define IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
109	#endif
110
111	/** Enables access to even callee saved registers. */
112	/# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS/
113
114	#if defined(DOXYGEN_RUNNING) \|\| 1
115	/** @def IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
116	* Delay the writeback or dirty registers as long as possible. */
117	# define IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
118	#endif
119
120	/** @def IEM_WITH_TLB_STATISTICS
121	* Enables all TLB statistics. */
122	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
123	# define IEM_WITH_TLB_STATISTICS
124	#endif
125
126	/** @def IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
127	* Enable this to use native emitters for certain SIMD FP operations. */
128	#if 1 \|\| defined(DOXYGEN_RUNNING)
129	# define IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
130	#endif
131
132	/** @def VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
133	* Enable this to create a saved state file with the threaded translation
134	* blocks fed to the native recompiler on VCPU \#0. The resulting file can
135	* then be fed into the native recompiler for code profiling purposes.
136	* This is not a feature that should be normally be enabled! */
137	#if 0 \|\| defined(DOXYGEN_RUNNING)
138	# define VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
139	#endif
140
141	/** @def VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
142	* Enables a quicker alternative to throw/longjmp for IEM_DO_LONGJMP when
143	* executing native translation blocks.
144	*
145	* This exploits the fact that we save all non-volatile registers in the TB
146	* prologue and thus just need to do the same as the TB epilogue to get the
147	* effect of a longjmp/throw. Since MSC marks XMM6 thru XMM15 as
148	* non-volatile (and does something even more crazy for ARM), this probably
149	* won't work reliably on Windows. */
150	#ifdef RT_ARCH_ARM64
151	# ifndef RT_OS_WINDOWS
152	# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
153	# endif
154	#endif
155	/* ASM-NOINC-START */
156	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
157	# if !defined(IN_RING3) \
158	\|\| !defined(VBOX_WITH_IEM_RECOMPILER) \
159	\|\| !defined(VBOX_WITH_IEM_NATIVE_RECOMPILER)
160	# undef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
161	# elif defined(RT_OS_WINDOWS)
162	# pragma message("VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP is not safe to use on windows")
163	# endif
164	#endif
165
166
167	/** @def IEM_DO_LONGJMP
168	*
169	* Wrapper around longjmp / throw.
170	*
171	* @param a_pVCpu The CPU handle.
172	* @param a_rc The status code jump back with / throw.
173	*/
174	#ifdef IEM_WITH_THROW_CATCH
175	# ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
176	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) do { \
177	if ((a_pVCpu)->iem.s.pvTbFramePointerR3) \
178	iemNativeTbLongJmp((a_pVCpu)->iem.s.pvTbFramePointerR3, (a_rc)); \
179	throw int(a_rc); \
180	} while (0)
181	# else
182	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) throw int(a_rc)
183	# endif
184	#else
185	# define IEM_DO_LONGJMP(a_pVCpu, a_rc) longjmp(*(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf), (a_rc))
186	#endif
187
188	/** For use with IEM function that may do a longjmp (when enabled).
189	*
190	* Visual C++ has trouble longjmp'ing from/over functions with the noexcept
191	* attribute. So, we indicate that function that may be part of a longjmp may
192	* throw "exceptions" and that the compiler should definitely not generate and
193	* std::terminate calling unwind code.
194	*
195	* Here is one example of this ending in std::terminate:
196	* @code{.txt}
197	00 00000041`cadfda10 00007ffc`5d5a1f9f ucrtbase!abort+0x4e
198	01 00000041`cadfda40 00007ffc`57af229a ucrtbase!terminate+0x1f
199	02 00000041`cadfda70 00007ffb`eec91030 VCRUNTIME140!__std_terminate+0xa [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\ehhelpers.cpp @ 192]
200	03 00000041`cadfdaa0 00007ffb`eec92c6d VCRUNTIME140_1!_CallSettingFrame+0x20 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\handlers.asm @ 50]
201	04 00000041`cadfdad0 00007ffb`eec93ae5 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToState+0x241 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 1085]
202	05 00000041`cadfdc00 00007ffb`eec92258 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToEmptyState+0x2d [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 218]
203	06 00000041`cadfdc30 00007ffb`eec940e9 VCRUNTIME140_1!__InternalCxxFrameHandler<__FrameHandler4>+0x194 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 304]
204	07 00000041`cadfdcd0 00007ffc`5f9f249f VCRUNTIME140_1!__CxxFrameHandler4+0xa9 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 290]
205	08 00000041`cadfdd40 00007ffc`5f980939 ntdll!RtlpExecuteHandlerForUnwind+0xf
206	09 00000041`cadfdd70 00007ffc`5f9a0edd ntdll!RtlUnwindEx+0x339
207	0a 00000041`cadfe490 00007ffc`57aff976 ntdll!RtlUnwind+0xcd
208	0b 00000041`cadfea00 00007ffb`e1b5de01 VCRUNTIME140!__longjmp_internal+0xe6 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\longjmp.asm @ 140]
209	0c (Inline Function) --------`-------- VBoxVMM!iemOpcodeGetNextU8SlowJmp+0x95 [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 1155]
210	0d 00000041`cadfea50 00007ffb`e1b60f6b VBoxVMM!iemOpcodeGetNextU8Jmp+0xc1 [L:\vbox-intern\src\VBox\VMM\include\IEMInline.h @ 402]
211	0e 00000041`cadfea90 00007ffb`e1cc6201 VBoxVMM!IEMExecForExits+0xdb [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 10185]
212	0f 00000041`cadfec70 00007ffb`e1d0df8d VBoxVMM!EMHistoryExec+0x4f1 [L:\vbox-intern\src\VBox\VMM\VMMAll\EMAll.cpp @ 452]
213	10 00000041`cadfed60 00007ffb`e1d0d4c0 VBoxVMM!nemR3WinHandleExitCpuId+0x79d [L:\vbox-intern\src\VBox\VMM\VMMAll\NEMAllNativeTemplate-win.cpp.h @ 1829] @encode
214	@endcode
215	*
216	* @see https://developercommunity.visualstudio.com/t/fragile-behavior-of-longjmp-called-from-noexcept-f/1532859
217	*/
218	#if defined(_MSC_VER) \|\| defined(IEM_WITH_THROW_CATCH)
219	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT_EX(false)
220	#else
221	# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT
222	#endif
223	/* ASM-NOINC-END */
224
225
226	//#define IEM_WITH_CODE_TLB // - work in progress
227	//#define IEM_WITH_DATA_TLB // - work in progress
228
229
230	/** @def IEM_USE_UNALIGNED_DATA_ACCESS
231	* Use unaligned accesses instead of elaborate byte assembly. */
232	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86) \|\| defined(DOXYGEN_RUNNING) /ASM-NOINC/
233	# define IEM_USE_UNALIGNED_DATA_ACCESS
234	#endif /ASM-NOINC/
235
236	//#define IEM_LOG_MEMORY_WRITES
237
238
239	/** @def IEM_CFG_TARGET_CPU
240	* The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
241	*
242	* By default we allow this to be configured by the user via the
243	* CPUM/GuestCpuName config string, but this comes at a slight cost during
244	* decoding. So, for applications of this code where there is no need to
245	* be dynamic wrt target CPU, just modify this define.
246	*/
247	#if !defined(IEM_CFG_TARGET_CPU) \|\| defined(DOXYGEN_RUNNING)
248	# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
249	#endif
250
251
252	/*
253	* X86 config.
254	*/
255
256	#define IEM_IMPLEMENTS_TASKSWITCH
257
258	/** @def IEM_WITH_3DNOW
259	* Includes the 3DNow decoding. */
260	#if !defined(IEM_WITH_3DNOW) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
261	# ifndef IEM_WITHOUT_3DNOW
262	# define IEM_WITH_3DNOW
263	# endif
264	#endif
265
266	/** @def IEM_WITH_THREE_0F_38
267	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
268	#if !defined(IEM_WITH_THREE_0F_38) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
269	# ifndef IEM_WITHOUT_THREE_0F_38
270	# define IEM_WITH_THREE_0F_38
271	# endif
272	#endif
273
274	/** @def IEM_WITH_THREE_0F_3A
275	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
276	#if !defined(IEM_WITH_THREE_0F_3A) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
277	# ifndef IEM_WITHOUT_THREE_0F_3A
278	# define IEM_WITH_THREE_0F_3A
279	# endif
280	#endif
281
282	/** @def IEM_WITH_VEX
283	* Includes the VEX decoding. */
284	#if !defined(IEM_WITH_VEX) \|\| defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
285	# ifndef IEM_WITHOUT_VEX
286	# define IEM_WITH_VEX
287	# endif
288	#endif
289
290
291	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
292
293	# if !defined(IEM_WITHOUT_INSTRUCTION_STATS) && !defined(DOXYGEN_RUNNING)
294	/** Instruction statistics. */
295	typedef struct IEMINSTRSTATS
296	{
297	# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
298	# include "IEMInstructionStatisticsTmpl.h"
299	# undef IEM_DO_INSTR_STAT
300	} IEMINSTRSTATS;
301	#else
302	struct IEMINSTRSTATS;
303	typedef struct IEMINSTRSTATS IEMINSTRSTATS;
304	#endif
305	/** Pointer to IEM instruction statistics. */
306	typedef IEMINSTRSTATS *PIEMINSTRSTATS;
307
308
309	/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
310	* @{ */
311	#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /*< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. /
312	#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /*< Intel EFLAGS result. /
313	#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /*< AMD EFLAGS result /
314	#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /*< Reserved/dummy entry slot that's the same as 0. /
315	#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /*< For masking the index before use. /
316	/** Selects the right variant from a_aArray.
317	* pVCpu is implicit in the caller context. */
318	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
319	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
320	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
321	* be used because the host CPU does not support the operation. */
322	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
323	(a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
324	/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
325	* array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
326	* into the two.
327	* @sa IEM_SELECT_NATIVE_OR_FALLBACK */
328	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
329	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
330	(a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
331	#else
332	# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
333	(a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
334	#endif
335	/** @} */
336
337	/**
338	* Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
339	* indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
340	*
341	* On non-x86 hosts, this will shortcut to the fallback w/o checking the
342	* indicator.
343	*
344	* @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
345	*/
346	#if (defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
347	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
348	(g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
349	#else
350	# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
351	#endif
352
353	/** @name Helpers for passing C++ template arguments to an
354	* IEM_MC_NATIVE_EMIT_3/4/5 style macro.
355	* @{
356	*/
357	#define IEM_TEMPL_ARG_1(a1) <a1>
358	#define IEM_TEMPL_ARG_2(a1, a2) <a1,a2>
359	#define IEM_TEMPL_ARG_3(a1, a2, a3) <a1,a2,a3>
360	/** @} */
361
362
363	/**
364	* IEM TLB entry.
365	*
366	* Lookup assembly:
367	* @code{.asm}
368	; Calculate tag.
369	mov rax, [VA]
370	shl rax, 16
371	shr rax, 16 + X86_PAGE_SHIFT
372	or rax, [uTlbRevision]
373
374	; Do indexing.
375	movzx ecx, al
376	lea rcx, [pTlbEntries + rcx]
377
378	; Check tag.
379	cmp [rcx + IEMTLBENTRY.uTag], rax
380	jne .TlbMiss
381
382	; Check access.
383	mov rax, ACCESS_FLAGS \| MAPPING_R3_NOT_VALID \| 0xffffff00
384	and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
385	cmp rax, [uTlbPhysRev]
386	jne .TlbMiss
387
388	; Calc address and we're done.
389	mov eax, X86_PAGE_OFFSET_MASK
390	and eax, [VA]
391	or rax, [rcx + IEMTLBENTRY.pMappingR3]
392	%ifdef VBOX_WITH_STATISTICS
393	inc qword [cTlbHits]
394	%endif
395	jmp .Done
396
397	.TlbMiss:
398	mov r8d, ACCESS_FLAGS
399	mov rdx, [VA]
400	mov rcx, [pVCpu]
401	call iemTlbTypeMiss
402	.Done:
403
404	@endcode
405	*
406	*/
407	typedef struct IEMTLBENTRY
408	{
409	/** The TLB entry tag.
410	* Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
411	* is ASSUMING a virtual address width of 48 bits.
412	*
413	* Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
414	*
415	* The TLB lookup code uses the current TLB revision, which won't ever be zero,
416	* enabling an extremely cheap TLB invalidation most of the time. When the TLB
417	* revision wraps around though, the tags needs to be zeroed.
418	*
419	* @note Try use SHRD instruction? After seeing
420	* https://gmplib.org/~tege/x86-timing.pdf, maybe not.
421	*
422	* @todo This will need to be reorganized for 57-bit wide virtual address and
423	* PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
424	* have to move the TLB entry versioning entirely to the
425	* fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
426	* 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
427	* consumed by PCID and ASID (12 + 6 = 18).
428	* Update: Put the PCID + ASID in fFlagsAndPhysRev; that doesn't solve
429	* the 57-bit problem, though.
430	*/
431	uint64_t uTag;
432	/** Access flags and physical TLB revision.
433	*
434	* @x86
435	* @parblock
436	* - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
437	* - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
438	* - Bit 2 - page tables - not user (complemented X86_PTE_US).
439	* - Bit 3 - pgm phys/virt - not directly writable.
440	* - Bit 4 - pgm phys page - not directly readable.
441	* - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
442	* - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
443	* - Bit 7 - page tables - large page.
444	* - Bit 8 - tlb entry - pMappingR3 member not valid.
445	* - Bit 9 - phys - Unassigned memory.
446	* - Bit 10 - phys - Code page.
447	* - Bits 63:11 - phys - Physical TLB revision number.
448	*
449	* We're using complemented bit meanings here because it makes it easy to check
450	* whether special action is required. For instance a user mode write access
451	* would do a "TEST fFlags, (X86_PTE_RW \| X86_PTE_US \| X86_PTE_D)" and a
452	* non-zero result would mean special handling needed because either it wasn't
453	* writable, or it wasn't user, or the page wasn't dirty. A user mode read
454	* access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
455	* need to check any PTE flag.
456	* @endparblock
457	*
458	* @arm
459	* @parblock
460	* - Bit 0 - stage 1+2 - not unprivileged read accessible.
461	* - Bit 1 - stage 1+2 - not unprivileged write accessible.
462	* - Bit 2 - stage 1+2 - not unprivileged execute accessible.
463	* - Bit 3 - stage 1+2 - not unprivileged guarded control stack accessible.
464	* - Bit 4 - stage 1+2 - not privileged readable accessible.
465	* - Bit 5 - stage 1+2 - not privileged writable accessible.
466	* - Bit 6 - stage 1+2 - not privileged executable accessible.
467	* - Bit 7 - stage 1+2 - not privileged guarded control stack accessible.
468	* - Bit 8 - stage 2 - no limited write access (?).
469	* - Bit 9 - stage 2 - TopLevel0 (?)
470	* - Bit 10 - stage 2 - TopLevel1 (?)
471	* - Bit 11 - stage 1+2 leaf - not dirty.
472	* - Bit 12 - stage 1+2 leaf - alternate MECID (AMEC).
473	* - Bit 13 - pgm phys page - not directly readable.
474	* - Bit 14 - pgm phys/virt - not directly writable.
475	* - Bit 15 - tlb entry - pMappingR3 member not valid.
476	* - Bit 16 - phys - Unassigned memory.
477	* - Bit 17 - phys - Code page.
478	* - Bit 18 - stage 1 leaf - NS (not-secure).
479	* - Bit 19 - stage 1 leaf - NSE (root).
480	* - Bits 20:21 - stage 1+2 - Page size.
481	* - Bits 37:22 - stage 1 reg - Address space ID (ASID).
482	* - Bits 53:38 - stage 2 reg - Virtual Machine ID (VMID).
483	* - Bits 63:54 - tlb entry - physical TLB revision number.
484	*
485	* The ASIDs and VMIDs are kept with the physical TLB revision number, so
486	* there is no extra overhead there. How the NSE:NS stuff will be handled
487	* is a question for later.
488	*
489	* The above is a preliminary sketch...
490	* @endparblock
491	*
492	* @todo arm64: Not sure if we can combine the stage 1 and 2 AMEC bits,
493	* but hope so... Doubt we'll be needing this any time soon.
494	*/
495	uint64_t fFlagsAndPhysRev;
496	/** The guest physical page address. */
497	uint64_t GCPhys;
498	/** Pointer to the ring-3 mapping. */
499	R3PTRTYPE(uint8_t *) pbMappingR3;
500	#if HC_ARCH_BITS == 32
501	uint32_t u32Padding1;
502	#endif
503	} IEMTLBENTRY;
504	AssertCompileSize(IEMTLBENTRY, 32);
505	/** Pointer to an IEM TLB entry. */
506	typedef IEMTLBENTRY *PIEMTLBENTRY;
507	/** Pointer to a const IEM TLB entry. */
508	typedef IEMTLBENTRY const *PCIEMTLBENTRY;
509
510	/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
511	* @{ */
512	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
513	# define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /*< Page tables: Not executable. /
514	# define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /*< Page tables: Not writable. /
515	# define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /*< Page tables: Not user accessible (supervisor only). /
516	# define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /*< Phys page: Not writable (access handler, ROM, whatever). /
517	# define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /*< Phys page: Not readable (MMIO / access handler, ROM) /
518	# define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /*< Phys tables: Not accessed (need to be marked accessed). /
519	# define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /*< Page tables: Not dirty (needs to be made dirty on write). /
520	# define IEMTLBE_F_PT_LARGE_PAGE RT_BIT_64(7) /*< Page tables: Large 2 or 4 MiB page (for flushing). /
521	# define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(8) /*< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. /
522	# define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(9) /*< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). /
523	# define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(10) /*< Phys page: Code page. /
524	# define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffff800) /*< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR /
525	#endif
526	#if defined(VBOX_VMM_TARGET_ARMV8) \|\| defined(DOXYGEN_RUNNING)
527	# define IEMTLBE_F_EFF_U_NO_READ RT_BIT_64(0) /*< Stage 1+2: No unprivileged read access. /
528	# define IEMTLBE_F_EFF_U_NO_WRITE RT_BIT_64(1) /*< Stage 1+2: No unprivileged write access. /
529	# define IEMTLBE_F_EFF_U_NO_EXEC RT_BIT_64(2) /*< Stage 1+2: No unprivileged execute access. /
530	# define IEMTLBE_F_EFF_U_NO_GCS RT_BIT_64(3) /*< Stage 1+2: No unprivileged guard control stack access. /
531	# define IEMTLBE_F_EFF_P_NO_READ RT_BIT_64(4) /*< Stage 1+2: No privileged read access. /
532	# define IEMTLBE_F_EFF_P_NO_WRITE RT_BIT_64(5) /*< Stage 1+2: No privileged write access. /
533	# define IEMTLBE_F_EFF_P_NO_EXEC RT_BIT_64(6) /*< Stage 1+2: No privileged execute access. /
534	# define IEMTLBE_F_EFF_P_NO_GCS RT_BIT_64(7) /*< Stage 1+2: No privileged guard control stack access. /
535	# define IEMTLBE_F_S2_NO_LIM_WRITE RT_BIT_64(8) /*< Stage 2: No limited write access. /
536	# define IEMTLBE_F_S2_NO_LIM_WRITE_SHIFT 8 /*< @see IEMTLBE_F_S2_NO_LIM_WRITE /
537	# define IEMTLBE_F_S2_TL0 RT_BIT_64(9) /*< Stage 2: TopLevel0. /
538	# define IEMTLBE_F_S2_TL1 RT_BIT_64(10) /*< Stage 2: TopLevel1. /
539	# define IEMTLBE_F_EFF_NO_DIRTY RT_BIT_64(11) /*< Stage 1+2: Not dirty. /
540	# define IEMTLBE_F_EFF_AMEC RT_BIT_64(12) /*< Stage 1+2: Alternative MECID. /
541	# define IEMTLBE_F_PG_NO_READ RT_BIT_64(13) /*< Phys page: Not readable (MMIO / access handler, ROM) /
542	# define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(14) /*< Phys page: Not writable (access handler, ROM, whatever). /
543	# define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(15) /*< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. /
544	# define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(16) /*< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). /
545	# define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(17) /*< Phys page: Code page. /
546	# define IEMTLBE_F_S1_NS RT_BIT_64(18) /*< Stage 1: Non-secure bit. /
547	# define IEMTLBE_F_S1_NSE RT_BIT_64(19) /*< Stage 1: Non-secure extension/whatever bit. /
548	# define IEMTLBE_F_EFF_SIZE_MASK UINT64(0x300000) /*< Stage 1+2: Page size. @todo may need separate bits for each stage since they may use different page sizes. Or perhaps a single bit suffices? /
549	# define IEMTLBE_F_EFF_SIZE_L3 UINT64(0x000000) /*< Stage 1+2: Smallest page size. /
550	# define IEMTLBE_F_EFF_SIZE_L2 UINT64(0x100000) /*< Stage 1+2: Level 2 block. /
551	# define IEMTLBE_F_EFF_SIZE_L1 UINT64(0x200000) /*< Stage 1+2: Level 1 block. /
552	# define IEMTLBE_F_EFF_SIZE_L0 UINT64(0x300000) /*< Stage 1+2: Level 0 block. /
553	# define IEMTLBE_F_EFF_SIZE_SHIFT 20 /*< @see IEMTLBE_F_EFF_SIZE_MASK /
554	# define IEMTLBE_F_S1_ASID (UINT64_C(0xffff) << 22) /*< Stage 1: Address space ID (from stage 1 root register). /
555	# define IEMTLBE_F_S1_ASID_SHIFT 22 /*< @see IEMTLBE_F_S1_ASID /
556	# define IEMTLBE_F_S2_VMID (UINT64_C(0xffff) << 38) /*< Stage 2: Virtual machine ID (from stage 2 root register). /
557	# define IEMTLBE_F_S2_VMID_SHIFT 38 /*< @see IEMTLBE_F_S2_VMID /
558	# ifndef DOXYGEN_RUNNING
559	# define IEMTLBE_F_PHYS_REV UINT64_C(0xffc0000000000000) /*< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR /
560	# endif
561	#endif
562	/** @} */
563	/** The bits set by PGMPhysIemGCPhys2PtrNoLock. */
564	#define IEMTLBE_GCPHYS2PTR_MASK ( PGMIEMGCPHYS2PTR_F_NO_WRITE \
565	\| PGMIEMGCPHYS2PTR_F_NO_READ \
566	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \
567	\| PGMIEMGCPHYS2PTR_F_UNASSIGNED \
568	\| PGMIEMGCPHYS2PTR_F_CODE_PAGE \
569	\| IEMTLBE_F_PHYS_REV )
570	#if defined(VBOX_VMM_TARGET_X86) /// @todo \|\| defined(VBOX_VMM_TARGET_ARMV8)
571	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
572	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
573	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
574	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
575	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
576	# ifdef VBOX_VMM_TARGET_X86
577	AssertCompile(PGM_WALKINFO_BIG_PAGE == IEMTLBE_F_PT_LARGE_PAGE);
578	# endif
579	#endif
580
581
582	/** The TLB size (power of two).
583	* We initially chose 256 because that way we can obtain the result directly
584	* from a 8-bit register without an additional AND instruction.
585	* See also @bugref{10687}. */
586	#if defined(RT_ARCH_AMD64)
587	# define IEMTLB_ENTRY_COUNT 256
588	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 8
589	#else
590	# define IEMTLB_ENTRY_COUNT 8192
591	# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 13
592	#endif
593	AssertCompile(RT_BIT_32(IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO) == IEMTLB_ENTRY_COUNT);
594
595	/** TLB slot format spec (assumes uint32_t or unsigned value). */
596	#if IEMTLB_ENTRY_COUNT <= 0x100 / 2
597	# define IEMTLB_SLOT_FMT "%02x"
598	#elif IEMTLB_ENTRY_COUNT <= 0x1000 / 2
599	# define IEMTLB_SLOT_FMT "%03x"
600	#elif IEMTLB_ENTRY_COUNT <= 0x10000 / 2
601	# define IEMTLB_SLOT_FMT "%04x"
602	#else
603	# define IEMTLB_SLOT_FMT "%05x"
604	#endif
605
606	/** Enable the large page bitmap TLB optimization.
607	*
608	* The idea here is to avoid scanning the full 32 KB (2MB pages, 2*512 TLB
609	* entries) or 64 KB (4MB pages, 2*1024 TLB entries) worth of TLB entries during
610	* invlpg when large pages are used, and instead just scan 128 or 256 bytes of
611	* the bmLargePage bitmap to determin which TLB entires that might be containing
612	* large pages and actually require checking.
613	*
614	* There is a good posibility of false positives since we currently don't clear
615	* the bitmap when flushing the TLB, but it should help reduce the workload when
616	* the large pages aren't fully loaded into the TLB in their entirity...
617	*/
618	#define IEMTLB_WITH_LARGE_PAGE_BITMAP
619
620	/**
621	* An IEM TLB.
622	*
623	* We've got two of these, one for data and one for instructions.
624	*/
625	typedef struct IEMTLB
626	{
627	/** The non-global TLB revision.
628	* This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
629	* by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
630	* the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
631	* (The revision zero indicates an invalid TLB entry.)
632	*
633	* The initial value is choosen to cause an early wraparound. */
634	uint64_t uTlbRevision;
635	/** The TLB physical address revision - shadow of PGM variable.
636	*
637	* This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
638	* incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
639	* a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
640	* as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
641	*
642	* The initial value is choosen to cause an early wraparound.
643	*
644	* @note This is placed between the two TLB revisions because we
645	* load it in pair with one or the other on arm64. */
646	uint64_t volatile uTlbPhysRev;
647	/** The global TLB revision.
648	* Same as uTlbRevision, but only increased for global flushes. */
649	uint64_t uTlbRevisionGlobal;
650
651	/** Large page tag range.
652	*
653	* This is used to avoid scanning a large page's worth of TLB entries for each
654	* INVLPG instruction, and only to do so iff we've loaded any and when the
655	* address is in this range. This is kept up to date when we loading new TLB
656	* entries.
657	*/
658	struct LARGEPAGERANGE
659	{
660	/** The lowest large page address tag, UINT64_MAX if none. */
661	uint64_t uFirstTag;
662	/** The highest large page address tag (with offset mask part set), 0 if none. */
663	uint64_t uLastTag;
664	}
665	/** Large page range for non-global pages. */
666	NonGlobalLargePageRange,
667	/** Large page range for global pages. */
668	GlobalLargePageRange;
669	/** Number of non-global entries for large pages loaded since last TLB flush. */
670	uint32_t cTlbNonGlobalLargePageCurLoads;
671	/** Number of global entries for large pages loaded since last TLB flush. */
672	uint32_t cTlbGlobalLargePageCurLoads;
673
674	/* Statistics: */
675
676	/** TLB hits in IEMAll.cpp code (IEM_WITH_TLB_STATISTICS only; both).
677	* @note For the data TLB this is only used in iemMemMap and and for direct (i.e.
678	* not via safe read/write path) calls to iemMemMapJmp. */
679	uint64_t cTlbCoreHits;
680	/** Safe read/write TLB hits in iemMemMapJmp (IEM_WITH_TLB_STATISTICS
681	* only; data tlb only). */
682	uint64_t cTlbSafeHits;
683	/** TLB hits in IEMAllMemRWTmplInline.cpp.h (data + IEM_WITH_TLB_STATISTICS only). */
684	uint64_t cTlbInlineCodeHits;
685
686	/** TLB misses in IEMAll.cpp code (both).
687	* @note For the data TLB this is only used in iemMemMap and for direct (i.e.
688	* not via safe read/write path) calls to iemMemMapJmp. So,
689	* for the data TLB this more like 'other misses', while for the code
690	* TLB is all misses. */
691	uint64_t cTlbCoreMisses;
692	/** Subset of cTlbCoreMisses that results in PTE.G=1 loads (odd entries). */
693	uint64_t cTlbCoreGlobalLoads;
694	/** Safe read/write TLB misses in iemMemMapJmp (so data only). */
695	uint64_t cTlbSafeMisses;
696	/** Subset of cTlbSafeMisses that results in PTE.G=1 loads (odd entries). */
697	uint64_t cTlbSafeGlobalLoads;
698	/** Safe read path taken (data only). */
699	uint64_t cTlbSafeReadPath;
700	/** Safe write path taken (data only). */
701	uint64_t cTlbSafeWritePath;
702
703	/** @name Details for native code TLB misses.
704	* @note These counts are included in the above counters (cTlbSafeReadPath,
705	* cTlbSafeWritePath, cTlbInlineCodeHits).
706	* @{ */
707	/** TLB misses in native code due to tag mismatch. */
708	STAMCOUNTER cTlbNativeMissTag;
709	/** TLB misses in native code due to flags or physical revision mismatch. */
710	STAMCOUNTER cTlbNativeMissFlagsAndPhysRev;
711	/** TLB misses in native code due to misaligned access. */
712	STAMCOUNTER cTlbNativeMissAlignment;
713	/** TLB misses in native code due to cross page access. */
714	uint32_t cTlbNativeMissCrossPage;
715	/** TLB misses in native code due to non-canonical address. */
716	uint32_t cTlbNativeMissNonCanonical;
717	/** @} */
718
719	/** Slow read path (code only). */
720	uint32_t cTlbSlowCodeReadPath;
721
722	/** Regular TLB flush count. */
723	uint32_t cTlsFlushes;
724	/** Global TLB flush count. */
725	uint32_t cTlsGlobalFlushes;
726	/** Revision rollovers. */
727	uint32_t cTlbRevisionRollovers;
728	/** Physical revision flushes. */
729	uint32_t cTlbPhysRevFlushes;
730	/** Physical revision rollovers. */
731	uint32_t cTlbPhysRevRollovers;
732
733	/** Number of INVLPG (and similar) operations. */
734	uint32_t cTlbInvlPg;
735	/** Subset of cTlbInvlPg that involved non-global large pages. */
736	uint32_t cTlbInvlPgLargeNonGlobal;
737	/** Subset of cTlbInvlPg that involved global large pages. */
738	uint32_t cTlbInvlPgLargeGlobal;
739
740	uint32_t au32Padding[13];
741
742	/** The TLB entries.
743	* Even entries are for PTE.G=0 and uses uTlbRevision.
744	* Odd entries are for PTE.G=1 and uses uTlbRevisionGlobal. */
745	IEMTLBENTRY aEntries[IEMTLB_ENTRY_COUNT * 2];
746	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
747	/** Bitmap tracking TLB entries for large pages.
748	* This duplicates IEMTLBE_F_PT_LARGE_PAGE for each TLB entry. */
749	uint64_t bmLargePage[IEMTLB_ENTRY_COUNT * 2 / 64];
750	#endif
751	} IEMTLB;
752	AssertCompileSizeAlignment(IEMTLB, 64);
753	#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
754	AssertCompile(IEMTLB_ENTRY_COUNT >= 32 /* bmLargePage ASSUMPTION */);
755	#endif
756	/** The width (in bits) of the address portion of the TLB tag. */
757	#define IEMTLB_TAG_ADDR_WIDTH 36
758	/** IEMTLB::uTlbRevision increment. */
759	#define IEMTLB_REVISION_INCR RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH)
760	/** IEMTLB::uTlbRevision mask. */
761	#define IEMTLB_REVISION_MASK (~(RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH) - 1))
762
763	/** IEMTLB::uTlbPhysRev increment.
764	* @sa IEMTLBE_F_PHYS_REV */
765	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
766	# define IEMTLB_PHYS_REV_INCR RT_BIT_64(11)
767	#elif defined(VBOX_VMM_TARGET_ARMV8)
768	# define IEMTLB_PHYS_REV_INCR RT_BIT_64(54)
769	#endif
770	#ifdef IEMTLBE_F_PHYS_REV
771	AssertCompile(IEMTLBE_F_PHYS_REV == ~(IEMTLB_PHYS_REV_INCR - 1U));
772	#endif
773
774	/**
775	* Calculates the TLB tag for a virtual address but without TLB revision.
776	* @returns Tag value for indexing and comparing with IEMTLB::uTag.
777	* @param a_GCPtr The virtual address. Must be RTGCPTR or same size or
778	* the clearing of the top 16 bits won't work (if 32-bit
779	* we'll end up with mostly zeros).
780	*/
781	#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
782	/**
783	* Converts a TLB tag value into a even TLB index.
784	* @returns Index into IEMTLB::aEntries.
785	* @param a_uTag Value returned by IEMTLB_CALC_TAG.
786	*/
787	#if IEMTLB_ENTRY_COUNT == 256
788	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( (uint8_t)(a_uTag) * 2U )
789	#else
790	# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( ((a_uTag) & (IEMTLB_ENTRY_COUNT - 1U)) * 2U )
791	AssertCompile(RT_IS_POWER_OF_TWO(IEMTLB_ENTRY_COUNT));
792	#endif
793	/**
794	* Converts a TLB tag value into an even TLB index.
795	* @returns Pointer into IEMTLB::aEntries corresponding to .
796	* @param a_pTlb The TLB.
797	* @param a_uTag Value returned by IEMTLB_CALC_TAG or
798	* IEMTLB_CALC_TAG_NO_REV.
799	*/
800	#define IEMTLB_TAG_TO_EVEN_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_EVEN_INDEX(a_uTag)] )
801
802	/** Converts a GC address to an even TLB index. */
803	#define IEMTLB_ADDR_TO_EVEN_INDEX(a_GCPtr) IEMTLB_TAG_TO_EVEN_INDEX(IEMTLB_CALC_TAG_NO_REV(a_GCPtr))
804
805
806	/** @def IEM_WITH_TLB_TRACE
807	* Enables the TLB tracing.
808	* Adjust buffer size in IEMR3Init. */
809	#if defined(DOXYGEN_RUNNING) \|\| 0
810	# define IEM_WITH_TLB_TRACE
811	#endif
812
813	#ifdef IEM_WITH_TLB_TRACE
814
815	/** TLB trace entry types. */
816	typedef enum : uint8_t
817	{
818	kIemTlbTraceType_Invalid,
819	kIemTlbTraceType_InvlPg,
820	kIemTlbTraceType_EvictSlot,
821	kIemTlbTraceType_LargeEvictSlot,
822	kIemTlbTraceType_LargeScan,
823	kIemTlbTraceType_Flush,
824	kIemTlbTraceType_FlushGlobal, /*< x86 specific /
825	kIemTlbTraceType_Load,
826	kIemTlbTraceType_LoadGlobal, /*< x86 specific /
827	kIemTlbTraceType_Load_Cr0, /*< x86 specific /
828	kIemTlbTraceType_Load_Cr3, /*< x86 specific /
829	kIemTlbTraceType_Load_Cr4, /*< x86 specific /
830	kIemTlbTraceType_Load_Efer, /*< x86 specific /
831	kIemTlbTraceType_Irq,
832	kIemTlbTraceType_Xcpt,
833	kIemTlbTraceType_IRet, /*< x86 specific /
834	kIemTlbTraceType_Tb_Compile,
835	kIemTlbTraceType_Tb_Exec_Threaded,
836	kIemTlbTraceType_Tb_Exec_Native,
837	kIemTlbTraceType_User0,
838	kIemTlbTraceType_User1,
839	kIemTlbTraceType_User2,
840	kIemTlbTraceType_User3,
841	} IEMTLBTRACETYPE;
842
843	/** TLB trace entry. */
844	typedef struct IEMTLBTRACEENTRY
845	{
846	/** The flattened RIP for the event. */
847	uint64_t rip;
848	/** The event type. */
849	IEMTLBTRACETYPE enmType;
850	/** Byte parameter - typically used as 'bool fDataTlb'. */
851	uint8_t bParam;
852	/** 16-bit parameter value. */
853	uint16_t u16Param;
854	/** 32-bit parameter value. */
855	uint32_t u32Param;
856	/** 64-bit parameter value. */
857	uint64_t u64Param;
858	/** 64-bit parameter value. */
859	uint64_t u64Param2;
860	} IEMTLBTRACEENTRY;
861	AssertCompileSize(IEMTLBTRACEENTRY, 32);
862	/** Pointer to a TLB trace entry. */
863	typedef IEMTLBTRACEENTRY *PIEMTLBTRACEENTRY;
864	/** Pointer to a const TLB trace entry. */
865	typedef IEMTLBTRACEENTRY const *PCIEMTLBTRACEENTRY;
866	#endif /* !IEM_WITH_TLB_TRACE */
867
868	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
869	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) \
870	iemTlbTrace(a_pVCpu, kIemTlbTraceType_InvlPg, a_GCPtr)
871	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
872	iemTlbTrace(a_pVCpu, kIemTlbTraceType_EvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
873	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
874	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeEvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
875	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) \
876	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeScan, 0, 0, a_fDataTlb, (uint8_t)a_fGlobal \| ((uint8_t)a_fNonGlobal << 1))
877	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) \
878	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Flush, a_uRev, 0, a_fDataTlb)
879	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) \
880	iemTlbTrace(a_pVCpu, kIemTlbTraceType_FlushGlobal, a_uRev, a_uGRev, a_fDataTlb)
881	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
882	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
883	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
884	iemTlbTrace(a_pVCpu, kIemTlbTraceType_LoadGlobal, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
885	#else
886	# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) do { } while (0)
887	# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
888	# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
889	# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) do { } while (0)
890	# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) do { } while (0)
891	# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) do { } while (0)
892	# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
893	# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
894	#endif
895
896	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
897	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr0, a_uNew, a_uOld)
898	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr3, a_uNew, a_uOld)
899	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr4, a_uNew, a_uOld)
900	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Efer, a_uNew, a_uOld)
901	#else
902	# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) do { } while (0)
903	# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) do { } while (0)
904	# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) do { } while (0)
905	# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) do { } while (0)
906	#endif
907
908	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
909	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) \
910	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Irq, a_fEFlags, 0, a_uVector, a_fFlags)
911	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) \
912	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Xcpt, a_uErr, a_uCr2, a_uVector, a_fFlags)
913	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) \
914	iemTlbTrace(a_pVCpu, kIemTlbTraceType_IRet, a_uRetRip, a_fEFlags, 0, a_uRetCs)
915	#else
916	# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) do { } while (0)
917	# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) do { } while (0)
918	# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) do { } while (0)
919	#endif
920
921	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
922	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) \
923	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Compile, a_GCPhysPc)
924	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) \
925	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Threaded, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
926	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) \
927	iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Native, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
928	#else
929	# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) do { } while (0)
930	# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) do { } while (0)
931	# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) do { } while (0)
932	#endif
933
934	#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
935	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
936	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User0, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
937	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
938	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User1, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
939	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
940	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User2, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
941	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
942	iemTlbTrace(a_pVCpu, kIemTlbTraceType_User3, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
943	#else
944	# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
945	# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
946	# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
947	# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
948	#endif
949
950
951	/** @name IEM_MC_F_XXX - MC block flags/clues.
952	* @note x86 specific
953	* @todo Merge with IEM_CIMPL_F_XXX
954	* @{ */
955	#define IEM_MC_F_ONLY_8086 RT_BIT_32(0)
956	#define IEM_MC_F_MIN_186 RT_BIT_32(1)
957	#define IEM_MC_F_MIN_286 RT_BIT_32(2)
958	#define IEM_MC_F_NOT_286_OR_OLDER IEM_MC_F_MIN_386
959	#define IEM_MC_F_MIN_386 RT_BIT_32(3)
960	#define IEM_MC_F_MIN_486 RT_BIT_32(4)
961	#define IEM_MC_F_MIN_PENTIUM RT_BIT_32(5)
962	#define IEM_MC_F_MIN_PENTIUM_II IEM_MC_F_MIN_PENTIUM
963	#define IEM_MC_F_MIN_CORE IEM_MC_F_MIN_PENTIUM
964	#define IEM_MC_F_64BIT RT_BIT_32(6)
965	#define IEM_MC_F_NOT_64BIT RT_BIT_32(7)
966	/** This is set by IEMAllN8vePython.py to indicate a variation with the
967	* flags-clearing-and-checking. */
968	#define IEM_MC_F_WITH_FLAGS RT_BIT_32(8)
969	/** This is set by IEMAllN8vePython.py to indicate a variation without the
970	* flags-clearing-and-checking, when there is also a variation with that.
971	* @note Do not set this manully, it's only for python and for testing in
972	* the native recompiler! */
973	#define IEM_MC_F_WITHOUT_FLAGS RT_BIT_32(9)
974	/** @} */
975
976	/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
977	*
978	* These clues are mainly for the recompiler, so that it can emit correct code.
979	*
980	* They are processed by the python script and which also automatically
981	* calculates flags for MC blocks based on the statements, extending the use of
982	* these flags to describe MC block behavior to the recompiler core. The python
983	* script pass the flags to the IEM_MC2_END_EMIT_CALLS macro, but mainly for
984	* error checking purposes. The script emits the necessary fEndTb = true and
985	* similar statements as this reduces compile time a tiny bit.
986	*
987	* @{ */
988	/** Flag set if direct branch, clear if absolute or indirect. */
989	#define IEM_CIMPL_F_BRANCH_DIRECT RT_BIT_32(0)
990	/** Flag set if indirect branch, clear if direct or relative.
991	* This is also used for all system control transfers (SYSCALL, SYSRET, INT, ++)
992	* as well as for return instructions (RET, IRET, RETF). */
993	#define IEM_CIMPL_F_BRANCH_INDIRECT RT_BIT_32(1)
994	/** Flag set if relative branch, clear if absolute or indirect. */
995	#define IEM_CIMPL_F_BRANCH_RELATIVE RT_BIT_32(2)
996	/** Flag set if conditional branch, clear if unconditional. */
997	#define IEM_CIMPL_F_BRANCH_CONDITIONAL RT_BIT_32(3)
998	/** Flag set if it's a far branch (changes CS).
999	* @note x86 specific */
1000	#define IEM_CIMPL_F_BRANCH_FAR RT_BIT_32(4)
1001	/** Convenience: Testing any kind of branch. */
1002	#define IEM_CIMPL_F_BRANCH_ANY (IEM_CIMPL_F_BRANCH_DIRECT \| IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_RELATIVE)
1003
1004	/** Execution flags may change (IEMCPU::fExec). */
1005	#define IEM_CIMPL_F_MODE RT_BIT_32(5)
1006	/** May change significant portions of RFLAGS.
1007	* @note x86 specific */
1008	#define IEM_CIMPL_F_RFLAGS RT_BIT_32(6)
1009	/** May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS.
1010	* @note x86 specific */
1011	#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(7)
1012	/** May trigger interrupt shadowing.
1013	* @note x86 specific */
1014	#define IEM_CIMPL_F_INHIBIT_SHADOW RT_BIT_32(8)
1015	/** May enable interrupts, so recheck IRQ immediately afterwards executing
1016	* the instruction. */
1017	#define IEM_CIMPL_F_CHECK_IRQ_AFTER RT_BIT_32(9)
1018	/** May disable interrupts, so recheck IRQ immediately before executing the
1019	* instruction. */
1020	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE RT_BIT_32(10)
1021	/** Convenience: Check for IRQ both before and after an instruction. */
1022	#define IEM_CIMPL_F_CHECK_IRQ_BEFORE_AND_AFTER (IEM_CIMPL_F_CHECK_IRQ_BEFORE \| IEM_CIMPL_F_CHECK_IRQ_AFTER)
1023	/** May trigger a VM exit (treated like IEM_CIMPL_F_MODE atm). */
1024	#define IEM_CIMPL_F_VMEXIT RT_BIT_32(11)
1025	/** May modify FPU state.
1026	* @todo Not sure if this is useful yet. */
1027	#define IEM_CIMPL_F_FPU RT_BIT_32(12)
1028	/** REP prefixed instruction which may yield before updating PC.
1029	* @todo Not sure if this is useful, REP functions now return non-zero
1030	* status if they don't update the PC.
1031	* @note x86 specific */
1032	#define IEM_CIMPL_F_REP RT_BIT_32(13)
1033	/** I/O instruction.
1034	* @todo Not sure if this is useful yet.
1035	* @note x86 specific */
1036	#define IEM_CIMPL_F_IO RT_BIT_32(14)
1037	/** Force end of TB after the instruction. */
1038	#define IEM_CIMPL_F_END_TB RT_BIT_32(15)
1039	/** Flag set if a branch may also modify the stack (push/pop return address). */
1040	#define IEM_CIMPL_F_BRANCH_STACK RT_BIT_32(16)
1041	/** Flag set if a branch may also modify the stack (push/pop return address)
1042	* and switch it (load/restore SS:RSP).
1043	* @note x86 specific */
1044	#define IEM_CIMPL_F_BRANCH_STACK_FAR RT_BIT_32(17)
1045	/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
1046	#define IEM_CIMPL_F_XCPT \
1047	(IEM_CIMPL_F_BRANCH_INDIRECT \| IEM_CIMPL_F_BRANCH_FAR \| IEM_CIMPL_F_BRANCH_STACK_FAR \
1048	\| IEM_CIMPL_F_MODE \| IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_VMEXIT)
1049
1050	/** The block calls a C-implementation instruction function with two implicit arguments.
1051	* Mutually exclusive with IEM_CIMPL_F_CALLS_AIMPL and
1052	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1053	* @note The python scripts will add this if missing. */
1054	#define IEM_CIMPL_F_CALLS_CIMPL RT_BIT_32(18)
1055	/** The block calls an ASM-implementation instruction function.
1056	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and
1057	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1058	* @note The python scripts will add this if missing. */
1059	#define IEM_CIMPL_F_CALLS_AIMPL RT_BIT_32(19)
1060	/** The block calls an ASM-implementation instruction function with an implicit
1061	* X86FXSTATE pointer argument.
1062	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
1063	* IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE.
1064	* @note The python scripts will add this if missing.
1065	* @note x86 specific */
1066	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE RT_BIT_32(20)
1067	/** The block calls an ASM-implementation instruction function with an implicit
1068	* X86XSAVEAREA pointer argument.
1069	* Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
1070	* IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1071	* @note No different from IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE, so same value.
1072	* @note The python scripts will add this if missing.
1073	* @note x86 specific */
1074	#define IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE
1075	/** @} */
1076
1077
1078	/** @name IEM_F_XXX - Execution mode flags (IEMCPU::fExec, IEMTB::fFlags).
1079	*
1080	* These flags are set when entering IEM and adjusted as code is executed, such
1081	* that they will always contain the current values as instructions are
1082	* finished.
1083	*
1084	* In recompiled execution mode, (most of) these flags are included in the
1085	* translation block selection key and stored in IEMTB::fFlags alongside the
1086	* IEMTB_F_XXX flags. The latter flags uses bits 31 thru 24, which are all zero
1087	* in IEMCPU::fExec.
1088	*
1089	* @{ */
1090	/** Mode: The block target mode mask.
1091	* X86: CPUMODE plus protected, v86 and pre-386 indicators.
1092	* ARM: PSTATE.nRW \| PSTATE.T \| PSTATE.EL.
1093	* This doesn't quite overlap with SPSR_ELx when in AARCH32 mode,
1094	* but that's life. */
1095	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1096	# define IEM_F_MODE_MASK UINT32_C(0x0000001f)
1097	#elif defined(VBOX_VMM_TARGET_ARMV8)
1098	# define IEM_F_MODE_MASK UINT32_C(0x0000003c)
1099	#endif
1100
1101	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1102	/** X86 Mode: The IEMMODE part of the IEMTB_F_MODE_MASK value. */
1103	# define IEM_F_MODE_X86_CPUMODE_MASK UINT32_C(0x00000003)
1104	/** X86 Mode: Bit used to indicating pre-386 CPU in 16-bit mode (for eliminating
1105	* conditional in EIP/IP updating), and flat wide open CS, SS, DS, and ES in
1106	* 32-bit mode (for simplifying most memory accesses). */
1107	# define IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK UINT32_C(0x00000004)
1108	/** X86 Mode: Bit indicating protected mode, real mode (or SMM) when not set. */
1109	# define IEM_F_MODE_X86_PROT_MASK UINT32_C(0x00000008)
1110	/** X86 Mode: Bit used to indicate virtual 8086 mode (only 16-bit). */
1111	# define IEM_F_MODE_X86_V86_MASK UINT32_C(0x00000010)
1112
1113	/** X86 Mode: 16-bit on 386 or later. */
1114	# define IEM_F_MODE_X86_16BIT UINT32_C(0x00000000)
1115	/** X86 Mode: 80286, 80186 and 8086/88 targetting blocks (EIP update opt). */
1116	# define IEM_F_MODE_X86_16BIT_PRE_386 UINT32_C(0x00000004)
1117	/** X86 Mode: 16-bit protected mode on 386 or later. */
1118	# define IEM_F_MODE_X86_16BIT_PROT UINT32_C(0x00000008)
1119	/** X86 Mode: 16-bit protected mode on 386 or later. */
1120	# define IEM_F_MODE_X86_16BIT_PROT_PRE_386 UINT32_C(0x0000000c)
1121	/** X86 Mode: 16-bit virtual 8086 protected mode (on 386 or later). */
1122	# define IEM_F_MODE_X86_16BIT_PROT_V86 UINT32_C(0x00000018)
1123
1124	/** X86 Mode: 32-bit on 386 or later. */
1125	# define IEM_F_MODE_X86_32BIT UINT32_C(0x00000001)
1126	/** X86 Mode: 32-bit mode with wide open flat CS, SS, DS and ES. */
1127	# define IEM_F_MODE_X86_32BIT_FLAT UINT32_C(0x00000005)
1128	/** X86 Mode: 32-bit protected mode. */
1129	# define IEM_F_MODE_X86_32BIT_PROT UINT32_C(0x00000009)
1130	/** X86 Mode: 32-bit protected mode with wide open flat CS, SS, DS and ES. */
1131	# define IEM_F_MODE_X86_32BIT_PROT_FLAT UINT32_C(0x0000000d)
1132
1133	/** X86 Mode: 64-bit (includes protected, but not the flat bit). */
1134	# define IEM_F_MODE_X86_64BIT UINT32_C(0x0000000a)
1135
1136	/** X86 Mode: Checks if @a a_fExec represent a FLAT mode. */
1137	# define IEM_F_MODE_X86_IS_FLAT(a_fExec) ( ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT \
1138	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT \
1139	\|\| ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT)
1140
1141	/** X86: The current protection level (CPL) shift factor. */
1142	# define IEM_F_X86_CPL_SHIFT 8
1143	/** X86: The current protection level (CPL) mask. */
1144	# define IEM_F_X86_CPL_MASK UINT32_C(0x00000300)
1145	/** X86: The current protection level (CPL) shifted mask. */
1146	# define IEM_F_X86_CPL_SMASK UINT32_C(0x00000003)
1147
1148	/** X86: Alignment checks enabled (CR0.AM=1 & EFLAGS.AC=1). */
1149	# define IEM_F_X86_AC UINT32_C(0x00080000)
1150
1151	/** X86 execution context.
1152	* The IEM_F_X86_CTX_XXX values are individual flags that can be combined (with
1153	* the exception of IEM_F_X86_CTX_NORMAL). This allows running VMs from SMM
1154	* mode. */
1155	# define IEM_F_X86_CTX_MASK UINT32_C(0x0000f000)
1156	/** X86 context: Plain regular execution context. */
1157	# define IEM_F_X86_CTX_NORMAL UINT32_C(0x00000000)
1158	/** X86 context: VT-x enabled. */
1159	# define IEM_F_X86_CTX_VMX UINT32_C(0x00001000)
1160	/** X86 context: AMD-V enabled. */
1161	# define IEM_F_X86_CTX_SVM UINT32_C(0x00002000)
1162	/** X86 context: In AMD-V or VT-x guest mode. */
1163	# define IEM_F_X86_CTX_IN_GUEST UINT32_C(0x00004000)
1164	/** X86 context: System management mode (SMM). */
1165	# define IEM_F_X86_CTX_SMM UINT32_C(0x00008000)
1166
1167	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1168	* iemRegFinishClearingRF() most for most situations (CPUMCTX_DBG_HIT_DRX_MASK
1169	* and CPUMCTX_DBG_DBGF_MASK are covered by the IEM_F_PENDING_BRK_XXX bits
1170	* alread). */
1171
1172	/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1173	* iemRegFinishClearingRF() most for most situations
1174	* (CPUMCTX_DBG_HIT_DRX_MASK and CPUMCTX_DBG_DBGF_MASK are covered by
1175	* the IEM_F_PENDING_BRK_XXX bits alread). */
1176
1177	#endif /* X86 \|\| doxygen */
1178
1179	#if defined(VBOX_VMM_TARGET_ARMV8) \|\| defined(DOXYGEN_RUNNING)
1180	/** ARM Mode: Exception (privilege) level shift count. */
1181	# define IEM_F_MODE_ARM_EL_SHIFT 2
1182	/** ARM Mode: Exception (privilege) level mask. */
1183	# define IEM_F_MODE_ARM_EL_MASK UINT32_C(0x0000000c)
1184	/** ARM Mode: Exception (privilege) level shifted down mask. */
1185	# define IEM_F_MODE_ARM_EL_SMASK UINT32_C(0x00000003)
1186	/** ARM Mode: 32-bit (set) or 64-bit (clear) indicator (SPSR_ELx.M[4]). */
1187	# define IEM_F_MODE_ARM_32BIT UINT32_C(0x00000010)
1188	/** ARM Mode: Thumb mode indicator (SPSR_ELx.T). */
1189	# define IEM_F_MODE_ARM_T32 UINT32_C(0x00000020)
1190
1191	/** ARM Mode: Get the exception (privilege) level. */
1192	# define IEM_F_MODE_ARM_GET_EL(a_fExec) (((a_fExec) >> IEM_F_MODE_ARM_EL_SHIFT) & IEM_F_MODE_ARM_EL_SMASK)
1193
1194	/** ARM: The stack pointer index - not part of mode */
1195	# define IEM_F_ARM_SP_IDX UINT32_C(0x00000003)
1196	/** ARM: Get the SP register index. */
1197	# define IEM_F_ARM_GET_SP_IDX(a_fExec) ((a_fExec) & IEM_F_ARM_SP_IDX)
1198
1199	/** ARM: Normal alignment checks enabled (SCTRL_ELx.A). */
1200	# define IEM_F_ARM_A UINT32_C(0x00004000)
1201	/** ARM: LSE2 alignment checks enabled (~SCTRL_ELx.nAA). */
1202	# define IEM_F_ARM_AA UINT32_C(0x00080000)
1203	#endif /* ARM \|\| doxygen */
1204
1205	/** Bypass access handlers when set. */
1206	#define IEM_F_BYPASS_HANDLERS UINT32_C(0x00010000)
1207	/** Have pending hardware instruction breakpoints. */
1208	#define IEM_F_PENDING_BRK_INSTR UINT32_C(0x00020000)
1209	/** Have pending hardware data breakpoints. */
1210	#define IEM_F_PENDING_BRK_DATA UINT32_C(0x00040000)
1211
1212	/** X86: Have pending hardware I/O breakpoints. */
1213	#define IEM_F_PENDING_BRK_X86_IO UINT32_C(0x00000400)
1214	/** X86: Disregard the lock prefix (implied or not) when set. */
1215	#define IEM_F_X86_DISREGARD_LOCK UINT32_C(0x00000800)
1216
1217	/** ARM: Software step (single step).
1218	* @todo make generic? */
1219	#define IEM_F_ARM_SOFTWARE_STEP UINT32_C(0x00000400)
1220
1221	/** Pending breakpoint mask (what iemCalcExecDbgFlags works out). */
1222	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1223	# define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR \| IEM_F_PENDING_BRK_DATA \| IEM_F_PENDING_BRK_X86_IO)
1224	#else
1225	# define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR \| IEM_F_PENDING_BRK_DATA)
1226	#endif
1227
1228	/** Caller configurable options. */
1229	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1230	# define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS \| IEM_F_X86_DISREGARD_LOCK)
1231	#else
1232	# define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS)
1233	#endif
1234	/** @} */
1235
1236
1237	/** @name IEMTB_F_XXX - Translation block flags (IEMTB::fFlags).
1238	*
1239	* Extends the IEM_F_XXX flags (subject to IEMTB_F_IEM_F_MASK) to make up the
1240	* translation block flags. The combined flag mask (subject to
1241	* IEMTB_F_KEY_MASK) is used as part of the lookup key for translation blocks.
1242	*
1243	* @{ */
1244	/** Mask of IEM_F_XXX flags included in IEMTB_F_XXX. */
1245	#define IEMTB_F_IEM_F_MASK UINT32_C(0x00ffffff)
1246
1247	/** Type: The block type mask. */
1248	#define IEMTB_F_TYPE_MASK UINT32_C(0x03000000)
1249	/** Type: Purly threaded recompiler (via tables). */
1250	#define IEMTB_F_TYPE_THREADED UINT32_C(0x01000000)
1251	/** Type: Native recompilation. */
1252	#define IEMTB_F_TYPE_NATIVE UINT32_C(0x02000000)
1253
1254	/** Set when we're starting the block in an "interrupt shadow".
1255	* We don't need to distingish between the two types of this mask, thus the one.
1256	* @see CPUMCTX_INHIBIT_SHADOW, CPUMIsInInterruptShadow() */
1257	#define IEMTB_F_X86_INHIBIT_SHADOW UINT32_C(0x04000000)
1258	/** Set when we're currently inhibiting NMIs
1259	* @see CPUMCTX_INHIBIT_NMI, CPUMAreInterruptsInhibitedByNmi() */
1260	#define IEMTB_F_X86_INHIBIT_NMI UINT32_C(0x08000000)
1261
1262	/** Checks that EIP/IP is wihin CS.LIM before each instruction. Used when
1263	* we're close the limit before starting a TB, as determined by
1264	* iemGetTbFlagsForCurrentPc(). */
1265	#define IEMTB_F_X86_CS_LIM_CHECKS UINT32_C(0x10000000)
1266
1267	/** Mask of the IEMTB_F_XXX flags that are part of the TB lookup key.
1268	*
1269	* @note We skip all of IEM_F_X86_CTX_MASK, with the exception of SMM (which we
1270	* don't implement), because we don't currently generate any context
1271	* specific code - that's all handled in CIMPL functions.
1272	*
1273	* For the threaded recompiler we don't generate any CPL specific code
1274	* either, but the native recompiler does for memory access (saves getting
1275	* the CPL from fExec and turning it into IEMTLBE_F_PT_NO_USER).
1276	* Since most OSes will not share code between rings, this shouldn't
1277	* have any real effect on TB/memory/recompiling load.
1278	*/
1279	#if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1280	# define IEMTB_F_KEY_MASK ((UINT32_MAX & ~(IEM_F_X86_CTX_MASK \| IEMTB_F_TYPE_MASK)) \| IEM_F_X86_CTX_SMM)
1281	#else
1282	# define IEMTB_F_KEY_MASK (UINT32_MAX)
1283	#endif
1284	/** @} */
1285
1286	#ifdef VBOX_VMM_TARGET_X86
1287	AssertCompile( (IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1288	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1289	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_PROT_MASK));
1290	AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_V86_MASK));
1291	AssertCompile( (IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1292	AssertCompile( IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1293	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_PROT_MASK));
1294	AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1295	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1296	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1297	AssertCompile( IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1298	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_V86_MASK));
1299	AssertCompile( (IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1300	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1301	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_PROT_MASK);
1302	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1303	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_PROT_MASK);
1304	AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1305	AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_V86_MASK);
1306
1307	AssertCompile( (IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1308	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1309	AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_PROT_MASK));
1310	AssertCompile( (IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1311	AssertCompile( IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1312	AssertCompile(!(IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_PROT_MASK));
1313	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1314	AssertCompile(!(IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1315	AssertCompile( IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1316	AssertCompile( (IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1317	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1318	AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_PROT_MASK);
1319
1320	AssertCompile( (IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_64BIT);
1321	AssertCompile( IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_PROT_MASK);
1322	AssertCompile(!(IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1323	#endif /* VBOX_VMM_TARGET_X86 */
1324
1325	#ifdef VBOX_VMM_TARGET_ARMV8
1326	AssertCompile(IEM_F_MODE_ARM_EL_SHIFT == ARMV8_SPSR_EL2_AARCH64_EL_SHIFT);
1327	AssertCompile(IEM_F_MODE_ARM_EL_MASK == ARMV8_SPSR_EL2_AARCH64_EL);
1328	AssertCompile(IEM_F_MODE_ARM_32BIT == ARMV8_SPSR_EL2_AARCH64_M4);
1329	AssertCompile(IEM_F_MODE_ARM_T32 == ARMV8_SPSR_EL2_AARCH64_T);
1330	#endif
1331
1332	/** Native instruction type for use with the native code generator.
1333	* This is a byte (uint8_t) for x86 and amd64 and uint32_t for the other(s). */
1334	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
1335	typedef uint8_t IEMNATIVEINSTR;
1336	#else
1337	typedef uint32_t IEMNATIVEINSTR;
1338	#endif
1339	/** Pointer to a native instruction unit. */
1340	typedef IEMNATIVEINSTR *PIEMNATIVEINSTR;
1341	/** Pointer to a const native instruction unit. */
1342	typedef IEMNATIVEINSTR const *PCIEMNATIVEINSTR;
1343
1344	/**
1345	* A call for the threaded call table.
1346	*/
1347	typedef struct IEMTHRDEDCALLENTRY
1348	{
1349	/** The function to call (IEMTHREADEDFUNCS). */
1350	uint16_t enmFunction;
1351
1352	/** Instruction number in the TB (for statistics). */
1353	uint8_t idxInstr;
1354	/** The opcode length. */
1355	uint8_t cbOpcode;
1356	/** Offset into IEMTB::pabOpcodes. */
1357	uint16_t offOpcode;
1358
1359	/** TB lookup table index (7 bits) and large size (1 bits).
1360	*
1361	* The default size is 1 entry, but for indirect calls and returns we set the
1362	* top bit and allocate 4 (IEM_TB_LOOKUP_TAB_LARGE_SIZE) entries. The large
1363	* tables uses RIP for selecting the entry to use, as it is assumed a hash table
1364	* lookup isn't that slow compared to sequentially trying out 4 TBs.
1365	*
1366	* By default lookup table entry 0 for a TB is reserved as a fallback for
1367	* calltable entries w/o explicit entreis, so this member will be non-zero if
1368	* there is a lookup entry associated with this call.
1369	*
1370	* @sa IEM_TB_LOOKUP_TAB_GET_SIZE, IEM_TB_LOOKUP_TAB_GET_IDX
1371	*/
1372	uint8_t uTbLookup;
1373
1374	/** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
1375	uint8_t fFlags;
1376
1377	/** Generic parameters.
1378	* @todo ARM: Hope we can get away with one param here... */
1379	uint64_t auParams[3];
1380	} IEMTHRDEDCALLENTRY;
1381	AssertCompileSize(IEMTHRDEDCALLENTRY, sizeof(uint64_t) * 4);
1382	/** Pointer to a threaded call entry. */
1383	typedef struct IEMTHRDEDCALLENTRY *PIEMTHRDEDCALLENTRY;
1384	/** Pointer to a const threaded call entry. */
1385	typedef IEMTHRDEDCALLENTRY const *PCIEMTHRDEDCALLENTRY;
1386
1387	/** The number of TB lookup table entries for a large allocation
1388	* (IEMTHRDEDCALLENTRY::uTbLookup bit 7 set). */
1389	#define IEM_TB_LOOKUP_TAB_LARGE_SIZE 4
1390	/** Get the lookup table size from IEMTHRDEDCALLENTRY::uTbLookup. */
1391	#define IEM_TB_LOOKUP_TAB_GET_SIZE(a_uTbLookup) (!((a_uTbLookup) & 0x80) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE)
1392	/** Get the first lookup table index from IEMTHRDEDCALLENTRY::uTbLookup. */
1393	#define IEM_TB_LOOKUP_TAB_GET_IDX(a_uTbLookup) ((a_uTbLookup) & 0x7f)
1394	/** Get the lookup table index from IEMTHRDEDCALLENTRY::uTbLookup and PC. */
1395	#define IEM_TB_LOOKUP_TAB_GET_IDX_WITH_PC(a_uTbLookup, a_Pc) \
1396	(!((a_uTbLookup) & 0x80) ? (a_uTbLookup) & 0x7f : ((a_uTbLookup) & 0x7f) + ((a_Pc) & (IEM_TB_LOOKUP_TAB_LARGE_SIZE - 1)) )
1397
1398	/** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
1399	#define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge) ((a_idxTable) \| ((a_fLarge) ? 0x80 : 0))
1400
1401
1402	/** The call entry is a jump target. */
1403	#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET UINT8_C(0x01)
1404
1405
1406	/**
1407	* Native IEM TB 'function' typedef.
1408	*
1409	* This will throw/longjmp on occation.
1410	*
1411	* @note AMD64 doesn't have that many non-volatile registers and does sport
1412	* 32-bit address displacments, so we don't need pCtx.
1413	*
1414	* On ARM64 pCtx allows us to directly address the whole register
1415	* context without requiring a separate indexing register holding the
1416	* offset. This saves an instruction loading the offset for each guest
1417	* CPU context access, at the cost of a non-volatile register.
1418	* Fortunately, ARM64 has quite a lot more registers.
1419	*/
1420	typedef
1421	#ifdef RT_ARCH_AMD64
1422	int FNIEMTBNATIVE(PVMCPUCC pVCpu)
1423	#else
1424	int FNIEMTBNATIVE(PVMCPUCC pVCpu, PCPUMCTX pCtx)
1425	#endif
1426	#if RT_CPLUSPLUS_PREREQ(201700)
1427	IEM_NOEXCEPT_MAY_LONGJMP
1428	#endif
1429	;
1430	/** Pointer to a native IEM TB entry point function.
1431	* This will throw/longjmp on occation. */
1432	typedef FNIEMTBNATIVE *PFNIEMTBNATIVE;
1433
1434
1435	/**
1436	* Translation block.
1437	*
1438	* The current plan is to just keep TBs and associated lookup hash table private
1439	* to each VCpu as that simplifies TB removal greatly (no races) and generally
1440	* avoids using expensive atomic primitives for updating lists and stuff.
1441	*/
1442	#pragma pack(2) /* to prevent the Thrd structure from being padded unnecessarily */
1443	typedef struct IEMTB
1444	{
1445	/** Next block with the same hash table entry. */
1446	struct IEMTB *pNext;
1447	/** Usage counter. */
1448	uint32_t cUsed;
1449	/** The IEMCPU::msRecompilerPollNow last time it was used. */
1450	uint32_t msLastUsed;
1451
1452	/** @name What uniquely identifies the block.
1453	* @{ */
1454	RTGCPHYS GCPhysPc;
1455	/** IEMTB_F_XXX (i.e. IEM_F_XXX ++). */
1456	uint32_t fFlags;
1457	union
1458	{
1459	struct
1460	{
1461	/*< Relevant CS X86DESCATTR_XXX bits. /
1462	uint16_t fAttr;
1463	} x86;
1464	};
1465	/** @} */
1466
1467	/** Number of opcode ranges. */
1468	uint8_t cRanges;
1469	/** Statistics: Number of instructions in the block. */
1470	uint8_t cInstructions;
1471
1472	/** Type specific info. */
1473	union
1474	{
1475	struct
1476	{
1477	/** The call sequence table. */
1478	PIEMTHRDEDCALLENTRY paCalls;
1479	/** Number of calls in paCalls. */
1480	uint16_t cCalls;
1481	/** Number of calls allocated. */
1482	uint16_t cAllocated;
1483	} Thrd;
1484	struct
1485	{
1486	/** The native instructions (PFNIEMTBNATIVE). */
1487	PIEMNATIVEINSTR paInstructions;
1488	/** Number of instructions pointed to by paInstructions. */
1489	uint32_t cInstructions;
1490	} Native;
1491	/** Generic view for zeroing when freeing. */
1492	struct
1493	{
1494	uintptr_t uPtr;
1495	uint32_t uData;
1496	} Gen;
1497	};
1498
1499	/** The allocation chunk this TB belongs to. */
1500	uint8_t idxAllocChunk;
1501	/** The number of entries in the lookup table.
1502	* Because we're out of space, the TB lookup table is located before the
1503	* opcodes pointed to by pabOpcodes. */
1504	uint8_t cTbLookupEntries;
1505
1506	/** Number of bytes of opcodes stored in pabOpcodes.
1507	* @todo this field isn't really needed, aRanges keeps the actual info. */
1508	uint16_t cbOpcodes;
1509	/** Pointer to the opcode bytes this block was recompiled from.
1510	* This also points to the TB lookup table, which starts cTbLookupEntries
1511	* entries before the opcodes (we don't have room atm for another point). */
1512	uint8_t *pabOpcodes;
1513
1514	union
1515	{
1516	/** Native recompilation debug info if enabled.
1517	* This is only generated by the native recompiler. */
1518	struct IEMTBDBG *pDbgInfo;
1519	/** For threaded TBs and natives when debug info is disabled, this is the flat
1520	* PC corresponding to GCPhysPc. */
1521	RTGCPTR FlatPc;
1522	};
1523
1524	/* --- 64 byte cache line end --- */
1525
1526	/** Opcode ranges.
1527	*
1528	* The opcode checkers and maybe TLB loading functions will use this to figure
1529	* out what to do. The parameter will specify an entry and the opcode offset to
1530	* start at and the minimum number of bytes to verify (instruction length).
1531	*
1532	* When VT-x and AMD-V looks up the opcode bytes for an exitting instruction,
1533	* they'll first translate RIP (+ cbInstr - 1) to a physical address using the
1534	* code TLB (must have a valid entry for that address) and scan the ranges to
1535	* locate the corresponding opcodes. Probably.
1536	*/
1537	struct IEMTBOPCODERANGE
1538	{
1539	/** Offset within pabOpcodes. */
1540	uint16_t offOpcodes;
1541	/** Number of bytes. */
1542	uint16_t cbOpcodes;
1543	/** The page offset. */
1544	RT_GCC_EXTENSION
1545	uint16_t offPhysPage : 12;
1546	/** Unused bits. */
1547	RT_GCC_EXTENSION
1548	uint16_t u2Unused : 2;
1549	/** Index into GCPhysPc + aGCPhysPages for the physical page address. */
1550	RT_GCC_EXTENSION
1551	uint16_t idxPhysPage : 2;
1552	} aRanges[8];
1553
1554	/** Physical pages that this TB covers.
1555	* The GCPhysPc w/o page offset is element zero, so starting here with 1. */
1556	RTGCPHYS aGCPhysPages[2];
1557	} IEMTB;
1558	#pragma pack()
1559	AssertCompileMemberAlignment(IEMTB, GCPhysPc, sizeof(RTGCPHYS));
1560	AssertCompileMemberAlignment(IEMTB, Thrd, sizeof(void *));
1561	AssertCompileMemberAlignment(IEMTB, pabOpcodes, sizeof(void *));
1562	AssertCompileMemberAlignment(IEMTB, pDbgInfo, sizeof(void *));
1563	AssertCompileMemberAlignment(IEMTB, aGCPhysPages, sizeof(RTGCPHYS));
1564	AssertCompileMemberOffset(IEMTB, aRanges, 64);
1565	AssertCompileMemberSize(IEMTB, aRanges[0], 6);
1566	#if 1
1567	AssertCompileSize(IEMTB, 128);
1568	# define IEMTB_SIZE_IS_POWER_OF_TWO /*< The IEMTB size is a power of two. /
1569	#else
1570	AssertCompileSize(IEMTB, 168);
1571	# undef IEMTB_SIZE_IS_POWER_OF_TWO
1572	#endif
1573
1574	/** Pointer to a translation block. */
1575	typedef IEMTB *PIEMTB;
1576	/** Pointer to a const translation block. */
1577	typedef IEMTB const *PCIEMTB;
1578
1579	/** Gets address of the given TB lookup table entry. */
1580	#define IEMTB_GET_TB_LOOKUP_TAB_ENTRY(a_pTb, a_idx) \
1581	((PIEMTB )&(a_pTb)->pabOpcodes[-(int)((a_pTb)->cTbLookupEntries - (a_idx)) sizeof(PIEMTB)])
1582
1583	/**
1584	* Gets the physical address for a TB opcode range.
1585	*/
1586	DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1587	{
1588	Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1589	uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1590	Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1591	if (idxPage == 0)
1592	return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1593	Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1594	return pTb->aGCPhysPages[idxPage - 1];
1595	}
1596
1597
1598	/**
1599	* A chunk of memory in the TB allocator.
1600	*/
1601	typedef struct IEMTBCHUNK
1602	{
1603	/** Pointer to the translation blocks in this chunk. */
1604	PIEMTB paTbs;
1605	#ifdef IN_RING0
1606	/** Allocation handle. */
1607	RTR0MEMOBJ hMemObj;
1608	#endif
1609	} IEMTBCHUNK;
1610
1611	/**
1612	* A per-CPU translation block allocator.
1613	*
1614	* Because of how the IEMTBCACHE uses the lower 6 bits of the TB address to keep
1615	* the length of the collision list, and of course also for cache line alignment
1616	* reasons, the TBs must be allocated with at least 64-byte alignment.
1617	* Memory is there therefore allocated using one of the page aligned allocators.
1618	*
1619	*
1620	* To avoid wasting too much memory, it is allocated piecemeal as needed,
1621	* in chunks (IEMTBCHUNK) of 2 MiB or more. The TB has an 8-bit chunk index
1622	* that enables us to quickly calculate the allocation bitmap position when
1623	* freeing the translation block.
1624	*/
1625	typedef struct IEMTBALLOCATOR
1626	{
1627	/** Magic value (IEMTBALLOCATOR_MAGIC). */
1628	uint32_t uMagic;
1629
1630	#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
1631	/** Mask corresponding to cTbsPerChunk - 1. */
1632	uint32_t fChunkMask;
1633	/** Shift count corresponding to cTbsPerChunk. */
1634	uint8_t cChunkShift;
1635	#else
1636	uint32_t uUnused;
1637	uint8_t bUnused;
1638	#endif
1639	/** Number of chunks we're allowed to allocate. */
1640	uint8_t cMaxChunks;
1641	/** Number of chunks currently populated. */
1642	uint16_t cAllocatedChunks;
1643	/** Number of translation blocks per chunk. */
1644	uint32_t cTbsPerChunk;
1645	/** Chunk size. */
1646	uint32_t cbPerChunk;
1647
1648	/** The maximum number of TBs. */
1649	uint32_t cMaxTbs;
1650	/** Total number of TBs in the populated chunks.
1651	* (cAllocatedChunks * cTbsPerChunk) */
1652	uint32_t cTotalTbs;
1653	/** The current number of TBs in use.
1654	* The number of free TBs: cAllocatedTbs - cInUseTbs; */
1655	uint32_t cInUseTbs;
1656	/** Statistics: Number of the cInUseTbs that are native ones. */
1657	uint32_t cNativeTbs;
1658	/** Statistics: Number of the cInUseTbs that are threaded ones. */
1659	uint32_t cThreadedTbs;
1660
1661	/** Where to start pruning TBs from when we're out.
1662	* See iemTbAllocatorAllocSlow for details. */
1663	uint32_t iPruneFrom;
1664	/** Where to start pruning native TBs from when we're out of executable memory.
1665	* See iemTbAllocatorFreeupNativeSpace for details. */
1666	uint32_t iPruneNativeFrom;
1667	uint64_t u64Padding;
1668
1669	/** Statistics: Number of TB allocation calls. */
1670	STAMCOUNTER StatAllocs;
1671	/** Statistics: Number of TB free calls. */
1672	STAMCOUNTER StatFrees;
1673	/** Statistics: Time spend pruning. */
1674	STAMPROFILE StatPrune;
1675	/** Statistics: Time spend pruning native TBs. */
1676	STAMPROFILE StatPruneNative;
1677
1678	/** The delayed free list (see iemTbAlloctorScheduleForFree). */
1679	PIEMTB pDelayedFreeHead;
1680	/* Head of the list of free TBs. */
1681	PIEMTB pTbsFreeHead;
1682
1683	/** Allocation chunks. */
1684	IEMTBCHUNK aChunks[256];
1685	} IEMTBALLOCATOR;
1686	/** Pointer to a TB allocator. */
1687	typedef struct IEMTBALLOCATOR *PIEMTBALLOCATOR;
1688
1689	/** Magic value for the TB allocator (Emmet Harley Cohen). */
1690	#define IEMTBALLOCATOR_MAGIC UINT32_C(0x19900525)
1691
1692
1693	/**
1694	* A per-CPU translation block cache (hash table).
1695	*
1696	* The hash table is allocated once during IEM initialization and size double
1697	* the max TB count, rounded up to the nearest power of two (so we can use and
1698	* AND mask rather than a rest division when hashing).
1699	*/
1700	typedef struct IEMTBCACHE
1701	{
1702	/** Magic value (IEMTBCACHE_MAGIC). */
1703	uint32_t uMagic;
1704	/** Size of the hash table. This is a power of two. */
1705	uint32_t cHash;
1706	/** The mask corresponding to cHash. */
1707	uint32_t uHashMask;
1708	uint32_t uPadding;
1709
1710	/** @name Statistics
1711	* @{ */
1712	/** Number of collisions ever. */
1713	STAMCOUNTER cCollisions;
1714
1715	/** Statistics: Number of TB lookup misses. */
1716	STAMCOUNTER cLookupMisses;
1717	/** Statistics: Number of TB lookup hits via hash table (debug only). */
1718	STAMCOUNTER cLookupHits;
1719	/** Statistics: Number of TB lookup hits via TB associated lookup table (debug only). */
1720	STAMCOUNTER cLookupHitsViaTbLookupTable;
1721	STAMCOUNTER auPadding2[2];
1722	/** Statistics: Collision list length pruning. */
1723	STAMPROFILE StatPrune;
1724	/** @} */
1725
1726	/** The hash table itself.
1727	* @note The lower 6 bits of the pointer is used for keeping the collision
1728	* list length, so we can take action when it grows too long.
1729	* This works because TBs are allocated using a 64 byte (or
1730	* higher) alignment from page aligned chunks of memory, so the lower
1731	* 6 bits of the address will always be zero.
1732	* See IEMTBCACHE_PTR_COUNT_MASK, IEMTBCACHE_PTR_MAKE and friends.
1733	*/
1734	RT_FLEXIBLE_ARRAY_EXTENSION
1735	PIEMTB apHash[RT_FLEXIBLE_ARRAY];
1736	} IEMTBCACHE;
1737	/** Pointer to a per-CPU translation block cache. */
1738	typedef IEMTBCACHE *PIEMTBCACHE;
1739
1740	/** Magic value for IEMTBCACHE (Johnny O'Neal). */
1741	#define IEMTBCACHE_MAGIC UINT32_C(0x19561010)
1742
1743	/** The collision count mask for IEMTBCACHE::apHash entries. */
1744	#define IEMTBCACHE_PTR_COUNT_MASK ((uintptr_t)0x3f)
1745	/** The max collision count for IEMTBCACHE::apHash entries before pruning. */
1746	#define IEMTBCACHE_PTR_MAX_COUNT ((uintptr_t)0x30)
1747	/** Combine a TB pointer and a collision list length into a value for an
1748	* IEMTBCACHE::apHash entry. */
1749	#define IEMTBCACHE_PTR_MAKE(a_pTb, a_cCount) (PIEMTB)((uintptr_t)(a_pTb) \| (a_cCount))
1750	/** Combine a TB pointer and a collision list length into a value for an
1751	* IEMTBCACHE::apHash entry. */
1752	#define IEMTBCACHE_PTR_GET_TB(a_pHashEntry) (PIEMTB)((uintptr_t)(a_pHashEntry) & ~IEMTBCACHE_PTR_COUNT_MASK)
1753	/** Combine a TB pointer and a collision list length into a value for an
1754	* IEMTBCACHE::apHash entry. */
1755	#define IEMTBCACHE_PTR_GET_COUNT(a_pHashEntry) ((uintptr_t)(a_pHashEntry) & IEMTBCACHE_PTR_COUNT_MASK)
1756
1757	/**
1758	* Calculates the hash table slot for a TB from physical PC address and TB flags.
1759	*/
1760	#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
1761	IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, (a_fTbFlags) & IEMTB_F_KEY_MASK, a_GCPhysPc)
1762
1763	/**
1764	* Calculates the hash table slot for a TB from physical PC address and TB
1765	* flags, ASSUMING the caller has applied IEMTB_F_KEY_MASK to @a a_fTbFlags.
1766	*/
1767	#define IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, a_fTbFlags, a_GCPhysPc) \
1768	(((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
1769
1770
1771	/** @name IEMBRANCHED_F_XXX - Branched indicator (IEMCPU::fTbBranched).
1772	*
1773	* These flags parallels the main IEM_CIMPL_F_BRANCH_XXX flags.
1774	*
1775	* @{ */
1776	/** Value if no branching happened recently. */
1777	#define IEMBRANCHED_F_NO UINT8_C(0x00)
1778	/** Flag set if direct branch, clear if absolute or indirect. */
1779	#define IEMBRANCHED_F_DIRECT UINT8_C(0x01)
1780	/** Flag set if indirect branch, clear if direct or relative. */
1781	#define IEMBRANCHED_F_INDIRECT UINT8_C(0x02)
1782	/** Flag set if relative branch, clear if absolute or indirect. */
1783	#define IEMBRANCHED_F_RELATIVE UINT8_C(0x04)
1784	/** Flag set if conditional branch, clear if unconditional. */
1785	#define IEMBRANCHED_F_CONDITIONAL UINT8_C(0x08)
1786	/** Flag set if it's a far branch.
1787	* @note x86 specific */
1788	#define IEMBRANCHED_F_FAR UINT8_C(0x10)
1789	/** Flag set if the stack pointer is modified. */
1790	#define IEMBRANCHED_F_STACK UINT8_C(0x20)
1791	/** Flag set if the stack pointer and (maybe) the stack segment are modified.
1792	* @note x86 specific */
1793	#define IEMBRANCHED_F_STACK_FAR UINT8_C(0x40)
1794	/** Flag set (by IEM_MC_REL_JMP_XXX) if it's a zero bytes relative jump. */
1795	#define IEMBRANCHED_F_ZERO UINT8_C(0x80)
1796	/** @} */
1797
1798
1799	/**
1800	* The per-CPU IEM state.
1801	*/
1802	typedef struct IEMCPU
1803	{
1804	/** Info status code that needs to be propagated to the IEM caller.
1805	* This cannot be passed internally, as it would complicate all success
1806	* checks within the interpreter making the code larger and almost impossible
1807	* to get right. Instead, we'll store status codes to pass on here. Each
1808	* source of these codes will perform appropriate sanity checks. */
1809	int32_t rcPassUp; /* 0x00 */
1810	/** Execution flag, IEM_F_XXX. */
1811	uint32_t fExec; /* 0x04 */
1812
1813	/** @name Decoder state.
1814	* @{ */
1815	#ifdef IEM_WITH_CODE_TLB
1816	/** The offset of the next instruction byte. */
1817	uint32_t offInstrNextByte; /* 0x08 */
1818	# if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1819	/** X86: The number of bytes available at pbInstrBuf for the current
1820	* instruction. This takes the max opcode length into account so that doesn't
1821	* need to be checked separately. */
1822	uint32_t cbInstrBuf; /* x86: 0x0c */
1823	# else
1824	/** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1825	* @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1826	uint32_t cbInstrBufTotal; /* !x86: 0x0c */
1827	# endif
1828	/** Pointer to the page containing PC, user specified buffer or abOpcode.
1829	* This can be NULL if the page isn't mappable for some reason, in which
1830	* case we'll do fallback stuff.
1831	*
1832	* If we're executing an instruction from a user specified buffer,
1833	* IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
1834	* aligned pointer but pointer to the user data.
1835	*
1836	* X86: For instructions crossing pages, this will start on the first page and
1837	* be advanced to the next page by the time we've decoded the instruction. This
1838	* therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
1839	*/
1840	uint8_t const pbInstrBuf; / 0x10 */
1841	# if ARCH_BITS == 32
1842	uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
1843	# endif
1844	/** The program counter corresponding to pbInstrBuf.
1845	* This is set to a non-canonical address when we need to invalidate it. */
1846	uint64_t uInstrBufPc; /* 0x18 */
1847	/** The guest physical address corresponding to pbInstrBuf. */
1848	RTGCPHYS GCPhysInstrBuf; /* 0x20 */
1849	# if defined(VBOX_VMM_TARGET_X86) \|\| defined(DOXYGEN_RUNNING)
1850	/** X86: The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1851	* This takes the CS segment limit into account.
1852	* @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1853	uint16_t cbInstrBufTotal; /* x86: 0x28 */
1854	/** X86: Offset into pbInstrBuf of the first byte of the current instruction.
1855	* Can be negative to efficiently handle cross page instructions. */
1856	int16_t offCurInstrStart; /* x86: 0x2a */
1857	# endif
1858
1859	# if (!defined(IEM_WITH_OPAQUE_DECODER_STATE) && defined(VBOX_VMM_TARGET_X86)) \|\| defined(DOXYGEN_RUNNING)
1860	/** X86: The prefix mask (IEM_OP_PRF_XXX). */
1861	uint32_t fPrefixes; /* x86: 0x2c */
1862	/** X86: The extra REX ModR/M register field bit (REX.R << 3). */
1863	uint8_t uRexReg; /* x86: 0x30 */
1864	/** X86: The extra REX ModR/M r/m field, SIB base and opcode reg bit
1865	* (REX.B << 3). */
1866	uint8_t uRexB; /* x86: 0x31 */
1867	/** X86: The extra REX SIB index field bit (REX.X << 3). */
1868	uint8_t uRexIndex; /* x86: 0x32 */
1869
1870	/** X86: The effective segment register (X86_SREG_XXX). */
1871	uint8_t iEffSeg; /* x86: 0x33 */
1872
1873	/** X86: The offset of the ModR/M byte relative to the start of the instruction. */
1874	uint8_t offModRm; /* x86: 0x34 */
1875
1876	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1877	/** X86: The current offset into abOpcode. */
1878	uint8_t offOpcode; /* x86: 0x35 */
1879	# else
1880	uint8_t bUnused; /* x86: 0x35 */
1881	# endif
1882	# else /* IEM_WITH_OPAQUE_DECODER_STATE \|\| !X86 */
1883	# ifdef VBOX_VMM_TARGET_X86
1884	uint8_t abOpaqueDecoderPart1[0x36 - 0x2c];
1885	# endif
1886	# endif /* IEM_WITH_OPAQUE_DECODER_STATE \|\| !X86 */
1887
1888	#else /* !IEM_WITH_CODE_TLB */
1889	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1890	/** The size of what has currently been fetched into abOpcode. */
1891	uint8_t cbOpcode; /* 0x08 */
1892
1893	# ifdef VBOX_VMM_TARGET_X86
1894	/** X86: The current offset into abOpcode. */
1895	uint8_t offOpcode; /* x86: 0x09 */
1896	/** X86: The offset of the ModR/M byte relative to the start of the
1897	* instruction. */
1898	uint8_t offModRm; /* x86: 0x0a */
1899
1900	/** X86: The effective segment register (X86_SREG_XXX). */
1901	uint8_t iEffSeg; /* x86: 0x0b */
1902
1903	/** X86: The prefix mask (IEM_OP_PRF_XXX). */
1904	uint32_t fPrefixes; /* x86: 0x0c */
1905	/** X86: The extra REX ModR/M register field bit (REX.R << 3). */
1906	uint8_t uRexReg; /* x86: 0x10 */
1907	/** X86: The extra REX ModR/M r/m field, SIB base and opcode reg bit
1908	* (REX.B << 3). */
1909	uint8_t uRexB; /* x86: 0x11 */
1910	/** X86: The extra REX SIB index field bit (REX.X << 3). */
1911	uint8_t uRexIndex; /* x86: 0x12 */
1912	# endif
1913	# else /* IEM_WITH_OPAQUE_DECODER_STATE */
1914	# ifndef VBOX_VMM_TARGET_X86
1915	uint8_t abOpaqueDecoderPart1[1];
1916	# else
1917	uint8_t abOpaqueDecoderPart1[0x13 - 0x08];
1918	# endif
1919	# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1920	#endif /* !IEM_WITH_CODE_TLB */
1921
1922	#if (!defined(IEM_WITH_OPAQUE_DECODER_STATE) && (defined(VBOX_VMM_TARGET_X86) \|\| !defined(IEM_WITH_CODE_TLB))) \
1923	\|\| defined(DOXGYEN_RUNNING)
1924	# ifdef VBOX_VMM_TARGET_X86
1925	/** X86: The effective operand mode. */
1926	IEMMODE enmEffOpSize; /* x86: 0x36, 0x13 */
1927	/** X86: The default addressing mode. */
1928	IEMMODE enmDefAddrMode; /* x86: 0x37, 0x14 */
1929	/** X86: The effective addressing mode. */
1930	IEMMODE enmEffAddrMode; /* x86: 0x38, 0x15 */
1931	/** X86: The default operand mode. */
1932	IEMMODE enmDefOpSize; /* x86: 0x39, 0x16 */
1933
1934	/** X86: Prefix index (VEX.pp) for two byte and three byte tables. */
1935	uint8_t idxPrefix; /* x86: 0x3a, 0x17 */
1936	/** X86: 3rd VEX/EVEX/XOP register.
1937	* Please use IEM_GET_EFFECTIVE_VVVV to access. */
1938	uint8_t uVex3rdReg; /* x86: 0x3b, 0x18 */
1939	/** X86: The VEX/EVEX/XOP length field. */
1940	uint8_t uVexLength; /* x86: 0x3c, 0x19 */
1941	/** X86: Additional EVEX stuff. */
1942	uint8_t fEvexStuff; /* x86: 0x3d, 0x1a */
1943
1944	# ifndef IEM_WITH_CODE_TLB
1945	/** Explicit alignment padding. */
1946	uint8_t abAlignment2a[1]; /* x86: 0x1b */
1947	# endif
1948	/** X86: The FPU opcode (FOP). */
1949	uint16_t uFpuOpcode; /* x86: 0x3e, 0x1c */
1950	# ifndef IEM_WITH_CODE_TLB
1951	/** Opcode buffer alignment padding. */
1952	uint8_t abAlignment2b[2]; /* x86: 0x1e */
1953	# endif
1954	# else /* !VBOX_VMM_TARGET_X86 */
1955	/** Opcode buffer alignment padding. */
1956	uint8_t abAlignment2b[3+4]; /* !x86: 0x09 */
1957	# endif /* !VBOX_VMM_TARGET_X86 */
1958
1959	/** The opcode bytes. */
1960	# ifdef VBOX_VMM_TARGET_X86
1961	uint8_t abOpcode[15]; /* x86: 0x40, 0x20 */
1962	# else
1963	union
1964	{
1965	uint8_t abOpcode[ 32]; /* !x86: 0x10 */
1966	uint16_t au16Opcode[16];
1967	uint32_t au32Opcode[ 8];
1968	};
1969	# endif
1970	/** Explicit alignment padding. */
1971	# ifdef VBOX_VMM_TARGET_X86
1972	# ifdef IEM_WITH_CODE_TLB
1973	//uint8_t abAlignment2c[0x4f - 0x4f]; /* x86: 0x4f */
1974	# else
1975	uint8_t abAlignment2c[0x4f - 0x2f]; /* x86: 0x2f */
1976	# endif
1977	# else
1978	uint8_t abAlignment2c[0x4f - 0x30]; /* !x86: 0x30 */
1979	# endif
1980
1981	#else /* IEM_WITH_OPAQUE_DECODER_STATE \|\| (!x86 && TLB) */
1982	# ifdef IEM_WITH_CODE_TLB
1983	# ifdef VBOX_VMM_TARGET_X86
1984	uint8_t abOpaqueDecoderPart2[0x4f - 0x36];
1985	# else
1986	uint8_t abOpaqueDecoderPart2[0x4f - 0x28];
1987	# endif
1988	# else
1989	# ifdef VBOX_VMM_TARGET_X86
1990	uint8_t abOpaqueDecoderPart2[0x4f - 0x13];
1991	# else
1992	uint8_t abOpaqueDecoderPart2[0x4f - 0x09];
1993	# endif
1994	# endif
1995	#endif /* IEM_WITH_OPAQUE_DECODER_STATE */
1996	/** @} */
1997
1998
1999	/** The number of active guest memory mappings. */
2000	uint8_t cActiveMappings; /* 0x4f, 0x4f */
2001
2002	/** Records for tracking guest memory mappings. */
2003	struct
2004	{
2005	/** The address of the mapped bytes. */
2006	R3R0PTRTYPE(void *) pv;
2007	/** The access flags (IEM_ACCESS_XXX).
2008	* IEM_ACCESS_INVALID if the entry is unused. */
2009	uint32_t fAccess;
2010	#if HC_ARCH_BITS == 64
2011	uint32_t u32Alignment4; /*< Alignment padding. /
2012	#endif
2013	} aMemMappings[3]; /* 0x50 LB 0x30 */
2014
2015	/** Locking records for the mapped memory. */
2016	union
2017	{
2018	PGMPAGEMAPLOCK Lock;
2019	uint64_t au64Padding[2];
2020	} aMemMappingLocks[3]; /* 0x80 LB 0x30 */
2021
2022	/** Bounce buffer info.
2023	* This runs in parallel to aMemMappings. */
2024	struct
2025	{
2026	/** The physical address of the first byte. */
2027	RTGCPHYS GCPhysFirst;
2028	/** The physical address of the second page. */
2029	RTGCPHYS GCPhysSecond;
2030	/** The number of bytes in the first page. */
2031	uint16_t cbFirst;
2032	/** The number of bytes in the second page. */
2033	uint16_t cbSecond;
2034	/** Whether it's unassigned memory. */
2035	bool fUnassigned;
2036	/** Explicit alignment padding. */
2037	bool afAlignment5[3];
2038	} aMemBbMappings[3]; /* 0xb0 LB 0x48 */
2039
2040	/** The flags of the current exception / interrupt.
2041	* @note X86 specific? */
2042	uint32_t fCurXcpt; /* 0xf8 */
2043	/** The current exception / interrupt.
2044	@note X86 specific? /
2045	uint8_t uCurXcpt; /* 0xfc */
2046	/** Exception / interrupt recursion depth.
2047	@note X86 specific? /
2048	int8_t cXcptRecursions; /* 0xfb */
2049
2050	/** The next unused mapping index.
2051	* @todo try find room for this up with cActiveMappings. */
2052	uint8_t iNextMapping; /* 0xfd */
2053	uint8_t abAlignment7[1];
2054
2055	/** Bounce buffer storage.
2056	* This runs in parallel to aMemMappings and aMemBbMappings. */
2057	struct
2058	{
2059	uint8_t ab[512];
2060	} aBounceBuffers[3]; /* 0x100 LB 0x600 */
2061
2062
2063	/** Pointer set jump buffer - ring-3 context. */
2064	R3PTRTYPE(jmp_buf *) pJmpBufR3;
2065	/** Pointer set jump buffer - ring-0 context. */
2066	R0PTRTYPE(jmp_buf *) pJmpBufR0;
2067
2068	/** @todo Should move this near @a fCurXcpt later. */
2069	/** The CR2 for the current exception / interrupt. */
2070	uint64_t uCurXcptCr2;
2071	/** The error code for the current exception / interrupt. */
2072	uint32_t uCurXcptErr;
2073
2074	/** @name Statistics
2075	* @{ */
2076	/** The number of instructions we've executed. */
2077	uint32_t cInstructions;
2078	/** The number of potential exits. */
2079	uint32_t cPotentialExits;
2080	/** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
2081	uint32_t cRetInstrNotImplemented;
2082	/** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
2083	uint32_t cRetAspectNotImplemented;
2084	/** Counts informational statuses returned (other than VINF_SUCCESS). */
2085	uint32_t cRetInfStatuses;
2086	/** Counts other error statuses returned. */
2087	uint32_t cRetErrStatuses;
2088	/** Number of times rcPassUp has been used. */
2089	uint32_t cRetPassUpStatus;
2090	/** Number of times RZ left with instruction commit pending for ring-3. */
2091	uint32_t cPendingCommit;
2092	/** Number of misaligned (host sense) atomic instruction accesses. */
2093	uint32_t cMisalignedAtomics;
2094	/** Number of long jumps. */
2095	uint32_t cLongJumps;
2096	/** @} */
2097
2098	/** @name Target CPU information.
2099	* @{ */
2100	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
2101	/** The target CPU. */
2102	uint8_t uTargetCpu;
2103	#else
2104	uint8_t bTargetCpuPadding;
2105	#endif
2106	/** For selecting assembly works matching the target CPU EFLAGS behaviour, see
2107	* IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
2108	* native host support and the 2nd for when there is.
2109	*
2110	* The two values are typically indexed by a g_CpumHostFeatures bit.
2111	*
2112	* This is for instance used for the BSF & BSR instructions where AMD and
2113	* Intel CPUs produce different EFLAGS. */
2114	uint8_t aidxTargetCpuEflFlavour[2];
2115
2116	/** The CPU vendor. */
2117	CPUMCPUVENDOR enmCpuVendor;
2118	/** @} */
2119
2120	/** Counts RDMSR \#GP(0) LogRel(). */
2121	uint8_t cLogRelRdMsr;
2122	/** Counts WRMSR \#GP(0) LogRel(). */
2123	uint8_t cLogRelWrMsr;
2124	/** Alignment padding. */
2125	uint8_t abAlignment9[50];
2126
2127
2128	/** @name Recompiled Exection
2129	* @{ */
2130	/** Pointer to the current translation block.
2131	* This can either be one being executed or one being compiled. */
2132	R3PTRTYPE(PIEMTB) pCurTbR3;
2133	#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
2134	/** Frame pointer for the last native TB to execute. */
2135	R3PTRTYPE(void *) pvTbFramePointerR3;
2136	#else
2137	R3PTRTYPE(void *) pvUnusedR3;
2138	#endif
2139	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2140	/** The saved host floating point control register (MXCSR on x86, FPCR on arm64)
2141	* needing restore when the TB finished, IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED indicates the TB
2142	* didn't modify it so we don't need to restore it. */
2143	# ifdef RT_ARCH_AMD64
2144	uint32_t uRegFpCtrl;
2145	/** Temporary copy of MXCSR for stmxcsr/ldmxcsr (so we don't have to fiddle with stack pointers). */
2146	uint32_t uRegMxcsrTmp;
2147	# elif defined(RT_ARCH_ARM64)
2148	uint64_t uRegFpCtrl;
2149	# else
2150	# error "Port me"
2151	# endif
2152	#else
2153	uint64_t u64Unused;
2154	#endif
2155	/** Pointer to the ring-3 TB cache for this EMT. */
2156	R3PTRTYPE(PIEMTBCACHE) pTbCacheR3;
2157	/** Pointer to the ring-3 TB lookup entry.
2158	* This either points to pTbLookupEntryDummyR3 or an actually lookuptable
2159	* entry, thus it can always safely be used w/o NULL checking. */
2160	R3PTRTYPE(PIEMTB *) ppTbLookupEntryR3;
2161	#if 0 /* unused */
2162	/** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
2163	* The TBs are based on physical addresses, so this is needed to correleated
2164	* RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
2165	uint64_t uCurTbStartPc;
2166	#endif
2167
2168	/** Number of threaded TBs executed. */
2169	uint64_t cTbExecThreaded;
2170	/** Number of native TBs executed. */
2171	uint64_t cTbExecNative;
2172
2173	/** The number of IRQ/FF checks till the next timer poll call. */
2174	uint32_t cTbsTillNextTimerPoll;
2175	/** The virtual sync time at the last timer poll call in milliseconds. */
2176	uint32_t msRecompilerPollNow;
2177	/** The virtual sync time at the last timer poll call in nanoseconds. */
2178	uint64_t nsRecompilerPollNow;
2179	/** The previous cTbsTillNextTimerPoll value. */
2180	uint32_t cTbsTillNextTimerPollPrev;
2181
2182	/** The current instruction number in a native TB.
2183	* This is set by code that may trigger an unexpected TB exit (throw/longjmp)
2184	* and will be picked up by the TB execution loop. Only used when
2185	* IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
2186	uint8_t idxTbCurInstr;
2187	/** @} */
2188
2189	/** @name Recompilation
2190	* @{ */
2191	/** Whether we need to check the opcode bytes for the current instruction.
2192	* This is set by a previous instruction if it modified memory or similar. */
2193	bool fTbCheckOpcodes;
2194	/** Indicates whether and how we just branched - IEMBRANCHED_F_XXX. */
2195	uint8_t fTbBranched;
2196	/** Set when GCPhysInstrBuf is updated because of a page crossing. */
2197	bool fTbCrossedPage;
2198	/** Whether to end the current TB. */
2199	bool fEndTb;
2200	/** Indicates that the current instruction is an STI. This is set by the
2201	* iemCImpl_sti code and subsequently cleared by the recompiler. */
2202	bool fTbCurInstrIsSti;
2203	/** Spaced reserved for recompiler data / alignment. */
2204	bool afRecompilerStuff1[1];
2205	/** Number of instructions before we need emit an IRQ check call again.
2206	* This helps making sure we don't execute too long w/o checking for
2207	* interrupts and immediately following instructions that may enable
2208	* interrupts (e.g. POPF, IRET, STI). With STI an additional hack is
2209	* required to make sure we check following the next instruction as well, see
2210	* fTbCurInstrIsSti. */
2211	uint8_t cInstrTillIrqCheck;
2212	/** The index of the last CheckIrq call during threaded recompilation. */
2213	uint16_t idxLastCheckIrqCallNo;
2214	/** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
2215	uint16_t cbOpcodesAllocated;
2216	/** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
2217	uint32_t uTbNativeRecompileAtUsedCount;
2218	/** The IEM_CIMPL_F_XXX mask for the current instruction. */
2219	uint32_t fTbCurInstr;
2220	/** The IEM_CIMPL_F_XXX mask for the previous instruction. */
2221	uint32_t fTbPrevInstr;
2222	/** Strict: Tracking skipped EFLAGS calculations. Any bits set here are
2223	* currently not up to date in EFLAGS. */
2224	uint32_t fSkippingEFlags;
2225	#if 0 /* unused */
2226	/** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set. */
2227	RTGCPHYS GCPhysInstrBufPrev;
2228	#endif
2229
2230	/** Fixed TB used for threaded recompilation.
2231	* This is allocated once with maxed-out sizes and re-used afterwards. */
2232	R3PTRTYPE(PIEMTB) pThrdCompileTbR3;
2233	/** Pointer to the ring-3 TB allocator for this EMT. */
2234	R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
2235	/** Pointer to the ring-3 executable memory allocator for this EMT. */
2236	R3PTRTYPE(struct IEMEXECMEMALLOCATOR *) pExecMemAllocatorR3;
2237	/** Pointer to the native recompiler state for ring-3. */
2238	R3PTRTYPE(struct IEMRECOMPILERSTATE *) pNativeRecompilerStateR3;
2239	/** Dummy entry for ppTbLookupEntryR3. */
2240	R3PTRTYPE(PIEMTB) pTbLookupEntryDummyR3;
2241	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2242	/** The debug code advances this register as if it was CPUMCTX::rip and we
2243	* didn't do delayed PC updating. When CPUMCTX::rip is finally updated,
2244	* the result is compared with this value. */
2245	uint64_t uPcUpdatingDebug;
2246	#elif defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
2247	/** The SSM handle used for saving threaded TBs for recompiler profiling. */
2248	R3PTRTYPE(PSSMHANDLE) pSsmThreadedTbsForProfiling;
2249	#else
2250	uint64_t u64Placeholder;
2251	#endif
2252	/**
2253	* Whether we should use the host instruction invalidation APIs of the
2254	* host OS or our own version of it (macOS). */
2255	uint8_t fHostICacheInvalidation;
2256	#define IEMNATIVE_ICACHE_F_USE_HOST_API UINT8_C(0x01) /*< Use the host API (macOS) instead of our code. /
2257	#define IEMNATIVE_ICACHE_F_END_WITH_ISH UINT8_C(0x02) /*< Whether to end with a ISH barrier (arm). /
2258	bool afRecompilerStuff2[7];
2259	/** @} */
2260
2261	/** Dummy TLB entry used for accesses to pages with databreakpoints. */
2262	IEMTLBENTRY DataBreakpointTlbe;
2263
2264	/** Threaded TB statistics: Times TB execution was broken off before reaching the end. */
2265	STAMCOUNTER StatTbThreadedExecBreaks;
2266	/** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
2267	STAMCOUNTER StatCheckIrqBreaks;
2268	/** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
2269	STAMCOUNTER StatCheckTimersBreaks;
2270	/** Statistics: Times BltIn_CheckMode breaks out of the TB. */
2271	STAMCOUNTER StatCheckModeBreaks;
2272	/** Threaded TB statistics: Times execution break on call with lookup entries. */
2273	STAMCOUNTER StatTbThreadedExecBreaksWithLookup;
2274	/** Threaded TB statistics: Times execution break on call without lookup entries. */
2275	STAMCOUNTER StatTbThreadedExecBreaksWithoutLookup;
2276	/** Statistics: Times a post jump target check missed and had to find new TB. */
2277	STAMCOUNTER StatCheckBranchMisses;
2278	/** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
2279	STAMCOUNTER StatCheckNeedCsLimChecking;
2280	/** Statistics: Times a loop was detected within a TB. */
2281	STAMCOUNTER StatTbLoopInTbDetected;
2282	/** Statistics: Times a loop back to the start of the TB was detected. */
2283	STAMCOUNTER StatTbLoopFullTbDetected;
2284	/** Statistics: Times a loop back to the start of the TB was detected, var 2. */
2285	STAMCOUNTER StatTbLoopFullTbDetected2;
2286	/** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
2287	STAMCOUNTER StatNativeExecMemInstrBufAllocFailed;
2288	/** Native TB statistics: Number of fully recompiled TBs. */
2289	STAMCOUNTER StatNativeFullyRecompiledTbs;
2290	/** TB statistics: Number of instructions per TB. */
2291	STAMPROFILE StatTbInstr;
2292	/** TB statistics: Number of TB lookup table entries per TB. */
2293	STAMPROFILE StatTbLookupEntries;
2294	/** Threaded TB statistics: Number of calls per TB. */
2295	STAMPROFILE StatTbThreadedCalls;
2296	/** Native TB statistics: Native code size per TB. */
2297	STAMPROFILE StatTbNativeCode;
2298	/** Native TB statistics: Profiling native recompilation. */
2299	STAMPROFILE StatNativeRecompilation;
2300	/** Native TB statistics: Number of calls per TB that were recompiled properly. */
2301	STAMPROFILE StatNativeCallsRecompiled;
2302	/** Native TB statistics: Number of threaded calls per TB that weren't recompiled. */
2303	STAMPROFILE StatNativeCallsThreaded;
2304	/** Native recompiled execution: TLB hits for data fetches. */
2305	STAMCOUNTER StatNativeTlbHitsForFetch;
2306	/** Native recompiled execution: TLB hits for data stores. */
2307	STAMCOUNTER StatNativeTlbHitsForStore;
2308	/** Native recompiled execution: TLB hits for stack accesses. */
2309	STAMCOUNTER StatNativeTlbHitsForStack;
2310	/** Native recompiled execution: TLB hits for mapped accesses. */
2311	STAMCOUNTER StatNativeTlbHitsForMapped;
2312	/** Native recompiled execution: Code TLB misses for new page. */
2313	STAMCOUNTER StatNativeCodeTlbMissesNewPage;
2314	/** Native recompiled execution: Code TLB hits for new page. */
2315	STAMCOUNTER StatNativeCodeTlbHitsForNewPage;
2316	/** Native recompiled execution: Code TLB misses for new page with offset. */
2317	STAMCOUNTER StatNativeCodeTlbMissesNewPageWithOffset;
2318	/** Native recompiled execution: Code TLB hits for new page with offset. */
2319	STAMCOUNTER StatNativeCodeTlbHitsForNewPageWithOffset;
2320
2321	/** Native recompiler: Number of calls to iemNativeRegAllocFindFree. */
2322	STAMCOUNTER StatNativeRegFindFree;
2323	/** Native recompiler: Number of times iemNativeRegAllocFindFree needed
2324	* to free a variable. */
2325	STAMCOUNTER StatNativeRegFindFreeVar;
2326	/** Native recompiler: Number of times iemNativeRegAllocFindFree did
2327	* not need to free any variables. */
2328	STAMCOUNTER StatNativeRegFindFreeNoVar;
2329	/** Native recompiler: Liveness info freed shadowed guest registers in
2330	* iemNativeRegAllocFindFree. */
2331	STAMCOUNTER StatNativeRegFindFreeLivenessUnshadowed;
2332	/** Native recompiler: Liveness info helped with the allocation in
2333	* iemNativeRegAllocFindFree. */
2334	STAMCOUNTER StatNativeRegFindFreeLivenessHelped;
2335
2336	/** Native recompiler: Number of times status flags calc has been skipped. */
2337	STAMCOUNTER StatNativeEflSkippedArithmetic;
2338	/** Native recompiler: Number of times status flags calc has been postponed. */
2339	STAMCOUNTER StatNativeEflPostponedArithmetic;
2340	/** Native recompiler: Total number instructions in this category. */
2341	STAMCOUNTER StatNativeEflTotalArithmetic;
2342
2343	/** Native recompiler: Number of times status flags calc has been skipped. */
2344	STAMCOUNTER StatNativeEflSkippedLogical;
2345	/** Native recompiler: Number of times status flags calc has been postponed. */
2346	STAMCOUNTER StatNativeEflPostponedLogical;
2347	/** Native recompiler: Total number instructions in this category. */
2348	STAMCOUNTER StatNativeEflTotalLogical;
2349
2350	/** Native recompiler: Number of times status flags calc has been skipped. */
2351	STAMCOUNTER StatNativeEflSkippedShift;
2352	/** Native recompiler: Number of times status flags calc has been postponed. */
2353	STAMCOUNTER StatNativeEflPostponedShift;
2354	/** Native recompiler: Total number instructions in this category. */
2355	STAMCOUNTER StatNativeEflTotalShift;
2356
2357	/** Native recompiler: Number of emits per postponement. */
2358	STAMPROFILE StatNativeEflPostponedEmits;
2359
2360	/** Native recompiler: Number of opportunities to skip EFLAGS.CF updating. */
2361	STAMCOUNTER StatNativeLivenessEflCfSkippable;
2362	/** Native recompiler: Number of opportunities to skip EFLAGS.PF updating. */
2363	STAMCOUNTER StatNativeLivenessEflPfSkippable;
2364	/** Native recompiler: Number of opportunities to skip EFLAGS.AF updating. */
2365	STAMCOUNTER StatNativeLivenessEflAfSkippable;
2366	/** Native recompiler: Number of opportunities to skip EFLAGS.ZF updating. */
2367	STAMCOUNTER StatNativeLivenessEflZfSkippable;
2368	/** Native recompiler: Number of opportunities to skip EFLAGS.SF updating. */
2369	STAMCOUNTER StatNativeLivenessEflSfSkippable;
2370	/** Native recompiler: Number of opportunities to skip EFLAGS.OF updating. */
2371	STAMCOUNTER StatNativeLivenessEflOfSkippable;
2372	/** Native recompiler: Number of required EFLAGS.CF updates. */
2373	STAMCOUNTER StatNativeLivenessEflCfRequired;
2374	/** Native recompiler: Number of required EFLAGS.PF updates. */
2375	STAMCOUNTER StatNativeLivenessEflPfRequired;
2376	/** Native recompiler: Number of required EFLAGS.AF updates. */
2377	STAMCOUNTER StatNativeLivenessEflAfRequired;
2378	/** Native recompiler: Number of required EFLAGS.ZF updates. */
2379	STAMCOUNTER StatNativeLivenessEflZfRequired;
2380	/** Native recompiler: Number of required EFLAGS.SF updates. */
2381	STAMCOUNTER StatNativeLivenessEflSfRequired;
2382	/** Native recompiler: Number of required EFLAGS.OF updates. */
2383	STAMCOUNTER StatNativeLivenessEflOfRequired;
2384	/** Native recompiler: Number of potentially delayable EFLAGS.CF updates. */
2385	STAMCOUNTER StatNativeLivenessEflCfDelayable;
2386	/** Native recompiler: Number of potentially delayable EFLAGS.PF updates. */
2387	STAMCOUNTER StatNativeLivenessEflPfDelayable;
2388	/** Native recompiler: Number of potentially delayable EFLAGS.AF updates. */
2389	STAMCOUNTER StatNativeLivenessEflAfDelayable;
2390	/** Native recompiler: Number of potentially delayable EFLAGS.ZF updates. */
2391	STAMCOUNTER StatNativeLivenessEflZfDelayable;
2392	/** Native recompiler: Number of potentially delayable EFLAGS.SF updates. */
2393	STAMCOUNTER StatNativeLivenessEflSfDelayable;
2394	/** Native recompiler: Number of potentially delayable EFLAGS.OF updates. */
2395	STAMCOUNTER StatNativeLivenessEflOfDelayable;
2396
2397	/** Native recompiler: Number of potential PC updates in total. */
2398	STAMCOUNTER StatNativePcUpdateTotal;
2399	/** Native recompiler: Number of PC updates which could be delayed. */
2400	STAMCOUNTER StatNativePcUpdateDelayed;
2401
2402	/** Native recompiler: Number of time we had complicated dirty shadow
2403	* register situations with the other branch in IEM_MC_ENDIF. */
2404	STAMCOUNTER StatNativeEndIfOtherBranchDirty;
2405
2406	/** Native recompiler: Number of calls to iemNativeSimdRegAllocFindFree. */
2407	STAMCOUNTER StatNativeSimdRegFindFree;
2408	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree needed
2409	* to free a variable. */
2410	STAMCOUNTER StatNativeSimdRegFindFreeVar;
2411	/** Native recompiler: Number of times iemNativeSimdRegAllocFindFree did
2412	* not need to free any variables. */
2413	STAMCOUNTER StatNativeSimdRegFindFreeNoVar;
2414	/** Native recompiler: Liveness info freed shadowed guest registers in
2415	* iemNativeSimdRegAllocFindFree. */
2416	STAMCOUNTER StatNativeSimdRegFindFreeLivenessUnshadowed;
2417	/** Native recompiler: Liveness info helped with the allocation in
2418	* iemNativeSimdRegAllocFindFree. */
2419	STAMCOUNTER StatNativeSimdRegFindFreeLivenessHelped;
2420
2421	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks. */
2422	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckPotential;
2423	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks. */
2424	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckPotential;
2425	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks. */
2426	STAMCOUNTER StatNativeMaybeSseXcptCheckPotential;
2427	/** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks. */
2428	STAMCOUNTER StatNativeMaybeAvxXcptCheckPotential;
2429
2430	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks omitted. */
2431	STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckOmitted;
2432	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks omitted. */
2433	STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckOmitted;
2434	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks omitted. */
2435	STAMCOUNTER StatNativeMaybeSseXcptCheckOmitted;
2436	/** Native recompiler: Number of IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks omitted. */
2437	STAMCOUNTER StatNativeMaybeAvxXcptCheckOmitted;
2438
2439	/** Native recompiler: The TB finished executing completely without jumping to a an exit label.
2440	* Not availabe in release builds. */
2441	STAMCOUNTER StatNativeTbFinished;
2442	/** Native recompiler: The TB finished executing jumping to the ReturnBreak label. */
2443	STAMCOUNTER StatNativeTbExitReturnBreak;
2444	/** Native recompiler: The TB finished executing jumping to the ReturnBreakFF label. */
2445	STAMCOUNTER StatNativeTbExitReturnBreakFF;
2446	/** Native recompiler: The TB finished executing jumping to the ReturnWithFlags label. */
2447	STAMCOUNTER StatNativeTbExitReturnWithFlags;
2448	/** Native recompiler: The TB finished executing with other non-zero status. */
2449	STAMCOUNTER StatNativeTbExitReturnOtherStatus;
2450	/** Native recompiler: The TB finished executing via throw / long jump. */
2451	STAMCOUNTER StatNativeTbExitLongJump;
2452	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2453	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2454	STAMCOUNTER StatNativeTbExitDirectLinking1NoIrq;
2455	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2456	* label, but directly jumped to the next TB, scenario \#1 with IRQ checks. */
2457	STAMCOUNTER StatNativeTbExitDirectLinking1Irq;
2458	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2459	* label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2460	STAMCOUNTER StatNativeTbExitDirectLinking2NoIrq;
2461	/** Native recompiler: The TB finished executing jumping to the ReturnBreak
2462	* label, but directly jumped to the next TB, scenario \#2 with IRQ checks. */
2463	STAMCOUNTER StatNativeTbExitDirectLinking2Irq;
2464
2465	/** Native recompiler: The TB finished executing jumping to the RaiseDe label. */
2466	STAMCOUNTER StatNativeTbExitRaiseDe;
2467	/** Native recompiler: The TB finished executing jumping to the RaiseUd label. */
2468	STAMCOUNTER StatNativeTbExitRaiseUd;
2469	/** Native recompiler: The TB finished executing jumping to the RaiseSseRelated label. */
2470	STAMCOUNTER StatNativeTbExitRaiseSseRelated;
2471	/** Native recompiler: The TB finished executing jumping to the RaiseAvxRelated label. */
2472	STAMCOUNTER StatNativeTbExitRaiseAvxRelated;
2473	/** Native recompiler: The TB finished executing jumping to the RaiseSseAvxFpRelated label. */
2474	STAMCOUNTER StatNativeTbExitRaiseSseAvxFpRelated;
2475	/** Native recompiler: The TB finished executing jumping to the RaiseNm label. */
2476	STAMCOUNTER StatNativeTbExitRaiseNm;
2477	/** Native recompiler: The TB finished executing jumping to the RaiseGp0 label. */
2478	STAMCOUNTER StatNativeTbExitRaiseGp0;
2479	/** Native recompiler: The TB finished executing jumping to the RaiseMf label. */
2480	STAMCOUNTER StatNativeTbExitRaiseMf;
2481	/** Native recompiler: The TB finished executing jumping to the RaiseXf label. */
2482	STAMCOUNTER StatNativeTbExitRaiseXf;
2483	/** Native recompiler: The TB finished executing jumping to the ObsoleteTb label. */
2484	STAMCOUNTER StatNativeTbExitObsoleteTb;
2485
2486	/** Native recompiler: Number of full TB loops (jumps from end to start). */
2487	STAMCOUNTER StatNativeTbExitLoopFullTb;
2488
2489	/** Native recompiler: Failure situations with direct linking scenario \#1.
2490	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2491	* @{ */
2492	STAMCOUNTER StatNativeTbExitDirectLinking1NoTb;
2493	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchGCPhysPc;
2494	STAMCOUNTER StatNativeTbExitDirectLinking1MismatchFlags;
2495	STAMCOUNTER StatNativeTbExitDirectLinking1PendingIrq;
2496	/** @} */
2497
2498	/** Native recompiler: Failure situations with direct linking scenario \#2.
2499	* Counter with StatNativeTbExitReturnBreak. Not in release builds.
2500	* @{ */
2501	STAMCOUNTER StatNativeTbExitDirectLinking2NoTb;
2502	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchGCPhysPc;
2503	STAMCOUNTER StatNativeTbExitDirectLinking2MismatchFlags;
2504	STAMCOUNTER StatNativeTbExitDirectLinking2PendingIrq;
2505	/** @} */
2506
2507	/** iemMemMap and iemMemMapJmp statistics.
2508	* @{ */
2509	STAMCOUNTER StatMemMapJmp;
2510	STAMCOUNTER StatMemMapNoJmp;
2511	STAMCOUNTER StatMemBounceBufferCrossPage;
2512	STAMCOUNTER StatMemBounceBufferMapPhys;
2513	/** @} */
2514
2515	/** Timer polling statistics (debug only).
2516	* @{ */
2517	STAMPROFILE StatTimerPoll;
2518	STAMPROFILE StatTimerPollPoll;
2519	STAMPROFILE StatTimerPollRun;
2520	STAMCOUNTER StatTimerPollUnchanged;
2521	STAMCOUNTER StatTimerPollTiny;
2522	STAMCOUNTER StatTimerPollDefaultCalc;
2523	STAMCOUNTER StatTimerPollMax;
2524	STAMPROFILE StatTimerPollFactorDivision;
2525	STAMPROFILE StatTimerPollFactorMultiplication;
2526	/** @} */
2527
2528
2529	STAMCOUNTER aStatAdHoc[8];
2530
2531	#ifdef IEM_WITH_TLB_TRACE
2532	/uint64_t au64Padding[0];/
2533	#else
2534	uint64_t au64Padding[2];
2535	#endif
2536
2537	#ifdef IEM_WITH_TLB_TRACE
2538	/** The end (next) trace entry. */
2539	uint32_t idxTlbTraceEntry;
2540	/** Number of trace entries allocated expressed as a power of two. */
2541	uint32_t cTlbTraceEntriesShift;
2542	/** The trace entries. */
2543	PIEMTLBTRACEENTRY paTlbTraceEntries;
2544	#endif
2545
2546	/** Data TLB.
2547	* @remarks Must be 64-byte aligned. */
2548	IEMTLB DataTlb;
2549	/** Instruction TLB.
2550	* @remarks Must be 64-byte aligned. */
2551	IEMTLB CodeTlb;
2552
2553	/** Exception statistics. */
2554	STAMCOUNTER aStatXcpts[32];
2555	/** Interrupt statistics. */
2556	uint32_t aStatInts[256];
2557
2558	#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING) && !defined(IEM_WITHOUT_INSTRUCTION_STATS)
2559	/** Instruction statistics for ring-0/raw-mode. */
2560	IEMINSTRSTATS StatsRZ;
2561	/** Instruction statistics for ring-3. */
2562	IEMINSTRSTATS StatsR3;
2563	# ifdef VBOX_WITH_IEM_RECOMPILER
2564	/** Statistics per threaded function call.
2565	* Updated by both the threaded and native recompilers. */
2566	uint32_t acThreadedFuncStats[0x6000 /24576/];
2567	# endif
2568	#endif
2569	} IEMCPU;
2570	AssertCompileMemberOffset(IEMCPU, cActiveMappings, 0x4f);
2571	AssertCompileMemberAlignment(IEMCPU, aMemMappings, 16);
2572	AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
2573	AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
2574	AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
2575	AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
2576	AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
2577
2578	/** Pointer to the per-CPU IEM state. */
2579	typedef IEMCPU *PIEMCPU;
2580	/** Pointer to the const per-CPU IEM state. */
2581	typedef IEMCPU const *PCIEMCPU;
2582
2583	/** @def IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED
2584	* Value indicating the TB didn't modified the floating point control register.
2585	* @note Neither FPCR nor MXCSR accept this as a valid value (MXCSR is not fully populated,
2586	* FPCR has the upper 32-bit reserved), so this is safe. */
2587	#if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) \|\| defined(DOXYGEN_RUNNING)
2588	# ifdef RT_ARCH_AMD64
2589	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT32_MAX
2590	# elif defined(RT_ARCH_ARM64)
2591	# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT64_MAX
2592	# else
2593	# error "Port me"
2594	# endif
2595	#endif
2596
2597	/** @def IEM_GET_CTX
2598	* Gets the guest CPU context for the calling EMT.
2599	* @returns PCPUMCTX
2600	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2601	*/
2602	#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
2603
2604	/** @def IEM_CTX_ASSERT
2605	* Asserts that the @a a_fExtrnMbz is present in the CPU context.
2606	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2607	* @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
2608	*/
2609	#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) \
2610	AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
2611	("fExtrn=%#RX64 & fExtrnMbz=%#RX64 -> %#RX64\n", \
2612	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz), (a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz) ))
2613
2614	/** @def IEM_CTX_IMPORT_RET
2615	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2616	*
2617	* Will call the keep to import the bits as needed.
2618	*
2619	* Returns on import failure.
2620	*
2621	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2622	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2623	*/
2624	#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
2625	do { \
2626	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2627	{ /* likely */ } \
2628	else \
2629	{ \
2630	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2631	AssertRCReturn(rcCtxImport, rcCtxImport); \
2632	} \
2633	} while (0)
2634
2635	/** @def IEM_CTX_IMPORT_NORET
2636	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2637	*
2638	* Will call the keep to import the bits as needed.
2639	*
2640	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2641	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2642	*/
2643	#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
2644	do { \
2645	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2646	{ /* likely */ } \
2647	else \
2648	{ \
2649	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2650	AssertLogRelRC(rcCtxImport); \
2651	} \
2652	} while (0)
2653
2654	/** @def IEM_CTX_IMPORT_JMP
2655	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2656	*
2657	* Will call the keep to import the bits as needed.
2658	*
2659	* Jumps on import failure.
2660	*
2661	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2662	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2663	*/
2664	#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
2665	do { \
2666	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2667	{ /* likely */ } \
2668	else \
2669	{ \
2670	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2671	AssertRCStmt(rcCtxImport, IEM_DO_LONGJMP(pVCpu, rcCtxImport)); \
2672	} \
2673	} while (0)
2674
2675
2676
2677	/** @def IEM_GET_TARGET_CPU
2678	* Gets the current IEMTARGETCPU value.
2679	* @returns IEMTARGETCPU value.
2680	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2681	*/
2682	#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
2683	# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
2684	#else
2685	# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
2686	#endif
2687
2688
2689	/** @def IEM_TRY_SETJMP
2690	* Wrapper around setjmp / try, hiding all the ugly differences.
2691	*
2692	* @note Use with extreme care as this is a fragile macro.
2693	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2694	* @param a_rcTarget The variable that should receive the status code in case
2695	* of a longjmp/throw.
2696	*/
2697	/** @def IEM_TRY_SETJMP_AGAIN
2698	* For when setjmp / try is used again in the same variable scope as a previous
2699	* IEM_TRY_SETJMP invocation.
2700	*/
2701	/** @def IEM_CATCH_LONGJMP_BEGIN
2702	* Start wrapper for catch / setjmp-else.
2703	*
2704	* This will set up a scope.
2705	*
2706	* @note Use with extreme care as this is a fragile macro.
2707	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2708	* @param a_rcTarget The variable that should receive the status code in case
2709	* of a longjmp/throw.
2710	*/
2711	/** @def IEM_CATCH_LONGJMP_END
2712	* End wrapper for catch / setjmp-else.
2713	*
2714	* This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the
2715	* state.
2716	*
2717	* @note Use with extreme care as this is a fragile macro.
2718	* @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2719	*/
2720	#ifdef IEM_WITH_THROW_CATCH
2721	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2722	a_rcTarget = VINF_SUCCESS; \
2723	try
2724	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2725	IEM_TRY_SETJMP(a_pVCpu, a_rcTarget)
2726	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2727	catch (int rcThrown) \
2728	{ \
2729	a_rcTarget = rcThrown
2730	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2731	} \
2732	((void)0)
2733	#else /* !IEM_WITH_THROW_CATCH */
2734	# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2735	jmp_buf JmpBuf; \
2736	jmp_buf * volatile pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2737	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2738	if ((rcStrict = setjmp(JmpBuf)) == 0)
2739	# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2740	pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2741	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2742	if ((rcStrict = setjmp(JmpBuf)) == 0)
2743	# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2744	else \
2745	{ \
2746	((void)0)
2747	# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2748	} \
2749	(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf
2750	#endif /* !IEM_WITH_THROW_CATCH */
2751
2752
2753	/**
2754	* Shared per-VM IEM data.
2755	*/
2756	typedef struct IEM
2757	{
2758	/** The VMX APIC-access page handler type. */
2759	PGMPHYSHANDLERTYPE hVmxApicAccessPage;
2760	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
2761	/** Set if the CPUID host call functionality is enabled. */
2762	bool fCpuIdHostCall;
2763	#endif
2764	} IEM;
2765
2766
2767
2768	/** @name IEM_ACCESS_XXX - Access details.
2769	* @{ */
2770	#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
2771	#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
2772	#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
2773	#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
2774	#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
2775	#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
2776	#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
2777	#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
2778	#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
2779	#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
2780	/** The writes are partial, so if initialize the bounce buffer with the
2781	* orignal RAM content. */
2782	#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
2783	/** Used in aMemMappings to indicate that the entry is bounce buffered. */
2784	#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
2785	/** Bounce buffer with ring-3 write pending, first page. */
2786	#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
2787	/** Bounce buffer with ring-3 write pending, second page. */
2788	#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
2789	/** Not locked, accessed via the TLB. */
2790	#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
2791	/** Atomic access.
2792	* This enables special alignment checks and the VINF_EM_EMULATE_SPLIT_LOCK
2793	* fallback for misaligned stuff. See @bugref{10547}. */
2794	#define IEM_ACCESS_ATOMIC UINT32_C(0x00002000)
2795	/** Valid bit mask. */
2796	#define IEM_ACCESS_VALID_MASK UINT32_C(0x00003fff)
2797	/** Shift count for the TLB flags (upper word). */
2798	#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
2799
2800	/** Atomic read+write data alias. */
2801	#define IEM_ACCESS_DATA_ATOMIC (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_ATOMIC)
2802	/** Read+write data alias. */
2803	#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2804	/** Write data alias. */
2805	#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
2806	/** Read data alias. */
2807	#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA)
2808	/** Instruction fetch alias. */
2809	#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_WHAT_CODE)
2810	/** Stack write alias. */
2811	#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2812	/** Stack read alias. */
2813	#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_STACK)
2814	/** Stack read+write alias. */
2815	#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
2816	/** Read system table alias. */
2817	#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_SYS)
2818	/** Read+write system table alias. */
2819	#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_SYS)
2820	/** @} */
2821
2822
2823	/** @def IEM_DECL_MSC_GUARD_IGNORE
2824	* Disables control flow guards checks inside a method and any function pointers
2825	* referenced by it. */
2826	#if defined(_MSC_VER) && !defined(IN_RING0)
2827	# define IEM_DECL_MSC_GUARD_IGNORE __declspec(guard(ignore))
2828	#else
2829	# define IEM_DECL_MSC_GUARD_IGNORE
2830	#endif
2831
2832	/** @def IEM_DECL_MSC_GUARD_NONE
2833	* Disables control flow guards checks inside a method and but continue track
2834	* function pointers references by it. */
2835	#if defined(_MSC_VER) && !defined(IN_RING0)
2836	# define IEM_DECL_MSC_GUARD_NONE __declspec(guard(nocf))
2837	#else
2838	# define IEM_DECL_MSC_GUARD_NONE
2839	#endif
2840
2841
2842	/** @def IEM_DECL_IMPL_TYPE
2843	* For typedef'ing an instruction implementation function.
2844	*
2845	* @param a_RetType The return type.
2846	* @param a_Name The name of the type.
2847	* @param a_ArgList The argument list enclosed in parentheses.
2848	*/
2849
2850	/** @def IEM_DECL_IMPL_DEF
2851	* For defining an instruction implementation function.
2852	*
2853	* @param a_RetType The return type.
2854	* @param a_Name The name of the type.
2855	* @param a_ArgList The argument list enclosed in parentheses.
2856	*/
2857
2858	#if defined(__GNUC__) && defined(RT_ARCH_X86)
2859	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2860	__attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
2861	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2862	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2863	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2864	__attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2865
2866	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2867	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2868	a_RetType (__fastcall a_Name) a_ArgList
2869	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2870	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2871	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2872	IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
2873
2874	#elif __cplusplus >= 201700 /* P0012R1 support */
2875	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2876	a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
2877	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2878	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2879	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2880	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
2881
2882	#else
2883	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2884	a_RetType (VBOXCALL a_Name) a_ArgList
2885	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2886	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2887	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2888	IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
2889
2890	#endif
2891
2892
2893	/** @name C instruction implementations for anything slightly complicated.
2894	* @{ */
2895
2896	/**
2897	* For typedef'ing or declaring a C instruction implementation function taking
2898	* no extra arguments.
2899	*
2900	* @param a_Name The name of the type.
2901	*/
2902	# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2903	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2904	/**
2905	* For defining a C instruction implementation function taking no extra
2906	* arguments.
2907	*
2908	* @param a_Name The name of the function
2909	*/
2910	# define IEM_CIMPL_DEF_0(a_Name) \
2911	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2912	/**
2913	* Prototype version of IEM_CIMPL_DEF_0.
2914	*/
2915	# define IEM_CIMPL_PROTO_0(a_Name) \
2916	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2917	/**
2918	* For calling a C instruction implementation function taking no extra
2919	* arguments.
2920	*
2921	* This special call macro adds default arguments to the call and allow us to
2922	* change these later.
2923	*
2924	* @param a_fn The name of the function.
2925	*/
2926	# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2927
2928	/** Type for a C instruction implementation function taking no extra
2929	* arguments. */
2930	typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
2931	/** Function pointer type for a C instruction implementation function taking
2932	* no extra arguments. */
2933	typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
2934
2935	/**
2936	* For typedef'ing or declaring a C instruction implementation function taking
2937	* one extra argument.
2938	*
2939	* @param a_Name The name of the type.
2940	* @param a_Type0 The argument type.
2941	* @param a_Arg0 The argument name.
2942	*/
2943	# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2944	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2945	/**
2946	* For defining a C instruction implementation function taking one extra
2947	* argument.
2948	*
2949	* @param a_Name The name of the function
2950	* @param a_Type0 The argument type.
2951	* @param a_Arg0 The argument name.
2952	*/
2953	# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2954	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2955	/**
2956	* Prototype version of IEM_CIMPL_DEF_1.
2957	*/
2958	# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2959	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2960	/**
2961	* For calling a C instruction implementation function taking one extra
2962	* argument.
2963	*
2964	* This special call macro adds default arguments to the call and allow us to
2965	* change these later.
2966	*
2967	* @param a_fn The name of the function.
2968	* @param a0 The name of the 1st argument.
2969	*/
2970	# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2971
2972	/**
2973	* For typedef'ing or declaring a C instruction implementation function taking
2974	* two extra arguments.
2975	*
2976	* @param a_Name The name of the type.
2977	* @param a_Type0 The type of the 1st argument
2978	* @param a_Arg0 The name of the 1st argument.
2979	* @param a_Type1 The type of the 2nd argument.
2980	* @param a_Arg1 The name of the 2nd argument.
2981	*/
2982	# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2983	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2984	/**
2985	* For defining a C instruction implementation function taking two extra
2986	* arguments.
2987	*
2988	* @param a_Name The name of the function.
2989	* @param a_Type0 The type of the 1st argument
2990	* @param a_Arg0 The name of the 1st argument.
2991	* @param a_Type1 The type of the 2nd argument.
2992	* @param a_Arg1 The name of the 2nd argument.
2993	*/
2994	# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2995	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2996	/**
2997	* Prototype version of IEM_CIMPL_DEF_2.
2998	*/
2999	# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
3000	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
3001	/**
3002	* For calling a C instruction implementation function taking two extra
3003	* arguments.
3004	*
3005	* This special call macro adds default arguments to the call and allow us to
3006	* change these later.
3007	*
3008	* @param a_fn The name of the function.
3009	* @param a0 The name of the 1st argument.
3010	* @param a1 The name of the 2nd argument.
3011	*/
3012	# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
3013
3014	/**
3015	* For typedef'ing or declaring a C instruction implementation function taking
3016	* three extra arguments.
3017	*
3018	* @param a_Name The name of the type.
3019	* @param a_Type0 The type of the 1st argument
3020	* @param a_Arg0 The name of the 1st argument.
3021	* @param a_Type1 The type of the 2nd argument.
3022	* @param a_Arg1 The name of the 2nd argument.
3023	* @param a_Type2 The type of the 3rd argument.
3024	* @param a_Arg2 The name of the 3rd argument.
3025	*/
3026	# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3027	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3028	/**
3029	* For defining a C instruction implementation function taking three extra
3030	* arguments.
3031	*
3032	* @param a_Name The name of the function.
3033	* @param a_Type0 The type of the 1st argument
3034	* @param a_Arg0 The name of the 1st argument.
3035	* @param a_Type1 The type of the 2nd argument.
3036	* @param a_Arg1 The name of the 2nd argument.
3037	* @param a_Type2 The type of the 3rd argument.
3038	* @param a_Arg2 The name of the 3rd argument.
3039	*/
3040	# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3041	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3042	/**
3043	* Prototype version of IEM_CIMPL_DEF_3.
3044	*/
3045	# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3046	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3047	/**
3048	* For calling a C instruction implementation function taking three extra
3049	* arguments.
3050	*
3051	* This special call macro adds default arguments to the call and allow us to
3052	* change these later.
3053	*
3054	* @param a_fn The name of the function.
3055	* @param a0 The name of the 1st argument.
3056	* @param a1 The name of the 2nd argument.
3057	* @param a2 The name of the 3rd argument.
3058	*/
3059	# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
3060
3061
3062	/**
3063	* For typedef'ing or declaring a C instruction implementation function taking
3064	* four extra arguments.
3065	*
3066	* @param a_Name The name of the type.
3067	* @param a_Type0 The type of the 1st argument
3068	* @param a_Arg0 The name of the 1st argument.
3069	* @param a_Type1 The type of the 2nd argument.
3070	* @param a_Arg1 The name of the 2nd argument.
3071	* @param a_Type2 The type of the 3rd argument.
3072	* @param a_Arg2 The name of the 3rd argument.
3073	* @param a_Type3 The type of the 4th argument.
3074	* @param a_Arg3 The name of the 4th argument.
3075	*/
3076	# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3077	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
3078	/**
3079	* For defining a C instruction implementation function taking four extra
3080	* arguments.
3081	*
3082	* @param a_Name The name of the function.
3083	* @param a_Type0 The type of the 1st argument
3084	* @param a_Arg0 The name of the 1st argument.
3085	* @param a_Type1 The type of the 2nd argument.
3086	* @param a_Arg1 The name of the 2nd argument.
3087	* @param a_Type2 The type of the 3rd argument.
3088	* @param a_Arg2 The name of the 3rd argument.
3089	* @param a_Type3 The type of the 4th argument.
3090	* @param a_Arg3 The name of the 4th argument.
3091	*/
3092	# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3093	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3094	a_Type2 a_Arg2, a_Type3 a_Arg3))
3095	/**
3096	* Prototype version of IEM_CIMPL_DEF_4.
3097	*/
3098	# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3099	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3100	a_Type2 a_Arg2, a_Type3 a_Arg3))
3101	/**
3102	* For calling a C instruction implementation function taking four extra
3103	* arguments.
3104	*
3105	* This special call macro adds default arguments to the call and allow us to
3106	* change these later.
3107	*
3108	* @param a_fn The name of the function.
3109	* @param a0 The name of the 1st argument.
3110	* @param a1 The name of the 2nd argument.
3111	* @param a2 The name of the 3rd argument.
3112	* @param a3 The name of the 4th argument.
3113	*/
3114	# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
3115
3116
3117	/**
3118	* For typedef'ing or declaring a C instruction implementation function taking
3119	* five extra arguments.
3120	*
3121	* @param a_Name The name of the type.
3122	* @param a_Type0 The type of the 1st argument
3123	* @param a_Arg0 The name of the 1st argument.
3124	* @param a_Type1 The type of the 2nd argument.
3125	* @param a_Arg1 The name of the 2nd argument.
3126	* @param a_Type2 The type of the 3rd argument.
3127	* @param a_Arg2 The name of the 3rd argument.
3128	* @param a_Type3 The type of the 4th argument.
3129	* @param a_Arg3 The name of the 4th argument.
3130	* @param a_Type4 The type of the 5th argument.
3131	* @param a_Arg4 The name of the 5th argument.
3132	*/
3133	# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3134	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
3135	a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
3136	a_Type3 a_Arg3, a_Type4 a_Arg4))
3137	/**
3138	* For defining a C instruction implementation function taking five extra
3139	* arguments.
3140	*
3141	* @param a_Name The name of the function.
3142	* @param a_Type0 The type of the 1st argument
3143	* @param a_Arg0 The name of the 1st argument.
3144	* @param a_Type1 The type of the 2nd argument.
3145	* @param a_Arg1 The name of the 2nd argument.
3146	* @param a_Type2 The type of the 3rd argument.
3147	* @param a_Arg2 The name of the 3rd argument.
3148	* @param a_Type3 The type of the 4th argument.
3149	* @param a_Arg3 The name of the 4th argument.
3150	* @param a_Type4 The type of the 5th argument.
3151	* @param a_Arg4 The name of the 5th argument.
3152	*/
3153	# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3154	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3155	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
3156	/**
3157	* Prototype version of IEM_CIMPL_DEF_5.
3158	*/
3159	# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3160	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3161	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
3162	/**
3163	* For calling a C instruction implementation function taking five extra
3164	* arguments.
3165	*
3166	* This special call macro adds default arguments to the call and allow us to
3167	* change these later.
3168	*
3169	* @param a_fn The name of the function.
3170	* @param a0 The name of the 1st argument.
3171	* @param a1 The name of the 2nd argument.
3172	* @param a2 The name of the 3rd argument.
3173	* @param a3 The name of the 4th argument.
3174	* @param a4 The name of the 5th argument.
3175	*/
3176	# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
3177
3178	/** @} */
3179
3180
3181	/** @name Opcode Decoder Function Types.
3182	* @{ */
3183
3184	/** @typedef PFNIEMOP
3185	* Pointer to an opcode decoder function.
3186	*/
3187
3188	/** @def FNIEMOP_DEF
3189	* Define an opcode decoder function.
3190	*
3191	* We're using macors for this so that adding and removing parameters as well as
3192	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
3193	*
3194	* @param a_Name The function name.
3195	*/
3196
3197	/** @typedef PFNIEMOPRM
3198	* Pointer to an opcode decoder function with RM byte.
3199	*/
3200
3201	/** @def FNIEMOPRM_DEF
3202	* Define an opcode decoder function with RM byte.
3203	*
3204	* We're using macors for this so that adding and removing parameters as well as
3205	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
3206	*
3207	* @param a_Name The function name.
3208	*/
3209
3210	#if defined(__GNUC__) && defined(RT_ARCH_X86)
3211	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
3212	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3213	# define FNIEMOP_DEF(a_Name) \
3214	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
3215	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3216	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3217	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3218	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3219
3220	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
3221	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
3222	typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3223	# define FNIEMOP_DEF(a_Name) \
3224	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3225	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3226	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3227	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3228	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3229
3230	#elif defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3231	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3232	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3233	# define FNIEMOP_DEF(a_Name) \
3234	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
3235	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3236	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3237	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3238	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3239
3240	#else
3241	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3242	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3243	# define FNIEMOP_DEF(a_Name) \
3244	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3245	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3246	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3247	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3248	IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3249
3250	#endif
3251	#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
3252
3253	/**
3254	* Call an opcode decoder function.
3255	*
3256	* We're using macors for this so that adding and removing parameters can be
3257	* done as we please. See FNIEMOP_DEF.
3258	*/
3259	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
3260
3261	/**
3262	* Call a common opcode decoder function taking one extra argument.
3263	*
3264	* We're using macors for this so that adding and removing parameters can be
3265	* done as we please. See FNIEMOP_DEF_1.
3266	*/
3267	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
3268
3269	/**
3270	* Call a common opcode decoder function taking one extra argument.
3271	*
3272	* We're using macors for this so that adding and removing parameters can be
3273	* done as we please. See FNIEMOP_DEF_1.
3274	*/
3275	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
3276	/** @} */
3277
3278
3279	/** @name Misc Helpers
3280	* @{ */
3281
3282	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
3283	* due to GCC lacking knowledge about the value range of a switch. */
3284	#if RT_CPLUSPLUS_PREREQ(202000)
3285	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: [[unlikely]] AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3286	#else
3287	# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3288	#endif
3289
3290	/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
3291	#if RT_CPLUSPLUS_PREREQ(202000)
3292	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: [[unlikely]] AssertFailedReturn(a_RetValue)
3293	#else
3294	# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
3295	#endif
3296
3297	/**
3298	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3299	* occation.
3300	*/
3301	#ifdef LOG_ENABLED
3302	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3303	do { \
3304	/Log/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
3305	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3306	} while (0)
3307	#else
3308	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3309	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3310	#endif
3311
3312	/**
3313	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3314	* occation using the supplied logger statement.
3315	*
3316	* @param a_LoggerArgs What to log on failure.
3317	*/
3318	#ifdef LOG_ENABLED
3319	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3320	do { \
3321	LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
3322	/LogFunc(a_LoggerArgs);/ \
3323	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3324	} while (0)
3325	#else
3326	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3327	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3328	#endif
3329
3330
3331	/** @} */
3332
3333	uint32_t iemCalcExecDbgFlagsSlow(PVMCPUCC pVCpu);
3334	VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu);
3335
3336	/** @} */
3337
3338
3339	/** @name Memory access.
3340	* @{ */
3341	VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void *ppvMem, uint8_t pbUnmapInfo,
3342	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) RT_NOEXCEPT;
3343	VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void *ppvMem, uint8_t pbUnmapInfo, size_t cbMem,
3344	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) RT_NOEXCEPT;
3345	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3346	#ifndef IN_RING3
3347	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3348	#endif
3349	void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3350	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3351
3352	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3353	void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3354	void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3355	void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3356	void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3357	void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3358
3359	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t cbAccess, uint32_t fAccess,
3360	PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
3361
3362	void iemTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3363	/** @} */
3364
3365	/*
3366	* Recompiler related stuff.
3367	*/
3368
3369	DECLHIDDEN(int) iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
3370
3371	DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
3372	uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
3373	void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree);
3374	DECLHIDDEN(void) iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
3375	void iemTbAllocatorProcessDelayedFrees(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator);
3376	void iemTbAllocatorFreeupNativeSpace(PVMCPUCC pVCpu, uint32_t cNeededInstrs);
3377	DECLHIDDEN(PIEMTBALLOCATOR) iemTbAllocatorFreeBulkStart(PVMCPUCC pVCpu);
3378	DECLHIDDEN(void) iemTbAllocatorFreeBulk(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator, PIEMTB pTb);
3379	DECLHIDDEN(const char ) iemTbFlagsToString(uint32_t fFlags, char pszBuf, size_t cbBuf) RT_NOEXCEPT;
3380	DECLHIDDEN(void) iemThreadedDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3381	#if defined(VBOX_WITH_IEM_NATIVE_RECOMPILER) && defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
3382	DECLHIDDEN(void) iemThreadedSaveTbForProfilingCleanup(PVMCPU pVCpu);
3383	#endif
3384
3385
3386	/** @todo FNIEMTHREADEDFUNC and friends may need more work... */
3387	#if defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3388	typedef VBOXSTRICTRC /__attribute__((__nothrow__))/ FNIEMTHREADEDFUNC(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3389	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3390	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3391	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3392	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3393	VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3394
3395	#else
3396	typedef VBOXSTRICTRC (FNIEMTHREADEDFUNC)(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3397	typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3398	# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3399	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3400	# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3401	IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3402	#endif
3403
3404
3405	/* Native recompiler public bits: */
3406
3407	DECLHIDDEN(PIEMTB) iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb) RT_NOEXCEPT;
3408	DECLHIDDEN(void) iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3409	int iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT;
3410	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, PIEMNATIVEINSTR *ppaExec,
3411	struct IEMNATIVEPERCHUNKCTX const **ppChunkCtx) RT_NOEXCEPT;
3412	DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq,
3413	PIEMNATIVEINSTR *ppaExec);
3414	DECLHIDDEN(void) iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3415	void iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3416	DECLASM(DECL_NO_RETURN(void)) iemNativeTbLongJmp(void *pvFramePointer, int rc) RT_NOEXCEPT;
3417	DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb);
3418	DECLHIDDEN(int) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk, struct IEMNATIVEPERCHUNKCTX const **ppCtx);
3419
3420	# ifdef VBOX_VMM_TARGET_X86
3421	# include "VMMAll/target-x86/IEMInternal-x86.h"
3422	# elif defined(VBOX_VMM_TARGET_ARMV8)
3423	//# include "VMMAll/target-armv8/IEMInternal-armv8.h"
3424	# endif
3425
3426	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3427
3428
3429	/** @} */
3430
3431	RT_C_DECLS_END
3432
3433	/* ASM-INC: %include "IEMInternalStruct.mac" */
3434
3435	#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3436

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 108702

Download in other formats: