VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h

Last change on this file was 108805, checked in by vboxsync, 7 days ago

VMM/IEM: Two more TLB todos for arm. jiraref:VBP-1598
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 163.2 KB
Line 
1/* $Id: IEMInternal.h 108805 2025-03-31 17:59:39Z vboxsync $ */
2/** @file
3 * IEM - Internal header file.
4 */
5
6/*
7 * Copyright (C) 2011-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
29#define VMM_INCLUDED_SRC_include_IEMInternal_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34#ifndef RT_IN_ASSEMBLER
35# include <VBox/vmm/cpum.h>
36# include <VBox/vmm/iem.h>
37# include <VBox/vmm/pgm.h>
38# include <VBox/vmm/stam.h>
39# include <VBox/param.h>
40
41# include <iprt/setjmp-without-sigmask.h>
42# include <iprt/list.h>
43#endif /* !RT_IN_ASSEMBLER */
44
45
46RT_C_DECLS_BEGIN
47
48
49/** @defgroup grp_iem_int Internals
50 * @ingroup grp_iem
51 * @internal
52 * @{
53 */
54
55/* Make doxygen happy w/o overcomplicating the #if checks. */
56#ifdef DOXYGEN_RUNNING
57# define IEM_WITH_THROW_CATCH
58# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
59#endif
60
61/** For expanding symbol in slickedit and other products tagging and
62 * crossreferencing IEM symbols. */
63#ifndef IEM_STATIC
64# define IEM_STATIC static
65#endif
66
67/** @def IEM_WITH_THROW_CATCH
68 * Enables using C++ throw/catch as an alternative to setjmp/longjmp in user
69 * mode code.
70 *
71 * With GCC 11.3.1 and code TLB on linux, using throw/catch instead of
72 * setjmp/long resulted in bs2-test-1 running 3.00% faster and all but on test
73 * result value improving by more than 1%. (Best out of three.)
74 *
75 * With Visual C++ 2019 and code TLB on windows, using throw/catch instead of
76 * setjmp/long resulted in bs2-test-1 running 3.68% faster and all but some of
77 * the MMIO and CPUID tests ran noticeably faster. Variation is greater than on
78 * Linux, but it should be quite a bit faster for normal code.
79 */
80#if defined(__cplusplus) && defined(IN_RING3) && (defined(__GNUC__) || defined(_MSC_VER)) /* ASM-NOINC-START */
81# define IEM_WITH_THROW_CATCH
82#endif /*ASM-NOINC-END*/
83
84/** @def IEM_WITH_ADAPTIVE_TIMER_POLLING
85 * Enables the adaptive timer polling code.
86 */
87#if defined(DOXYGEN_RUNNING) || 1
88# define IEM_WITH_ADAPTIVE_TIMER_POLLING
89#endif
90
91/** @def IEM_WITH_INTRA_TB_JUMPS
92 * Enables loop-jumps within a TB (currently only to the first call).
93 */
94#if defined(DOXYGEN_RUNNING) || 1
95# define IEM_WITH_INTRA_TB_JUMPS
96#endif
97
98/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
99 * Enables the delayed PC updating optimization (see @bugref{10373}).
100 */
101#if defined(DOXYGEN_RUNNING) || 1
102# define IEMNATIVE_WITH_DELAYED_PC_UPDATING
103#endif
104/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
105 * Enabled delayed PC updating debugging code.
106 * This is an alternative to the ARM64-only IEMNATIVE_REG_FIXED_PC_DBG. */
107#if defined(DOXYGEN_RUNNING) || 0
108# define IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
109#endif
110
111/** Enables access to even callee saved registers. */
112/*# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS*/
113
114#if defined(DOXYGEN_RUNNING) || 1
115/** @def IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
116 * Delay the writeback or dirty registers as long as possible. */
117# define IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
118#endif
119
120/** @def IEM_WITH_TLB_STATISTICS
121 * Enables all TLB statistics. */
122#if defined(VBOX_WITH_STATISTICS) || defined(DOXYGEN_RUNNING)
123# define IEM_WITH_TLB_STATISTICS
124#endif
125
126/** @def IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
127 * Enable this to use native emitters for certain SIMD FP operations. */
128#if 1 || defined(DOXYGEN_RUNNING)
129# define IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
130#endif
131
132/** @def VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
133 * Enable this to create a saved state file with the threaded translation
134 * blocks fed to the native recompiler on VCPU \#0. The resulting file can
135 * then be fed into the native recompiler for code profiling purposes.
136 * This is not a feature that should be normally be enabled! */
137#if 0 || defined(DOXYGEN_RUNNING)
138# define VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING
139#endif
140
141/** @def VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
142 * Enables a quicker alternative to throw/longjmp for IEM_DO_LONGJMP when
143 * executing native translation blocks.
144 *
145 * This exploits the fact that we save all non-volatile registers in the TB
146 * prologue and thus just need to do the same as the TB epilogue to get the
147 * effect of a longjmp/throw. Since MSC marks XMM6 thru XMM15 as
148 * non-volatile (and does something even more crazy for ARM), this probably
149 * won't work reliably on Windows. */
150#ifdef RT_ARCH_ARM64
151# ifndef RT_OS_WINDOWS
152# define VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
153# endif
154#endif
155/* ASM-NOINC-START */
156#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
157# if !defined(IN_RING3) \
158 || !defined(VBOX_WITH_IEM_RECOMPILER) \
159 || !defined(VBOX_WITH_IEM_NATIVE_RECOMPILER)
160# undef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
161# elif defined(RT_OS_WINDOWS)
162# pragma message("VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP is not safe to use on windows")
163# endif
164#endif
165
166
167/** @def IEM_DO_LONGJMP
168 *
169 * Wrapper around longjmp / throw.
170 *
171 * @param a_pVCpu The CPU handle.
172 * @param a_rc The status code jump back with / throw.
173 */
174#ifdef IEM_WITH_THROW_CATCH
175# ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
176# define IEM_DO_LONGJMP(a_pVCpu, a_rc) do { \
177 if ((a_pVCpu)->iem.s.pvTbFramePointerR3) \
178 iemNativeTbLongJmp((a_pVCpu)->iem.s.pvTbFramePointerR3, (a_rc)); \
179 throw int(a_rc); \
180 } while (0)
181# else
182# define IEM_DO_LONGJMP(a_pVCpu, a_rc) throw int(a_rc)
183# endif
184#else
185# define IEM_DO_LONGJMP(a_pVCpu, a_rc) longjmp(*(a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf), (a_rc))
186#endif
187
188/** For use with IEM function that may do a longjmp (when enabled).
189 *
190 * Visual C++ has trouble longjmp'ing from/over functions with the noexcept
191 * attribute. So, we indicate that function that may be part of a longjmp may
192 * throw "exceptions" and that the compiler should definitely not generate and
193 * std::terminate calling unwind code.
194 *
195 * Here is one example of this ending in std::terminate:
196 * @code{.txt}
19700 00000041`cadfda10 00007ffc`5d5a1f9f ucrtbase!abort+0x4e
19801 00000041`cadfda40 00007ffc`57af229a ucrtbase!terminate+0x1f
19902 00000041`cadfda70 00007ffb`eec91030 VCRUNTIME140!__std_terminate+0xa [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\ehhelpers.cpp @ 192]
20003 00000041`cadfdaa0 00007ffb`eec92c6d VCRUNTIME140_1!_CallSettingFrame+0x20 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\handlers.asm @ 50]
20104 00000041`cadfdad0 00007ffb`eec93ae5 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToState+0x241 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 1085]
20205 00000041`cadfdc00 00007ffb`eec92258 VCRUNTIME140_1!__FrameHandler4::FrameUnwindToEmptyState+0x2d [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 218]
20306 00000041`cadfdc30 00007ffb`eec940e9 VCRUNTIME140_1!__InternalCxxFrameHandler<__FrameHandler4>+0x194 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\frame.cpp @ 304]
20407 00000041`cadfdcd0 00007ffc`5f9f249f VCRUNTIME140_1!__CxxFrameHandler4+0xa9 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\risctrnsctrl.cpp @ 290]
20508 00000041`cadfdd40 00007ffc`5f980939 ntdll!RtlpExecuteHandlerForUnwind+0xf
20609 00000041`cadfdd70 00007ffc`5f9a0edd ntdll!RtlUnwindEx+0x339
2070a 00000041`cadfe490 00007ffc`57aff976 ntdll!RtlUnwind+0xcd
2080b 00000041`cadfea00 00007ffb`e1b5de01 VCRUNTIME140!__longjmp_internal+0xe6 [d:\agent\_work\1\s\src\vctools\crt\vcruntime\src\eh\amd64\longjmp.asm @ 140]
2090c (Inline Function) --------`-------- VBoxVMM!iemOpcodeGetNextU8SlowJmp+0x95 [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 1155]
2100d 00000041`cadfea50 00007ffb`e1b60f6b VBoxVMM!iemOpcodeGetNextU8Jmp+0xc1 [L:\vbox-intern\src\VBox\VMM\include\IEMInline.h @ 402]
2110e 00000041`cadfea90 00007ffb`e1cc6201 VBoxVMM!IEMExecForExits+0xdb [L:\vbox-intern\src\VBox\VMM\VMMAll\IEMAll.cpp @ 10185]
2120f 00000041`cadfec70 00007ffb`e1d0df8d VBoxVMM!EMHistoryExec+0x4f1 [L:\vbox-intern\src\VBox\VMM\VMMAll\EMAll.cpp @ 452]
21310 00000041`cadfed60 00007ffb`e1d0d4c0 VBoxVMM!nemR3WinHandleExitCpuId+0x79d [L:\vbox-intern\src\VBox\VMM\VMMAll\NEMAllNativeTemplate-win.cpp.h @ 1829] @encode
214 @endcode
215 *
216 * @see https://developercommunity.visualstudio.com/t/fragile-behavior-of-longjmp-called-from-noexcept-f/1532859
217 */
218#if defined(_MSC_VER) || defined(IEM_WITH_THROW_CATCH)
219# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT_EX(false)
220#else
221# define IEM_NOEXCEPT_MAY_LONGJMP RT_NOEXCEPT
222#endif
223/* ASM-NOINC-END */
224
225
226//#define IEM_WITH_CODE_TLB // - work in progress
227//#define IEM_WITH_DATA_TLB // - work in progress
228
229
230/** @def IEM_USE_UNALIGNED_DATA_ACCESS
231 * Use unaligned accesses instead of elaborate byte assembly. */
232#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING) /*ASM-NOINC*/
233# define IEM_USE_UNALIGNED_DATA_ACCESS
234#endif /*ASM-NOINC*/
235
236//#define IEM_LOG_MEMORY_WRITES
237
238
239/** @def IEM_CFG_TARGET_CPU
240 * The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
241 *
242 * By default we allow this to be configured by the user via the
243 * CPUM/GuestCpuName config string, but this comes at a slight cost during
244 * decoding. So, for applications of this code where there is no need to
245 * be dynamic wrt target CPU, just modify this define.
246 */
247#if !defined(IEM_CFG_TARGET_CPU) || defined(DOXYGEN_RUNNING)
248# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
249#endif
250
251
252/*
253 * X86 config.
254 */
255
256#define IEM_IMPLEMENTS_TASKSWITCH
257
258/** @def IEM_WITH_3DNOW
259 * Includes the 3DNow decoding. */
260#if !defined(IEM_WITH_3DNOW) || defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
261# ifndef IEM_WITHOUT_3DNOW
262# define IEM_WITH_3DNOW
263# endif
264#endif
265
266/** @def IEM_WITH_THREE_0F_38
267 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
268#if !defined(IEM_WITH_THREE_0F_38) || defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
269# ifndef IEM_WITHOUT_THREE_0F_38
270# define IEM_WITH_THREE_0F_38
271# endif
272#endif
273
274/** @def IEM_WITH_THREE_0F_3A
275 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
276#if !defined(IEM_WITH_THREE_0F_3A) || defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
277# ifndef IEM_WITHOUT_THREE_0F_3A
278# define IEM_WITH_THREE_0F_3A
279# endif
280#endif
281
282/** @def IEM_WITH_VEX
283 * Includes the VEX decoding. */
284#if !defined(IEM_WITH_VEX) || defined(DOXYGEN_RUNNING) /* For doxygen, set in Config.kmk. */
285# ifndef IEM_WITHOUT_VEX
286# define IEM_WITH_VEX
287# endif
288#endif
289
290
291#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
292
293# if !defined(IEM_WITHOUT_INSTRUCTION_STATS) && !defined(DOXYGEN_RUNNING)
294/** Instruction statistics. */
295typedef struct IEMINSTRSTATS
296{
297# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
298# include "IEMInstructionStatisticsTmpl.h"
299# undef IEM_DO_INSTR_STAT
300} IEMINSTRSTATS;
301#else
302struct IEMINSTRSTATS;
303typedef struct IEMINSTRSTATS IEMINSTRSTATS;
304#endif
305/** Pointer to IEM instruction statistics. */
306typedef IEMINSTRSTATS *PIEMINSTRSTATS;
307
308
309/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
310 * @{ */
311#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /**< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. */
312#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /**< Intel EFLAGS result. */
313#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /**< AMD EFLAGS result */
314#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /**< Reserved/dummy entry slot that's the same as 0. */
315#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /**< For masking the index before use. */
316/** Selects the right variant from a_aArray.
317 * pVCpu is implicit in the caller context. */
318#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
319 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
320/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
321 * be used because the host CPU does not support the operation. */
322#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
323 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
324/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
325 * array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
326 * into the two.
327 * @sa IEM_SELECT_NATIVE_OR_FALLBACK */
328#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
329# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
330 (a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
331#else
332# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
333 (a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
334#endif
335/** @} */
336
337/**
338 * Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
339 * indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
340 *
341 * On non-x86 hosts, this will shortcut to the fallback w/o checking the
342 * indicator.
343 *
344 * @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
345 */
346#if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
347# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
348 (g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
349#else
350# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
351#endif
352
353/** @name Helpers for passing C++ template arguments to an
354 * IEM_MC_NATIVE_EMIT_3/4/5 style macro.
355 * @{
356 */
357#define IEM_TEMPL_ARG_1(a1) <a1>
358#define IEM_TEMPL_ARG_2(a1, a2) <a1,a2>
359#define IEM_TEMPL_ARG_3(a1, a2, a3) <a1,a2,a3>
360/** @} */
361
362
363/**
364 * IEM TLB entry.
365 *
366 * Lookup assembly:
367 * @code{.asm}
368 ; Calculate tag.
369 mov rax, [VA]
370 shl rax, 16
371 shr rax, 16 + X86_PAGE_SHIFT
372 or rax, [uTlbRevision]
373
374 ; Do indexing.
375 movzx ecx, al
376 lea rcx, [pTlbEntries + rcx]
377
378 ; Check tag.
379 cmp [rcx + IEMTLBENTRY.uTag], rax
380 jne .TlbMiss
381
382 ; Check access.
383 mov rax, ACCESS_FLAGS | MAPPING_R3_NOT_VALID | 0xffffff00
384 and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
385 cmp rax, [uTlbPhysRev]
386 jne .TlbMiss
387
388 ; Calc address and we're done.
389 mov eax, X86_PAGE_OFFSET_MASK
390 and eax, [VA]
391 or rax, [rcx + IEMTLBENTRY.pMappingR3]
392 %ifdef VBOX_WITH_STATISTICS
393 inc qword [cTlbHits]
394 %endif
395 jmp .Done
396
397 .TlbMiss:
398 mov r8d, ACCESS_FLAGS
399 mov rdx, [VA]
400 mov rcx, [pVCpu]
401 call iemTlbTypeMiss
402 .Done:
403
404 @endcode
405 *
406 */
407typedef struct IEMTLBENTRY
408{
409 /** The TLB entry tag.
410 * Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
411 * is ASSUMING a virtual address width of 48 bits.
412 *
413 * Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
414 *
415 * The TLB lookup code uses the current TLB revision, which won't ever be zero,
416 * enabling an extremely cheap TLB invalidation most of the time. When the TLB
417 * revision wraps around though, the tags needs to be zeroed.
418 *
419 * @note Try use SHRD instruction? After seeing
420 * https://gmplib.org/~tege/x86-timing.pdf, maybe not.
421 *
422 * @todo This will need to be reorganized for 57-bit wide virtual address and
423 * PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
424 * have to move the TLB entry versioning entirely to the
425 * fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
426 * 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
427 * consumed by PCID and ASID (12 + 6 = 18).
428 * Update: Put the PCID + ASID in fFlagsAndPhysRev; that doesn't solve
429 * the 57-bit problem, though.
430 */
431 uint64_t uTag;
432 /** Access flags and physical TLB revision.
433 *
434 * @x86
435 * @parblock
436 * - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
437 * - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
438 * - Bit 2 - page tables - not user (complemented X86_PTE_US).
439 * - Bit 3 - pgm phys/virt - not directly writable.
440 * - Bit 4 - pgm phys page - not directly readable.
441 * - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
442 * - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
443 * - Bit 7 - page tables - large page.
444 * - Bit 8 - tlb entry - pMappingR3 member not valid.
445 * - Bit 9 - phys - Unassigned memory.
446 * - Bit 10 - phys - Code page.
447 * - Bits 63:11 - phys - Physical TLB revision number.
448 *
449 * We're using complemented bit meanings here because it makes it easy to check
450 * whether special action is required. For instance a user mode write access
451 * would do a "TEST fFlags, (X86_PTE_RW | X86_PTE_US | X86_PTE_D)" and a
452 * non-zero result would mean special handling needed because either it wasn't
453 * writable, or it wasn't user, or the page wasn't dirty. A user mode read
454 * access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
455 * need to check any PTE flag.
456 * @endparblock
457 *
458 * @arm
459 * @parblock
460 * - Bit 0 - stage 1+2 - not unprivileged read accessible.
461 * - Bit 1 - stage 1+2 - not unprivileged write accessible.
462 * - Bit 2 - stage 1+2 - not unprivileged execute accessible.
463 * - Bit 3 - stage 1+2 - not unprivileged guarded control stack accessible.
464 * - Bit 4 - stage 1+2 - not privileged readable accessible.
465 * - Bit 5 - stage 1+2 - not privileged writable accessible.
466 * - Bit 6 - stage 1+2 - not privileged executable accessible.
467 * - Bit 7 - stage 1+2 - not privileged guarded control stack accessible.
468 * - Bit 8 - stage 2 - no limited write access (?).
469 * - Bit 9 - stage 2 - TopLevel0 (?)
470 * - Bit 10 - stage 2 - TopLevel1 (?)
471 * - Bit 11 - stage 1+2 leaf - not dirty.
472 * - Bit 12 - stage 1+2 leaf - alternate MECID (AMEC).
473 * - Bit 13 - pgm phys page - not directly readable.
474 * - Bit 14 - pgm phys/virt - not directly writable.
475 * - Bit 15 - tlb entry - pMappingR3 member not valid.
476 * - Bit 16 - phys - Unassigned memory.
477 * - Bit 17 - phys - Code page.
478 * - Bit 18 - stage 1 leaf - NS (not-secure).
479 * - Bit 19 - stage 1 leaf - NSE (root).
480 * - Bits 20:21 - stage 1+2 - Page size.
481 * - Bits 37:22 - stage 1 reg - Address space ID (ASID).
482 * - Bits 53:38 - stage 2 reg - Virtual Machine ID (VMID).
483 * - Bits 63:54 - tlb entry - physical TLB revision number.
484 *
485 * The ASIDs and VMIDs are kept with the physical TLB revision number, so
486 * there is no extra overhead there. How the NSE:NS stuff will be handled
487 * is a question for later.
488 *
489 * The above is a preliminary sketch...
490 * @endparblock
491 *
492 * @todo arm64: Not sure if we can combine the stage 1 and 2 AMEC bits,
493 * but hope so... Doubt we'll be needing this any time soon.
494 */
495 uint64_t fFlagsAndPhysRev;
496 /** The guest physical page address. */
497 uint64_t GCPhys;
498 /** Pointer to the ring-3 mapping. */
499 R3PTRTYPE(uint8_t *) pbMappingR3;
500#if HC_ARCH_BITS == 32
501 uint32_t u32Padding1;
502#endif
503} IEMTLBENTRY;
504AssertCompileSize(IEMTLBENTRY, 32);
505/** Pointer to an IEM TLB entry. */
506typedef IEMTLBENTRY *PIEMTLBENTRY;
507/** Pointer to a const IEM TLB entry. */
508typedef IEMTLBENTRY const *PCIEMTLBENTRY;
509
510/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
511 * @{ */
512#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
513# define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /**< Page tables: Not executable. */
514# define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /**< Page tables: Not writable. */
515# define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /**< Page tables: Not user accessible (supervisor only). */
516# define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /**< Phys page: Not writable (access handler, ROM, whatever). */
517# define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /**< Phys page: Not readable (MMIO / access handler, ROM) */
518# define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /**< Phys tables: Not accessed (need to be marked accessed). */
519# define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /**< Page tables: Not dirty (needs to be made dirty on write). */
520# define IEMTLBE_F_PT_LARGE_PAGE RT_BIT_64(7) /**< Page tables: Large 2 or 4 MiB page (for flushing). */
521# define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(8) /**< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. */
522# define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(9) /**< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). */
523# define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(10) /**< Phys page: Code page. */
524# define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffff800) /**< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR */
525#endif
526#if defined(VBOX_VMM_TARGET_ARMV8) || defined(DOXYGEN_RUNNING)
527/** Stage 1+2: No unprivileged read access. */
528# define IEMTLBE_F_EFF_P_NO_READ_BIT 0
529# define IEMTLBE_F_EFF_P_NO_READ RT_BIT_64(IEMTLBE_F_EFF_P_NO_READ_BIT)
530/** Stage 1+2: No privileged write access. */
531# define IEMTLBE_F_EFF_P_NO_WRITE_BIT 1
532# define IEMTLBE_F_EFF_P_NO_WRITE RT_BIT_64(IEMTLBE_F_EFF_P_NO_WRITE_BIT)
533/** Stage 1+2: No privileged execute access. */
534# define IEMTLBE_F_EFF_P_NO_EXEC_BIT 2
535# define IEMTLBE_F_EFF_P_NO_EXEC RT_BIT_64(IEMTLBE_F_EFF_P_NO_EXEC_BIT)
536/** Stage 1+2: No privileged guard control stack access. */
537# define IEMTLBE_F_EFF_P_NO_GCS_BIT 3
538# define IEMTLBE_F_EFF_P_NO_GCS RT_BIT_64(IEMTLBE_F_EFF_P_NO_GCS_BIT)
539/** Stage 1+2: No unprivileged read access. */
540# define IEMTLBE_F_EFF_U_NO_READ_BIT 4
541# define IEMTLBE_F_EFF_U_NO_READ RT_BIT_64(IEMTLBE_F_EFF_U_NO_READ_BIT)
542/** Stage 1+2: No unprivileged write access. */
543# define IEMTLBE_F_EFF_U_NO_WRITE_BIT 5
544# define IEMTLBE_F_EFF_U_NO_WRITE RT_BIT_64(IEMTLBE_F_EFF_U_NO_WRITE_BIT)
545/** Stage 1+2: No unprivileged execute access. */
546# define IEMTLBE_F_EFF_U_NO_EXEC_BIT 6
547# define IEMTLBE_F_EFF_U_NO_EXEC RT_BIT_64(IEMTLBE_F_EFF_U_NO_EXEC_BIT)
548/** Stage 1+2: No unprivileged guard control stack access. */
549# define IEMTLBE_F_EFF_U_NO_GCS_BIT 7
550# define IEMTLBE_F_EFF_U_NO_GCS RT_BIT_64(IEMTLBE_F_EFF_U_NO_GCS_BIT)
551/** Stage 2: No limited write access. */
552# define IEMTLBE_F_S2_NO_LIM_WRITE_BIT 8
553# define IEMTLBE_F_S2_NO_LIM_WRITE RT_BIT_64(IEMTLBE_F_S2_NO_LIM_WRITE_BIT)
554/** Stage 2: TopLevel0. */
555# define IEMTLBE_F_S2_TL0_BIT 9
556# define IEMTLBE_F_S2_TL0 RT_BIT_64(IEMTLBE_F_S2_TL0_BIT)
557/** Stage 2: TopLevel1. */
558# define IEMTLBE_F_S2_TL1_BIT 10
559# define IEMTLBE_F_S2_TL1 RT_BIT_64(IEMTLBE_F_S2_TL1_BIT)
560/** Stage 1+2: Not dirty. */
561# define IEMTLBE_F_EFF_NO_DIRTY_BIT 11
562# define IEMTLBE_F_EFF_NO_DIRTY RT_BIT_64(IEMTLBE_F_EFF_NO_DIRTY_BIT)
563/** Stage 1+2: Alternative MECID. */
564# define IEMTLBE_F_EFF_AMEC_BIT 12
565# define IEMTLBE_F_EFF_AMEC RT_BIT_64(IEMTLBE_F_EFF_AMEC_BIT)
566/** Phys page: Not readable (MMIO / access handler, ROM) */
567# define IEMTLBE_F_PG_NO_READ_BIT 13
568# define IEMTLBE_F_PG_NO_READ RT_BIT_64(IEMTLBE_F_PG_NO_READ_BIT)
569/** Phys page: Not writable (access handler, ROM, whatever). */
570# define IEMTLBE_F_PG_NO_WRITE_BIT 14
571# define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(IEMTLBE_F_PG_NO_WRITE_BIT)
572/** TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. */
573# define IEMTLBE_F_NO_MAPPINGR3_BIT 15
574# define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(IEMTLBE_F_NO_MAPPINGR3_BIT)
575/** Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). */
576# define IEMTLBE_F_PG_UNASSIGNED_BIT 16
577# define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(IEMTLBE_F_PG_UNASSIGNED_BIT)
578/** Phys page: Code page. */
579# define IEMTLBE_F_PG_CODE_PAGE_BIT 17
580# define IEMTLBE_F_PG_CODE_PAGE RT_BIT_64(IEMTLBE_F_PG_CODE_PAGE_BIT)
581/** Stage 1: Non-secure bit. */
582# define IEMTLBE_F_S1_NS_BIT 18
583# define IEMTLBE_F_S1_NS RT_BIT_64(IEMTLBE_F_S1_NS_BIT)
584/** Stage 1: Non-secure extension/whatever bit. */
585# define IEMTLBE_F_S1_NSE_BIT 19
586# define IEMTLBE_F_S1_NSE RT_BIT_64(IEMTLBE_F_S1_NSE_BIT)
587/** Stage 1+2: Page size.
588 * @todo may need separate bits for each stage since they may use different
589 * page sizes. Or perhaps a single bit suffices? Also possible we
590 * don't need any of this at all because of a very very rich invalidation
591 * interface on arm. */
592# define IEMTLBE_F_EFF_SIZE_MASK UINT64(0x300000)
593/** @see IEMTLBE_F_EFF_SIZE_MASK */
594# define IEMTLBE_F_EFF_SIZE_SHIFT 20
595/** Stage 1+2: Smallest page size. */
596# define IEMTLBE_F_EFF_SIZE_L3 UINT64(0x000000)
597/** Stage 1+2: Level 2 block. */
598# define IEMTLBE_F_EFF_SIZE_L2 UINT64(0x100000)
599/** Stage 1+2: Level 1 block. */
600# define IEMTLBE_F_EFF_SIZE_L1 UINT64(0x200000)
601/** Stage 1+2: Level 0 block. */
602# define IEMTLBE_F_EFF_SIZE_L0 UINT64(0x300000)
603/** Stage 1+2: Device memory type (clear if normal memory type). */
604# define IEMTLBE_F_EFF_DEVICE_BIT 22
605# define IEMTLBE_F_EFF_DEVICE RT_BIT_64(IEMTLBE_F_EFF_DEVICE_BIT)
606/** Stage 1: Address space ID (from stage 1 root register). */
607# define IEMTLBE_F_S1_ASID (UINT64_C(0xffff) << IEMTLBE_F_S1_ASID_SHIFT)
608/** @see IEMTLBE_F_S1_ASID */
609# define IEMTLBE_F_S1_ASID_SHIFT 23
610/** Stage 2: Virtual machine ID (from stage 2 root register). */
611# define IEMTLBE_F_S2_VMID (UINT64_C(0xffff) << IEMTLBE_F_S2_VMID_SHIFT)
612/** @see IEMTLBE_F_S2_VMID */
613# define IEMTLBE_F_S2_VMID_SHIFT 39
614# ifndef DOXYGEN_RUNNING
615/** Physical revision mask. @sa IEMTLB_PHYS_REV_INCR */
616# define IEMTLBE_F_PHYS_REV UINT64_C(0xff80000000000000)
617# endif
618/** @todo ARM: We need to identify which EL the entry is from (EL1,
619 * EL2 or EL3)! */
620#endif
621/** @} */
622/** The bits set by PGMPhysIemGCPhys2PtrNoLock. */
623#define IEMTLBE_GCPHYS2PTR_MASK ( PGMIEMGCPHYS2PTR_F_NO_WRITE \
624 | PGMIEMGCPHYS2PTR_F_NO_READ \
625 | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \
626 | PGMIEMGCPHYS2PTR_F_UNASSIGNED \
627 | PGMIEMGCPHYS2PTR_F_CODE_PAGE \
628 | IEMTLBE_F_PHYS_REV )
629#if defined(VBOX_VMM_TARGET_X86) || defined(VBOX_VMM_TARGET_ARMV8)
630AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
631AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
632AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
633AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
634AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
635# ifdef VBOX_VMM_TARGET_X86
636AssertCompile(PGM_WALKINFO_BIG_PAGE == IEMTLBE_F_PT_LARGE_PAGE);
637# endif
638#endif
639
640/** Tests if the TLB entry is global (odd). */
641#define IEMTLBE_IS_GLOBAL(a_pTlbe) (((uintptr_t)(a_pTlbe) / sizeof(IEMTLBENTRY)) & 1)
642
643
644/** The TLB size (power of two).
645 * We initially chose 256 because that way we can obtain the result directly
646 * from a 8-bit register without an additional AND instruction.
647 * See also @bugref{10687}. */
648#if defined(RT_ARCH_AMD64)
649# define IEMTLB_ENTRY_COUNT 256
650# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 8
651#else
652# define IEMTLB_ENTRY_COUNT 8192
653# define IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO 13
654#endif
655AssertCompile(RT_BIT_32(IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO) == IEMTLB_ENTRY_COUNT);
656
657/** TLB slot format spec (assumes uint32_t or unsigned value). */
658#if IEMTLB_ENTRY_COUNT <= 0x100 / 2
659# define IEMTLB_SLOT_FMT "%02x"
660#elif IEMTLB_ENTRY_COUNT <= 0x1000 / 2
661# define IEMTLB_SLOT_FMT "%03x"
662#elif IEMTLB_ENTRY_COUNT <= 0x10000 / 2
663# define IEMTLB_SLOT_FMT "%04x"
664#else
665# define IEMTLB_SLOT_FMT "%05x"
666#endif
667
668/** Enable the large page bitmap TLB optimization.
669 *
670 * The idea here is to avoid scanning the full 32 KB (2MB pages, 2*512 TLB
671 * entries) or 64 KB (4MB pages, 2*1024 TLB entries) worth of TLB entries during
672 * invlpg when large pages are used, and instead just scan 128 or 256 bytes of
673 * the bmLargePage bitmap to determin which TLB entires that might be containing
674 * large pages and actually require checking.
675 *
676 * There is a good posibility of false positives since we currently don't clear
677 * the bitmap when flushing the TLB, but it should help reduce the workload when
678 * the large pages aren't fully loaded into the TLB in their entirity...
679 */
680#define IEMTLB_WITH_LARGE_PAGE_BITMAP
681
682/**
683 * An IEM TLB.
684 *
685 * We've got two of these, one for data and one for instructions.
686 */
687typedef struct IEMTLB
688{
689 /** The non-global TLB revision.
690 * This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
691 * by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
692 * the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
693 * (The revision zero indicates an invalid TLB entry.)
694 *
695 * The initial value is choosen to cause an early wraparound.
696 * @arm This includes the ASID & VM ID. */
697 uint64_t uTlbRevision;
698 /** The TLB physical address revision - shadow of PGM variable.
699 *
700 * The revision number is the top x bits (target dependent, see
701 * IEMTLBENTRY::fFlagsAndPhysRev) and is incremented by adding RT_BIT_64(x).
702 * When it wraps around and becomes zero, a rendezvous is called and each CPU
703 * wipe the IEMTLBENTRY::pMappingR3 as well as many of the
704 * IEMTLBENTRY::fFlagsAndPhysRev bits.
705 *
706 * @arm This includes the current ASID & VMID values.
707 * @todo arm: implement ASID & VMID.
708 *
709 * The initial value is choosen to cause an early wraparound.
710 *
711 * @note This is placed between the two TLB revisions because we
712 * load it in pair with one or the other on arm64.
713 */
714 uint64_t volatile uTlbPhysRev;
715 /** The global TLB revision.
716 * Same as uTlbRevision, but only increased for global flushes. */
717 uint64_t uTlbRevisionGlobal;
718
719 /** Large page tag range.
720 *
721 * This is used to avoid scanning a large page's worth of TLB entries for each
722 * INVLPG instruction, and only to do so iff we've loaded any and when the
723 * address is in this range. This is kept up to date when we loading new TLB
724 * entries.
725 */
726 struct LARGEPAGERANGE
727 {
728 /** The lowest large page address tag, UINT64_MAX if none. */
729 uint64_t uFirstTag;
730 /** The highest large page address tag (with offset mask part set), 0 if none. */
731 uint64_t uLastTag;
732 }
733 /** Large page range for non-global pages. */
734 NonGlobalLargePageRange,
735 /** Large page range for global pages. */
736 GlobalLargePageRange;
737 /** Number of non-global entries for large pages loaded since last TLB flush. */
738 uint32_t cTlbNonGlobalLargePageCurLoads;
739 /** Number of global entries for large pages loaded since last TLB flush. */
740 uint32_t cTlbGlobalLargePageCurLoads;
741
742 /* Statistics: */
743
744 /** TLB hits in IEMAll.cpp code (IEM_WITH_TLB_STATISTICS only; both).
745 * @note For the data TLB this is only used in iemMemMap and and for direct (i.e.
746 * not via safe read/write path) calls to iemMemMapJmp. */
747 uint64_t cTlbCoreHits;
748 /** Safe read/write TLB hits in iemMemMapJmp (IEM_WITH_TLB_STATISTICS
749 * only; data tlb only). */
750 uint64_t cTlbSafeHits;
751 /** TLB hits in IEMAllMemRWTmplInline.cpp.h (data + IEM_WITH_TLB_STATISTICS only). */
752 uint64_t cTlbInlineCodeHits;
753
754 /** TLB misses in IEMAll.cpp code (both).
755 * @note For the data TLB this is only used in iemMemMap and for direct (i.e.
756 * not via safe read/write path) calls to iemMemMapJmp. So,
757 * for the data TLB this more like 'other misses', while for the code
758 * TLB is all misses. */
759 uint64_t cTlbCoreMisses;
760 /** Subset of cTlbCoreMisses that results in PTE.G=1 loads (odd entries). */
761 uint64_t cTlbCoreGlobalLoads;
762 /** Safe read/write TLB misses in iemMemMapJmp (so data only). */
763 uint64_t cTlbSafeMisses;
764 /** Subset of cTlbSafeMisses that results in PTE.G=1 loads (odd entries). */
765 uint64_t cTlbSafeGlobalLoads;
766 /** Safe read path taken (data only). */
767 uint64_t cTlbSafeReadPath;
768 /** Safe write path taken (data only). */
769 uint64_t cTlbSafeWritePath;
770
771 /** @name Details for native code TLB misses.
772 * @note These counts are included in the above counters (cTlbSafeReadPath,
773 * cTlbSafeWritePath, cTlbInlineCodeHits).
774 * @{ */
775 /** TLB misses in native code due to tag mismatch. */
776 STAMCOUNTER cTlbNativeMissTag;
777 /** TLB misses in native code due to flags or physical revision mismatch. */
778 STAMCOUNTER cTlbNativeMissFlagsAndPhysRev;
779 /** TLB misses in native code due to misaligned access. */
780 STAMCOUNTER cTlbNativeMissAlignment;
781 /** TLB misses in native code due to cross page access. */
782 uint32_t cTlbNativeMissCrossPage;
783 /** TLB misses in native code due to non-canonical address. */
784 uint32_t cTlbNativeMissNonCanonical;
785 /** @} */
786
787 /** Slow read path (code only). */
788 uint32_t cTlbSlowCodeReadPath;
789
790 /** Regular TLB flush count. */
791 uint32_t cTlsFlushes;
792 /** Global TLB flush count. */
793 uint32_t cTlsGlobalFlushes;
794 /** Revision rollovers. */
795 uint32_t cTlbRevisionRollovers;
796 /** Physical revision flushes. */
797 uint32_t cTlbPhysRevFlushes;
798 /** Physical revision rollovers. */
799 uint32_t cTlbPhysRevRollovers;
800
801 /** Number of INVLPG (and similar) operations. */
802 uint32_t cTlbInvlPg;
803 /** Subset of cTlbInvlPg that involved non-global large pages. */
804 uint32_t cTlbInvlPgLargeNonGlobal;
805 /** Subset of cTlbInvlPg that involved global large pages. */
806 uint32_t cTlbInvlPgLargeGlobal;
807
808 uint32_t au32Padding[13];
809
810 /** The TLB entries.
811 * Even entries are for PTE.G=0 and uses uTlbRevision.
812 * Odd entries are for PTE.G=1 and uses uTlbRevisionGlobal. */
813 IEMTLBENTRY aEntries[IEMTLB_ENTRY_COUNT * 2];
814#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
815 /** Bitmap tracking TLB entries for large pages.
816 * This duplicates IEMTLBE_F_PT_LARGE_PAGE for each TLB entry. */
817 uint64_t bmLargePage[IEMTLB_ENTRY_COUNT * 2 / 64];
818#endif
819} IEMTLB;
820AssertCompileSizeAlignment(IEMTLB, 64);
821#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
822AssertCompile(IEMTLB_ENTRY_COUNT >= 32 /* bmLargePage ASSUMPTION */);
823#endif
824/** The width (in bits) of the address portion of the TLB tag. */
825#define IEMTLB_TAG_ADDR_WIDTH 36
826/** IEMTLB::uTlbRevision increment. */
827#define IEMTLB_REVISION_INCR RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH)
828/** IEMTLB::uTlbRevision mask. */
829#define IEMTLB_REVISION_MASK (~(RT_BIT_64(IEMTLB_TAG_ADDR_WIDTH) - 1))
830
831/** IEMTLB::uTlbPhysRev increment.
832 * @sa IEMTLBE_F_PHYS_REV */
833#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
834# define IEMTLB_PHYS_REV_INCR RT_BIT_64(11)
835#elif defined(VBOX_VMM_TARGET_ARMV8)
836# define IEMTLB_PHYS_REV_INCR RT_BIT_64(55)
837#endif
838#ifdef IEMTLBE_F_PHYS_REV
839AssertCompile(IEMTLBE_F_PHYS_REV == ~(IEMTLB_PHYS_REV_INCR - 1U));
840#endif
841
842/**
843 * Calculates the TLB tag for a virtual address but without TLB revision.
844 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
845 * @param a_pVCpu The CPU handle (for ARM targets to deal with
846 * configurable page size).
847 * @param a_GCPtr The virtual address. Must be RTGCPTR or same size or
848 * the clearing of the top 16 bits won't work (if 32-bit
849 * we'll end up with mostly zeros).
850 * @todo ARM: Support 52-bit and 56-bit address space size (FEAT_LVA,
851 * FEAT_LVA3) when we see hardware supporting such. */
852#ifdef VBOX_VMM_TARGET_ARMV8
853# define IEMTLB_CALC_TAG_NO_REV(a_pVCpu, a_GCPtr) ( (((a_GCPtr) << 16) >> (IEM_F_ARM_GET_TLB_PAGE_SHIFT(pVCpu->iem.s.fExec) + 16)) )
854#else
855# define IEMTLB_CALC_TAG_NO_REV(a_pVCpu, a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
856#endif
857/**
858 * Converts a TLB tag value into a even TLB index.
859 * @returns Index into IEMTLB::aEntries.
860 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
861 */
862#if IEMTLB_ENTRY_COUNT == 256
863# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( (uint8_t)(a_uTag) * 2U )
864#else
865# define IEMTLB_TAG_TO_EVEN_INDEX(a_uTag) ( ((a_uTag) & (IEMTLB_ENTRY_COUNT - 1U)) * 2U )
866AssertCompile(RT_IS_POWER_OF_TWO(IEMTLB_ENTRY_COUNT));
867#endif
868/**
869 * Converts a TLB tag value into an even TLB index.
870 * @returns Pointer into IEMTLB::aEntries corresponding to .
871 * @param a_pTlb The TLB.
872 * @param a_uTag Value returned by IEMTLB_CALC_TAG or
873 * IEMTLB_CALC_TAG_NO_REV.
874 */
875#define IEMTLB_TAG_TO_EVEN_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_EVEN_INDEX(a_uTag)] )
876
877/** Converts a GC address to an even TLB index. */
878#define IEMTLB_ADDR_TO_EVEN_INDEX(a_pVCpu, a_GCPtr) IEMTLB_TAG_TO_EVEN_INDEX(IEMTLB_CALC_TAG_NO_REV(a_pVCpu, a_GCPtr))
879
880
881/** @def IEM_WITH_TLB_TRACE
882 * Enables the TLB tracing.
883 * Adjust buffer size in IEMR3Init. */
884#if defined(DOXYGEN_RUNNING) || 0
885# define IEM_WITH_TLB_TRACE
886#endif
887
888#ifdef IEM_WITH_TLB_TRACE
889
890/** TLB trace entry types. */
891typedef enum : uint8_t
892{
893 kIemTlbTraceType_Invalid,
894 kIemTlbTraceType_InvlPg,
895 kIemTlbTraceType_EvictSlot,
896 kIemTlbTraceType_LargeEvictSlot,
897 kIemTlbTraceType_LargeScan,
898 kIemTlbTraceType_Flush,
899 kIemTlbTraceType_FlushGlobal, /**< x86 specific */
900 kIemTlbTraceType_Load,
901 kIemTlbTraceType_LoadGlobal, /**< x86 specific */
902 kIemTlbTraceType_Load_Cr0, /**< x86 specific */
903 kIemTlbTraceType_Load_Cr3, /**< x86 specific */
904 kIemTlbTraceType_Load_Cr4, /**< x86 specific */
905 kIemTlbTraceType_Load_Efer, /**< x86 specific */
906 kIemTlbTraceType_Irq,
907 kIemTlbTraceType_Xcpt,
908 kIemTlbTraceType_IRet, /**< x86 specific */
909 kIemTlbTraceType_Tb_Compile,
910 kIemTlbTraceType_Tb_Exec_Threaded,
911 kIemTlbTraceType_Tb_Exec_Native,
912 kIemTlbTraceType_User0,
913 kIemTlbTraceType_User1,
914 kIemTlbTraceType_User2,
915 kIemTlbTraceType_User3,
916} IEMTLBTRACETYPE;
917
918/** TLB trace entry. */
919typedef struct IEMTLBTRACEENTRY
920{
921 /** The flattened RIP for the event. */
922 uint64_t rip;
923 /** The event type. */
924 IEMTLBTRACETYPE enmType;
925 /** Byte parameter - typically used as 'bool fDataTlb'. */
926 uint8_t bParam;
927 /** 16-bit parameter value. */
928 uint16_t u16Param;
929 /** 32-bit parameter value. */
930 uint32_t u32Param;
931 /** 64-bit parameter value. */
932 uint64_t u64Param;
933 /** 64-bit parameter value. */
934 uint64_t u64Param2;
935} IEMTLBTRACEENTRY;
936AssertCompileSize(IEMTLBTRACEENTRY, 32);
937/** Pointer to a TLB trace entry. */
938typedef IEMTLBTRACEENTRY *PIEMTLBTRACEENTRY;
939/** Pointer to a const TLB trace entry. */
940typedef IEMTLBTRACEENTRY const *PCIEMTLBTRACEENTRY;
941#endif /* !IEM_WITH_TLB_TRACE */
942
943#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
944# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) \
945 iemTlbTrace(a_pVCpu, kIemTlbTraceType_InvlPg, a_GCPtr)
946# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
947 iemTlbTrace(a_pVCpu, kIemTlbTraceType_EvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
948# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) \
949 iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeEvictSlot, a_GCPtrTag, a_GCPhys, a_fDataTlb, a_idxSlot)
950# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) \
951 iemTlbTrace(a_pVCpu, kIemTlbTraceType_LargeScan, 0, 0, a_fDataTlb, (uint8_t)a_fGlobal | ((uint8_t)a_fNonGlobal << 1))
952# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) \
953 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Flush, a_uRev, 0, a_fDataTlb)
954# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) \
955 iemTlbTrace(a_pVCpu, kIemTlbTraceType_FlushGlobal, a_uRev, a_uGRev, a_fDataTlb)
956# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
957 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
958# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) \
959 iemTlbTrace(a_pVCpu, kIemTlbTraceType_LoadGlobal, a_GCPtr, a_GCPhys, a_fDataTlb, a_fTlb)
960#else
961# define IEMTLBTRACE_INVLPG(a_pVCpu, a_GCPtr) do { } while (0)
962# define IEMTLBTRACE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
963# define IEMTLBTRACE_LARGE_EVICT_SLOT(a_pVCpu, a_GCPtrTag, a_GCPhys, a_idxSlot, a_fDataTlb) do { } while (0)
964# define IEMTLBTRACE_LARGE_SCAN(a_pVCpu, a_fGlobal, a_fNonGlobal, a_fDataTlb) do { } while (0)
965# define IEMTLBTRACE_FLUSH(a_pVCpu, a_uRev, a_fDataTlb) do { } while (0)
966# define IEMTLBTRACE_FLUSH_GLOBAL(a_pVCpu, a_uRev, a_uGRev, a_fDataTlb) do { } while (0)
967# define IEMTLBTRACE_LOAD(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
968# define IEMTLBTRACE_LOAD_GLOBAL(a_pVCpu, a_GCPtr, a_GCPhys, a_fTlb, a_fDataTlb) do { } while (0)
969#endif
970
971#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
972# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr0, a_uNew, a_uOld)
973# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr3, a_uNew, a_uOld)
974# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Cr4, a_uNew, a_uOld)
975# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) iemTlbTrace(a_pVCpu, kIemTlbTraceType_Load_Efer, a_uNew, a_uOld)
976#else
977# define IEMTLBTRACE_LOAD_CR0(a_pVCpu, a_uNew, a_uOld) do { } while (0)
978# define IEMTLBTRACE_LOAD_CR3(a_pVCpu, a_uNew, a_uOld) do { } while (0)
979# define IEMTLBTRACE_LOAD_CR4(a_pVCpu, a_uNew, a_uOld) do { } while (0)
980# define IEMTLBTRACE_LOAD_EFER(a_pVCpu, a_uNew, a_uOld) do { } while (0)
981#endif
982
983#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
984# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) \
985 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Irq, a_fEFlags, 0, a_uVector, a_fFlags)
986# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) \
987 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Xcpt, a_uErr, a_uCr2, a_uVector, a_fFlags)
988# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) \
989 iemTlbTrace(a_pVCpu, kIemTlbTraceType_IRet, a_uRetRip, a_fEFlags, 0, a_uRetCs)
990#else
991# define IEMTLBTRACE_IRQ(a_pVCpu, a_uVector, a_fFlags, a_fEFlags) do { } while (0)
992# define IEMTLBTRACE_XCPT(a_pVCpu, a_uVector, a_uErr, a_uCr2, a_fFlags) do { } while (0)
993# define IEMTLBTRACE_IRET(a_pVCpu, a_uRetCs, a_uRetRip, a_fEFlags) do { } while (0)
994#endif
995
996#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
997# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) \
998 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Compile, a_GCPhysPc)
999# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) \
1000 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Threaded, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
1001# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) \
1002 iemTlbTrace(a_pVCpu, kIemTlbTraceType_Tb_Exec_Native, (a_pTb)->GCPhysPc, (uintptr_t)a_pTb, 0, (a_pTb)->cUsed)
1003#else
1004# define IEMTLBTRACE_TB_COMPILE(a_pVCpu, a_GCPhysPc) do { } while (0)
1005# define IEMTLBTRACE_TB_EXEC_THRD(a_pVCpu, a_pTb) do { } while (0)
1006# define IEMTLBTRACE_TB_EXEC_N8VE(a_pVCpu, a_pTb) do { } while (0)
1007#endif
1008
1009#if defined(IEM_WITH_TLB_TRACE) && defined(IN_RING3) && 1
1010# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
1011 iemTlbTrace(a_pVCpu, kIemTlbTraceType_User0, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
1012# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
1013 iemTlbTrace(a_pVCpu, kIemTlbTraceType_User1, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
1014# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
1015 iemTlbTrace(a_pVCpu, kIemTlbTraceType_User2, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
1016# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) \
1017 iemTlbTrace(a_pVCpu, kIemTlbTraceType_User3, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
1018#else
1019# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
1020# define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
1021# define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
1022# define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
1023#endif
1024
1025
1026/** @name IEM_MC_F_XXX - MC block flags/clues.
1027 * @note x86 specific
1028 * @todo Merge with IEM_CIMPL_F_XXX
1029 * @{ */
1030#define IEM_MC_F_ONLY_8086 RT_BIT_32(0)
1031#define IEM_MC_F_MIN_186 RT_BIT_32(1)
1032#define IEM_MC_F_MIN_286 RT_BIT_32(2)
1033#define IEM_MC_F_NOT_286_OR_OLDER IEM_MC_F_MIN_386
1034#define IEM_MC_F_MIN_386 RT_BIT_32(3)
1035#define IEM_MC_F_MIN_486 RT_BIT_32(4)
1036#define IEM_MC_F_MIN_PENTIUM RT_BIT_32(5)
1037#define IEM_MC_F_MIN_PENTIUM_II IEM_MC_F_MIN_PENTIUM
1038#define IEM_MC_F_MIN_CORE IEM_MC_F_MIN_PENTIUM
1039#define IEM_MC_F_64BIT RT_BIT_32(6)
1040#define IEM_MC_F_NOT_64BIT RT_BIT_32(7)
1041/** This is set by IEMAllN8vePython.py to indicate a variation with the
1042 * flags-clearing-and-checking. */
1043#define IEM_MC_F_WITH_FLAGS RT_BIT_32(8)
1044/** This is set by IEMAllN8vePython.py to indicate a variation without the
1045 * flags-clearing-and-checking, when there is also a variation with that.
1046 * @note Do not set this manully, it's only for python and for testing in
1047 * the native recompiler! */
1048#define IEM_MC_F_WITHOUT_FLAGS RT_BIT_32(9)
1049/** @} */
1050
1051/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
1052 *
1053 * These clues are mainly for the recompiler, so that it can emit correct code.
1054 *
1055 * They are processed by the python script and which also automatically
1056 * calculates flags for MC blocks based on the statements, extending the use of
1057 * these flags to describe MC block behavior to the recompiler core. The python
1058 * script pass the flags to the IEM_MC2_END_EMIT_CALLS macro, but mainly for
1059 * error checking purposes. The script emits the necessary fEndTb = true and
1060 * similar statements as this reduces compile time a tiny bit.
1061 *
1062 * @{ */
1063/** Flag set if direct branch, clear if absolute or indirect. */
1064#define IEM_CIMPL_F_BRANCH_DIRECT RT_BIT_32(0)
1065/** Flag set if indirect branch, clear if direct or relative.
1066 * This is also used for all system control transfers (SYSCALL, SYSRET, INT, ++)
1067 * as well as for return instructions (RET, IRET, RETF). */
1068#define IEM_CIMPL_F_BRANCH_INDIRECT RT_BIT_32(1)
1069/** Flag set if relative branch, clear if absolute or indirect. */
1070#define IEM_CIMPL_F_BRANCH_RELATIVE RT_BIT_32(2)
1071/** Flag set if conditional branch, clear if unconditional. */
1072#define IEM_CIMPL_F_BRANCH_CONDITIONAL RT_BIT_32(3)
1073/** Flag set if it's a far branch (changes CS).
1074 * @note x86 specific */
1075#define IEM_CIMPL_F_BRANCH_FAR RT_BIT_32(4)
1076/** Convenience: Testing any kind of branch. */
1077#define IEM_CIMPL_F_BRANCH_ANY (IEM_CIMPL_F_BRANCH_DIRECT | IEM_CIMPL_F_BRANCH_INDIRECT | IEM_CIMPL_F_BRANCH_RELATIVE)
1078
1079/** Execution flags may change (IEMCPU::fExec). */
1080#define IEM_CIMPL_F_MODE RT_BIT_32(5)
1081/** May change significant portions of RFLAGS.
1082 * @note x86 specific */
1083#define IEM_CIMPL_F_RFLAGS RT_BIT_32(6)
1084/** May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS.
1085 * @note x86 specific */
1086#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(7)
1087/** May trigger interrupt shadowing.
1088 * @note x86 specific */
1089#define IEM_CIMPL_F_INHIBIT_SHADOW RT_BIT_32(8)
1090/** May enable interrupts, so recheck IRQ immediately afterwards executing
1091 * the instruction. */
1092#define IEM_CIMPL_F_CHECK_IRQ_AFTER RT_BIT_32(9)
1093/** May disable interrupts, so recheck IRQ immediately before executing the
1094 * instruction. */
1095#define IEM_CIMPL_F_CHECK_IRQ_BEFORE RT_BIT_32(10)
1096/** Convenience: Check for IRQ both before and after an instruction. */
1097#define IEM_CIMPL_F_CHECK_IRQ_BEFORE_AND_AFTER (IEM_CIMPL_F_CHECK_IRQ_BEFORE | IEM_CIMPL_F_CHECK_IRQ_AFTER)
1098/** May trigger a VM exit (treated like IEM_CIMPL_F_MODE atm). */
1099#define IEM_CIMPL_F_VMEXIT RT_BIT_32(11)
1100/** May modify FPU state.
1101 * @todo Not sure if this is useful yet. */
1102#define IEM_CIMPL_F_FPU RT_BIT_32(12)
1103/** REP prefixed instruction which may yield before updating PC.
1104 * @todo Not sure if this is useful, REP functions now return non-zero
1105 * status if they don't update the PC.
1106 * @note x86 specific */
1107#define IEM_CIMPL_F_REP RT_BIT_32(13)
1108/** I/O instruction.
1109 * @todo Not sure if this is useful yet.
1110 * @note x86 specific */
1111#define IEM_CIMPL_F_IO RT_BIT_32(14)
1112/** Force end of TB after the instruction. */
1113#define IEM_CIMPL_F_END_TB RT_BIT_32(15)
1114/** Flag set if a branch may also modify the stack (push/pop return address). */
1115#define IEM_CIMPL_F_BRANCH_STACK RT_BIT_32(16)
1116/** Flag set if a branch may also modify the stack (push/pop return address)
1117 * and switch it (load/restore SS:RSP).
1118 * @note x86 specific */
1119#define IEM_CIMPL_F_BRANCH_STACK_FAR RT_BIT_32(17)
1120/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
1121#define IEM_CIMPL_F_XCPT \
1122 (IEM_CIMPL_F_BRANCH_INDIRECT | IEM_CIMPL_F_BRANCH_FAR | IEM_CIMPL_F_BRANCH_STACK_FAR \
1123 | IEM_CIMPL_F_MODE | IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_VMEXIT)
1124
1125/** The block calls a C-implementation instruction function with two implicit arguments.
1126 * Mutually exclusive with IEM_CIMPL_F_CALLS_AIMPL and
1127 * IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1128 * @note The python scripts will add this if missing. */
1129#define IEM_CIMPL_F_CALLS_CIMPL RT_BIT_32(18)
1130/** The block calls an ASM-implementation instruction function.
1131 * Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and
1132 * IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1133 * @note The python scripts will add this if missing. */
1134#define IEM_CIMPL_F_CALLS_AIMPL RT_BIT_32(19)
1135/** The block calls an ASM-implementation instruction function with an implicit
1136 * X86FXSTATE pointer argument.
1137 * Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
1138 * IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE.
1139 * @note The python scripts will add this if missing.
1140 * @note x86 specific */
1141#define IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE RT_BIT_32(20)
1142/** The block calls an ASM-implementation instruction function with an implicit
1143 * X86XSAVEAREA pointer argument.
1144 * Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL, IEM_CIMPL_F_CALLS_AIMPL and
1145 * IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
1146 * @note No different from IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE, so same value.
1147 * @note The python scripts will add this if missing.
1148 * @note x86 specific */
1149#define IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE
1150/** @} */
1151
1152
1153/** @name IEM_F_XXX - Execution mode flags (IEMCPU::fExec, IEMTB::fFlags).
1154 *
1155 * These flags are set when entering IEM and adjusted as code is executed, such
1156 * that they will always contain the current values as instructions are
1157 * finished.
1158 *
1159 * In recompiled execution mode, (most of) these flags are included in the
1160 * translation block selection key and stored in IEMTB::fFlags alongside the
1161 * IEMTB_F_XXX flags. The latter flags uses bits 31 thru 24, which are all zero
1162 * in IEMCPU::fExec.
1163 *
1164 * @{ */
1165/** Mode: The block target mode mask.
1166 * X86: CPUMODE plus protected, v86 and pre-386 indicators.
1167 * ARM: PSTATE.nRW | PSTATE.T | PSTATE.EL.
1168 * This doesn't quite overlap with SPSR_ELx when in AARCH32 mode,
1169 * but that's life. */
1170#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1171# define IEM_F_MODE_MASK UINT32_C(0x0000001f)
1172#elif defined(VBOX_VMM_TARGET_ARMV8)
1173# define IEM_F_MODE_MASK UINT32_C(0x0000003c)
1174#endif
1175
1176#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1177/** X86 Mode: The IEMMODE part of the IEMTB_F_MODE_MASK value. */
1178# define IEM_F_MODE_X86_CPUMODE_MASK UINT32_C(0x00000003)
1179/** X86 Mode: Bit used to indicating pre-386 CPU in 16-bit mode (for eliminating
1180 * conditional in EIP/IP updating), and flat wide open CS, SS, DS, and ES in
1181 * 32-bit mode (for simplifying most memory accesses). */
1182# define IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK UINT32_C(0x00000004)
1183/** X86 Mode: Bit indicating protected mode, real mode (or SMM) when not set. */
1184# define IEM_F_MODE_X86_PROT_MASK UINT32_C(0x00000008)
1185/** X86 Mode: Bit used to indicate virtual 8086 mode (only 16-bit). */
1186# define IEM_F_MODE_X86_V86_MASK UINT32_C(0x00000010)
1187
1188/** X86 Mode: 16-bit on 386 or later. */
1189# define IEM_F_MODE_X86_16BIT UINT32_C(0x00000000)
1190/** X86 Mode: 80286, 80186 and 8086/88 targetting blocks (EIP update opt). */
1191# define IEM_F_MODE_X86_16BIT_PRE_386 UINT32_C(0x00000004)
1192/** X86 Mode: 16-bit protected mode on 386 or later. */
1193# define IEM_F_MODE_X86_16BIT_PROT UINT32_C(0x00000008)
1194/** X86 Mode: 16-bit protected mode on 386 or later. */
1195# define IEM_F_MODE_X86_16BIT_PROT_PRE_386 UINT32_C(0x0000000c)
1196/** X86 Mode: 16-bit virtual 8086 protected mode (on 386 or later). */
1197# define IEM_F_MODE_X86_16BIT_PROT_V86 UINT32_C(0x00000018)
1198
1199/** X86 Mode: 32-bit on 386 or later. */
1200# define IEM_F_MODE_X86_32BIT UINT32_C(0x00000001)
1201/** X86 Mode: 32-bit mode with wide open flat CS, SS, DS and ES. */
1202# define IEM_F_MODE_X86_32BIT_FLAT UINT32_C(0x00000005)
1203/** X86 Mode: 32-bit protected mode. */
1204# define IEM_F_MODE_X86_32BIT_PROT UINT32_C(0x00000009)
1205/** X86 Mode: 32-bit protected mode with wide open flat CS, SS, DS and ES. */
1206# define IEM_F_MODE_X86_32BIT_PROT_FLAT UINT32_C(0x0000000d)
1207
1208/** X86 Mode: 64-bit (includes protected, but not the flat bit). */
1209# define IEM_F_MODE_X86_64BIT UINT32_C(0x0000000a)
1210
1211/** X86 Mode: Checks if @a a_fExec represent a FLAT mode. */
1212# define IEM_F_MODE_X86_IS_FLAT(a_fExec) ( ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT \
1213 || ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_PROT_FLAT \
1214 || ((a_fExec) & IEM_F_MODE_MASK) == IEM_F_MODE_X86_32BIT_FLAT)
1215
1216/** X86: The current protection level (CPL) shift factor. */
1217# define IEM_F_X86_CPL_SHIFT 8
1218/** X86: The current protection level (CPL) mask. */
1219# define IEM_F_X86_CPL_MASK UINT32_C(0x00000300)
1220/** X86: The current protection level (CPL) shifted mask. */
1221# define IEM_F_X86_CPL_SMASK UINT32_C(0x00000003)
1222
1223/** X86: Alignment checks enabled (CR0.AM=1 & EFLAGS.AC=1). */
1224# define IEM_F_X86_AC UINT32_C(0x00080000)
1225
1226/** X86 execution context.
1227 * The IEM_F_X86_CTX_XXX values are individual flags that can be combined (with
1228 * the exception of IEM_F_X86_CTX_NORMAL). This allows running VMs from SMM
1229 * mode. */
1230# define IEM_F_X86_CTX_MASK UINT32_C(0x0000f000)
1231/** X86 context: Plain regular execution context. */
1232# define IEM_F_X86_CTX_NORMAL UINT32_C(0x00000000)
1233/** X86 context: VT-x enabled. */
1234# define IEM_F_X86_CTX_VMX UINT32_C(0x00001000)
1235/** X86 context: AMD-V enabled. */
1236# define IEM_F_X86_CTX_SVM UINT32_C(0x00002000)
1237/** X86 context: In AMD-V or VT-x guest mode. */
1238# define IEM_F_X86_CTX_IN_GUEST UINT32_C(0x00004000)
1239/** X86 context: System management mode (SMM). */
1240# define IEM_F_X86_CTX_SMM UINT32_C(0x00008000)
1241
1242/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1243 * iemRegFinishClearingRF() most for most situations (CPUMCTX_DBG_HIT_DRX_MASK
1244 * and CPUMCTX_DBG_DBGF_MASK are covered by the IEM_F_PENDING_BRK_XXX bits
1245 * alread). */
1246
1247/** @todo Add TF+RF+INHIBIT indicator(s), so we can eliminate the conditional in
1248 * iemRegFinishClearingRF() most for most situations
1249 * (CPUMCTX_DBG_HIT_DRX_MASK and CPUMCTX_DBG_DBGF_MASK are covered by
1250 * the IEM_F_PENDING_BRK_XXX bits alread). */
1251
1252#endif /* X86 || doxygen */
1253
1254#if defined(VBOX_VMM_TARGET_ARMV8) || defined(DOXYGEN_RUNNING)
1255/** ARM Mode: Exception (privilege) level shift count. */
1256# define IEM_F_MODE_ARM_EL_SHIFT 2
1257/** ARM Mode: Exception (privilege) level mask. */
1258# define IEM_F_MODE_ARM_EL_MASK UINT32_C(0x0000000c)
1259/** ARM Mode: Exception (privilege) level shifted down mask. */
1260# define IEM_F_MODE_ARM_EL_SMASK UINT32_C(0x00000003)
1261/** ARM Mode: 32-bit (set) or 64-bit (clear) indicator (SPSR_ELx.M[4]). */
1262# define IEM_F_MODE_ARM_32BIT UINT32_C(0x00000010)
1263/** ARM Mode: Thumb mode indicator (SPSR_ELx.T). */
1264# define IEM_F_MODE_ARM_T32 UINT32_C(0x00000020)
1265
1266/** ARM Mode: Get the exception (privilege) level. */
1267# define IEM_F_MODE_ARM_GET_EL(a_fExec) (((a_fExec) >> IEM_F_MODE_ARM_EL_SHIFT) & IEM_F_MODE_ARM_EL_SMASK)
1268
1269/** ARM: The stack pointer index - not part of mode */
1270# define IEM_F_ARM_SP_IDX UINT32_C(0x00000003)
1271/** ARM: Get the SP register index. */
1272# define IEM_F_ARM_GET_SP_IDX(a_fExec) ((a_fExec) & IEM_F_ARM_SP_IDX)
1273
1274/** ARM: Normal alignment checks enabled (SCTRL_ELx.A). */
1275# define IEM_F_ARM_A UINT32_C(0x00004000)
1276/** ARM: LSE2 alignment checks enabled (~SCTRL_ELx.nAA). */
1277# define IEM_F_ARM_AA UINT32_C(0x00080000)
1278
1279/** ARM: 4K page (granule) size - stage 1. */
1280# define IEM_F_ARM_S1_PAGE_4K UINT32_C(0x00000000)
1281/** ARM: 16K page (granule) size - stage 1. */
1282# define IEM_F_ARM_S1_PAGE_16K UINT32_C(0x00000200)
1283/** ARM: 64K page (granule) size - stage 1. */
1284# define IEM_F_ARM_S1_PAGE_64K UINT32_C(0x00000400)
1285/** ARM: Mask for the stage 1 page (granule) size encoding.
1286 * This is the shift count - 12. */
1287# define IEM_F_ARM_S1_PAGE_MASK UINT32_C(0x00000700)
1288/** ARM: The shift count for the tage 1 page (granule) size encoding value. */
1289# define IEM_F_ARM_S1_PAGE_SHIFT 8
1290/** Get the current stage 1 page (granule) shift count. */
1291# define IEM_F_ARM_GET_S1_PAGE_SHIFT(a_fExec) (12 + (((a_fExec) & IEM_F_ARM_S1_PAGE_MASK) >> IEM_F_ARM_S1_PAGE_SHIFT))
1292/** Get the current stage 1 page (granule) size. */
1293# define IEM_F_ARM_GET_S1_PAGE_SIZE(a_fExec) (1 << IEM_F_ARM_GET_S1_PAGE_SHIFT(a_fExec))
1294/** Get the current stage 1 page (granule) offset mask. */
1295# define IEM_F_ARM_GET_S1_PAGE_OFFSET_MASK(a_fExec) (IEM_F_ARM_GET_S1_PAGE_SHIFT(a_fExec) - 1)
1296
1297/** Get the current TLB page (granule) shift count.
1298 * The TLB page size is the smallest of S1 and S2 page sizes.
1299 * @todo Implement stage 2 tables. */
1300# define IEM_F_ARM_GET_TLB_PAGE_SHIFT(a_fExec) IEM_F_ARM_GET_S1_PAGE_SHIFT(a_fExec)
1301/** Get the current TLB page (granule) size. */
1302# define IEM_F_ARM_GET_TLB_PAGE_SIZE(a_fExec) IEM_F_ARM_GET_S1_PAGE_SIZE(a_fExec)
1303/** Get the current TLB page (granule) offset mask. */
1304# define IEM_F_ARM_GET_TLB_PAGE_OFFSET_MASK(a_fExec) IEM_F_ARM_GET_S1_PAGE_OFFSET_MASK(a_fExec)
1305
1306/** @todo ARM: Need copy of TCR_ELx.EPD0 & TCR_ELx.EPD1. */
1307
1308#endif /* ARM || doxygen */
1309
1310/** Bypass access handlers when set. */
1311#define IEM_F_BYPASS_HANDLERS UINT32_C(0x00010000)
1312/** Have pending hardware instruction breakpoints. */
1313#define IEM_F_PENDING_BRK_INSTR UINT32_C(0x00020000)
1314/** Have pending hardware data breakpoints. */
1315#define IEM_F_PENDING_BRK_DATA UINT32_C(0x00040000)
1316
1317/** X86: Have pending hardware I/O breakpoints. */
1318#define IEM_F_PENDING_BRK_X86_IO UINT32_C(0x00000400)
1319/** X86: Disregard the lock prefix (implied or not) when set. */
1320#define IEM_F_X86_DISREGARD_LOCK UINT32_C(0x00000800)
1321
1322/** ARM: Software step (single step).
1323 * @todo make generic? */
1324#define IEM_F_ARM_SOFTWARE_STEP UINT32_C(0x00000400)
1325
1326/** Pending breakpoint mask (what iemCalcExecDbgFlags works out). */
1327#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1328# define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR | IEM_F_PENDING_BRK_DATA | IEM_F_PENDING_BRK_X86_IO)
1329#else
1330# define IEM_F_PENDING_BRK_MASK (IEM_F_PENDING_BRK_INSTR | IEM_F_PENDING_BRK_DATA)
1331#endif
1332
1333/** Caller configurable options. */
1334#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1335# define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS | IEM_F_X86_DISREGARD_LOCK)
1336#else
1337# define IEM_F_USER_OPTS (IEM_F_BYPASS_HANDLERS)
1338#endif
1339/** @} */
1340
1341
1342/** @name IEMTB_F_XXX - Translation block flags (IEMTB::fFlags).
1343 *
1344 * Extends the IEM_F_XXX flags (subject to IEMTB_F_IEM_F_MASK) to make up the
1345 * translation block flags. The combined flag mask (subject to
1346 * IEMTB_F_KEY_MASK) is used as part of the lookup key for translation blocks.
1347 *
1348 * @{ */
1349/** Mask of IEM_F_XXX flags included in IEMTB_F_XXX. */
1350#define IEMTB_F_IEM_F_MASK UINT32_C(0x00ffffff)
1351
1352/** Type: The block type mask. */
1353#define IEMTB_F_TYPE_MASK UINT32_C(0x03000000)
1354/** Type: Purly threaded recompiler (via tables). */
1355#define IEMTB_F_TYPE_THREADED UINT32_C(0x01000000)
1356/** Type: Native recompilation. */
1357#define IEMTB_F_TYPE_NATIVE UINT32_C(0x02000000)
1358
1359/** Set when we're starting the block in an "interrupt shadow".
1360 * We don't need to distingish between the two types of this mask, thus the one.
1361 * @see CPUMCTX_INHIBIT_SHADOW, CPUMIsInInterruptShadow() */
1362#define IEMTB_F_X86_INHIBIT_SHADOW UINT32_C(0x04000000)
1363/** Set when we're currently inhibiting NMIs
1364 * @see CPUMCTX_INHIBIT_NMI, CPUMAreInterruptsInhibitedByNmi() */
1365#define IEMTB_F_X86_INHIBIT_NMI UINT32_C(0x08000000)
1366
1367/** Checks that EIP/IP is wihin CS.LIM before each instruction. Used when
1368 * we're close the limit before starting a TB, as determined by
1369 * iemGetTbFlagsForCurrentPc(). */
1370#define IEMTB_F_X86_CS_LIM_CHECKS UINT32_C(0x10000000)
1371
1372/** Mask of the IEMTB_F_XXX flags that are part of the TB lookup key.
1373 *
1374 * @note We skip all of IEM_F_X86_CTX_MASK, with the exception of SMM (which we
1375 * don't implement), because we don't currently generate any context
1376 * specific code - that's all handled in CIMPL functions.
1377 *
1378 * For the threaded recompiler we don't generate any CPL specific code
1379 * either, but the native recompiler does for memory access (saves getting
1380 * the CPL from fExec and turning it into IEMTLBE_F_PT_NO_USER).
1381 * Since most OSes will not share code between rings, this shouldn't
1382 * have any real effect on TB/memory/recompiling load.
1383 */
1384#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1385# define IEMTB_F_KEY_MASK ((UINT32_MAX & ~(IEM_F_X86_CTX_MASK | IEMTB_F_TYPE_MASK)) | IEM_F_X86_CTX_SMM)
1386#else
1387# define IEMTB_F_KEY_MASK (UINT32_MAX)
1388#endif
1389/** @} */
1390
1391#ifdef VBOX_VMM_TARGET_X86
1392AssertCompile( (IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1393AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1394AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_PROT_MASK));
1395AssertCompile(!(IEM_F_MODE_X86_16BIT & IEM_F_MODE_X86_V86_MASK));
1396AssertCompile( (IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1397AssertCompile( IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1398AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_PROT_MASK));
1399AssertCompile(!(IEM_F_MODE_X86_16BIT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1400AssertCompile( (IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1401AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1402AssertCompile( IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1403AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT & IEM_F_MODE_X86_V86_MASK));
1404AssertCompile( (IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_16BIT);
1405AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1406AssertCompile( IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_PROT_MASK);
1407AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_PRE_386 & IEM_F_MODE_X86_V86_MASK));
1408AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_PROT_MASK);
1409AssertCompile(!(IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1410AssertCompile( IEM_F_MODE_X86_16BIT_PROT_V86 & IEM_F_MODE_X86_V86_MASK);
1411
1412AssertCompile( (IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1413AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1414AssertCompile(!(IEM_F_MODE_X86_32BIT & IEM_F_MODE_X86_PROT_MASK));
1415AssertCompile( (IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1416AssertCompile( IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1417AssertCompile(!(IEM_F_MODE_X86_32BIT_FLAT & IEM_F_MODE_X86_PROT_MASK));
1418AssertCompile( (IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1419AssertCompile(!(IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1420AssertCompile( IEM_F_MODE_X86_32BIT_PROT & IEM_F_MODE_X86_PROT_MASK);
1421AssertCompile( (IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_32BIT);
1422AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK);
1423AssertCompile( IEM_F_MODE_X86_32BIT_PROT_FLAT & IEM_F_MODE_X86_PROT_MASK);
1424
1425AssertCompile( (IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_CPUMODE_MASK) == IEMMODE_64BIT);
1426AssertCompile( IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_PROT_MASK);
1427AssertCompile(!(IEM_F_MODE_X86_64BIT & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
1428#endif /* VBOX_VMM_TARGET_X86 */
1429
1430#ifdef VBOX_VMM_TARGET_ARMV8
1431AssertCompile(IEM_F_MODE_ARM_EL_SHIFT == ARMV8_SPSR_EL2_AARCH64_EL_SHIFT);
1432AssertCompile(IEM_F_MODE_ARM_EL_MASK == ARMV8_SPSR_EL2_AARCH64_EL);
1433AssertCompile(IEM_F_MODE_ARM_32BIT == ARMV8_SPSR_EL2_AARCH64_M4);
1434AssertCompile(IEM_F_MODE_ARM_T32 == ARMV8_SPSR_EL2_AARCH64_T);
1435#endif
1436
1437/** Native instruction type for use with the native code generator.
1438 * This is a byte (uint8_t) for x86 and amd64 and uint32_t for the other(s). */
1439#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
1440typedef uint8_t IEMNATIVEINSTR;
1441#else
1442typedef uint32_t IEMNATIVEINSTR;
1443#endif
1444/** Pointer to a native instruction unit. */
1445typedef IEMNATIVEINSTR *PIEMNATIVEINSTR;
1446/** Pointer to a const native instruction unit. */
1447typedef IEMNATIVEINSTR const *PCIEMNATIVEINSTR;
1448
1449/**
1450 * A call for the threaded call table.
1451 */
1452typedef struct IEMTHRDEDCALLENTRY
1453{
1454 /** The function to call (IEMTHREADEDFUNCS). */
1455 uint16_t enmFunction;
1456
1457 /** Instruction number in the TB (for statistics). */
1458 uint8_t idxInstr;
1459 /** The opcode length. */
1460 uint8_t cbOpcode;
1461 /** Offset into IEMTB::pabOpcodes. */
1462 uint16_t offOpcode;
1463
1464 /** TB lookup table index (7 bits) and large size (1 bits).
1465 *
1466 * The default size is 1 entry, but for indirect calls and returns we set the
1467 * top bit and allocate 4 (IEM_TB_LOOKUP_TAB_LARGE_SIZE) entries. The large
1468 * tables uses RIP for selecting the entry to use, as it is assumed a hash table
1469 * lookup isn't that slow compared to sequentially trying out 4 TBs.
1470 *
1471 * By default lookup table entry 0 for a TB is reserved as a fallback for
1472 * calltable entries w/o explicit entreis, so this member will be non-zero if
1473 * there is a lookup entry associated with this call.
1474 *
1475 * @sa IEM_TB_LOOKUP_TAB_GET_SIZE, IEM_TB_LOOKUP_TAB_GET_IDX
1476 */
1477 uint8_t uTbLookup;
1478
1479 /** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
1480 uint8_t fFlags;
1481
1482 /** Generic parameters.
1483 * @todo ARM: Hope we can get away with one param here... */
1484 uint64_t auParams[3];
1485} IEMTHRDEDCALLENTRY;
1486AssertCompileSize(IEMTHRDEDCALLENTRY, sizeof(uint64_t) * 4);
1487/** Pointer to a threaded call entry. */
1488typedef struct IEMTHRDEDCALLENTRY *PIEMTHRDEDCALLENTRY;
1489/** Pointer to a const threaded call entry. */
1490typedef IEMTHRDEDCALLENTRY const *PCIEMTHRDEDCALLENTRY;
1491
1492/** The number of TB lookup table entries for a large allocation
1493 * (IEMTHRDEDCALLENTRY::uTbLookup bit 7 set). */
1494#define IEM_TB_LOOKUP_TAB_LARGE_SIZE 4
1495/** Get the lookup table size from IEMTHRDEDCALLENTRY::uTbLookup. */
1496#define IEM_TB_LOOKUP_TAB_GET_SIZE(a_uTbLookup) (!((a_uTbLookup) & 0x80) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE)
1497/** Get the first lookup table index from IEMTHRDEDCALLENTRY::uTbLookup. */
1498#define IEM_TB_LOOKUP_TAB_GET_IDX(a_uTbLookup) ((a_uTbLookup) & 0x7f)
1499/** Get the lookup table index from IEMTHRDEDCALLENTRY::uTbLookup and PC. */
1500#define IEM_TB_LOOKUP_TAB_GET_IDX_WITH_PC(a_uTbLookup, a_Pc) \
1501 (!((a_uTbLookup) & 0x80) ? (a_uTbLookup) & 0x7f : ((a_uTbLookup) & 0x7f) + ((a_Pc) & (IEM_TB_LOOKUP_TAB_LARGE_SIZE - 1)) )
1502
1503/** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
1504#define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge) ((a_idxTable) | ((a_fLarge) ? 0x80 : 0))
1505
1506
1507/** The call entry is a jump target. */
1508#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET UINT8_C(0x01)
1509
1510
1511/**
1512 * Native IEM TB 'function' typedef.
1513 *
1514 * This will throw/longjmp on occation.
1515 *
1516 * @note AMD64 doesn't have that many non-volatile registers and does sport
1517 * 32-bit address displacments, so we don't need pCtx.
1518 *
1519 * On ARM64 pCtx allows us to directly address the whole register
1520 * context without requiring a separate indexing register holding the
1521 * offset. This saves an instruction loading the offset for each guest
1522 * CPU context access, at the cost of a non-volatile register.
1523 * Fortunately, ARM64 has quite a lot more registers.
1524 */
1525typedef
1526#ifdef RT_ARCH_AMD64
1527int FNIEMTBNATIVE(PVMCPUCC pVCpu)
1528#else
1529int FNIEMTBNATIVE(PVMCPUCC pVCpu, PCPUMCTX pCtx)
1530#endif
1531#if RT_CPLUSPLUS_PREREQ(201700)
1532 IEM_NOEXCEPT_MAY_LONGJMP
1533#endif
1534 ;
1535/** Pointer to a native IEM TB entry point function.
1536 * This will throw/longjmp on occation. */
1537typedef FNIEMTBNATIVE *PFNIEMTBNATIVE;
1538
1539
1540/**
1541 * Translation block.
1542 *
1543 * The current plan is to just keep TBs and associated lookup hash table private
1544 * to each VCpu as that simplifies TB removal greatly (no races) and generally
1545 * avoids using expensive atomic primitives for updating lists and stuff.
1546 */
1547#pragma pack(2) /* to prevent the Thrd structure from being padded unnecessarily */
1548typedef struct IEMTB
1549{
1550 /** Next block with the same hash table entry. */
1551 struct IEMTB *pNext;
1552 /** Usage counter. */
1553 uint32_t cUsed;
1554 /** The IEMCPU::msRecompilerPollNow last time it was used. */
1555 uint32_t msLastUsed;
1556
1557 /** @name What uniquely identifies the block.
1558 * @{ */
1559 RTGCPHYS GCPhysPc;
1560 /** IEMTB_F_XXX (i.e. IEM_F_XXX ++). */
1561 uint32_t fFlags;
1562 union
1563 {
1564 struct
1565 {
1566 /**< Relevant CS X86DESCATTR_XXX bits. */
1567 uint16_t fAttr;
1568 } x86;
1569 };
1570 /** @} */
1571
1572 /** Number of opcode ranges. */
1573 uint8_t cRanges;
1574 /** Statistics: Number of instructions in the block. */
1575 uint8_t cInstructions;
1576
1577 /** Type specific info. */
1578 union
1579 {
1580 struct
1581 {
1582 /** The call sequence table. */
1583 PIEMTHRDEDCALLENTRY paCalls;
1584 /** Number of calls in paCalls. */
1585 uint16_t cCalls;
1586 /** Number of calls allocated. */
1587 uint16_t cAllocated;
1588 } Thrd;
1589 struct
1590 {
1591 /** The native instructions (PFNIEMTBNATIVE). */
1592 PIEMNATIVEINSTR paInstructions;
1593 /** Number of instructions pointed to by paInstructions. */
1594 uint32_t cInstructions;
1595 } Native;
1596 /** Generic view for zeroing when freeing. */
1597 struct
1598 {
1599 uintptr_t uPtr;
1600 uint32_t uData;
1601 } Gen;
1602 };
1603
1604 /** The allocation chunk this TB belongs to. */
1605 uint8_t idxAllocChunk;
1606 /** The number of entries in the lookup table.
1607 * Because we're out of space, the TB lookup table is located before the
1608 * opcodes pointed to by pabOpcodes. */
1609 uint8_t cTbLookupEntries;
1610
1611 /** Number of bytes of opcodes stored in pabOpcodes.
1612 * @todo this field isn't really needed, aRanges keeps the actual info. */
1613 uint16_t cbOpcodes;
1614 /** Pointer to the opcode bytes this block was recompiled from.
1615 * This also points to the TB lookup table, which starts cTbLookupEntries
1616 * entries before the opcodes (we don't have room atm for another point). */
1617 uint8_t *pabOpcodes;
1618
1619 union
1620 {
1621 /** Native recompilation debug info if enabled.
1622 * This is only generated by the native recompiler. */
1623 struct IEMTBDBG *pDbgInfo;
1624 /** For threaded TBs and natives when debug info is disabled, this is the flat
1625 * PC corresponding to GCPhysPc. */
1626 RTGCPTR FlatPc;
1627 };
1628
1629 /* --- 64 byte cache line end --- */
1630
1631 /** Opcode ranges.
1632 *
1633 * The opcode checkers and maybe TLB loading functions will use this to figure
1634 * out what to do. The parameter will specify an entry and the opcode offset to
1635 * start at and the minimum number of bytes to verify (instruction length).
1636 *
1637 * When VT-x and AMD-V looks up the opcode bytes for an exitting instruction,
1638 * they'll first translate RIP (+ cbInstr - 1) to a physical address using the
1639 * code TLB (must have a valid entry for that address) and scan the ranges to
1640 * locate the corresponding opcodes. Probably.
1641 */
1642 struct IEMTBOPCODERANGE
1643 {
1644 /** Offset within pabOpcodes. */
1645 uint16_t offOpcodes;
1646 /** Number of bytes. */
1647 uint16_t cbOpcodes;
1648 /** The page offset. */
1649 RT_GCC_EXTENSION
1650 uint16_t offPhysPage : 12;
1651 /** Unused bits. */
1652 RT_GCC_EXTENSION
1653 uint16_t u2Unused : 2;
1654 /** Index into GCPhysPc + aGCPhysPages for the physical page address. */
1655 RT_GCC_EXTENSION
1656 uint16_t idxPhysPage : 2;
1657 } aRanges[8];
1658
1659 /** Physical pages that this TB covers.
1660 * The GCPhysPc w/o page offset is element zero, so starting here with 1. */
1661 RTGCPHYS aGCPhysPages[2];
1662} IEMTB;
1663#pragma pack()
1664AssertCompileMemberAlignment(IEMTB, GCPhysPc, sizeof(RTGCPHYS));
1665AssertCompileMemberAlignment(IEMTB, Thrd, sizeof(void *));
1666AssertCompileMemberAlignment(IEMTB, pabOpcodes, sizeof(void *));
1667AssertCompileMemberAlignment(IEMTB, pDbgInfo, sizeof(void *));
1668AssertCompileMemberAlignment(IEMTB, aGCPhysPages, sizeof(RTGCPHYS));
1669AssertCompileMemberOffset(IEMTB, aRanges, 64);
1670AssertCompileMemberSize(IEMTB, aRanges[0], 6);
1671#if 1
1672AssertCompileSize(IEMTB, 128);
1673# define IEMTB_SIZE_IS_POWER_OF_TWO /**< The IEMTB size is a power of two. */
1674#else
1675AssertCompileSize(IEMTB, 168);
1676# undef IEMTB_SIZE_IS_POWER_OF_TWO
1677#endif
1678
1679/** Pointer to a translation block. */
1680typedef IEMTB *PIEMTB;
1681/** Pointer to a const translation block. */
1682typedef IEMTB const *PCIEMTB;
1683
1684/** Gets address of the given TB lookup table entry. */
1685#define IEMTB_GET_TB_LOOKUP_TAB_ENTRY(a_pTb, a_idx) \
1686 ((PIEMTB *)&(a_pTb)->pabOpcodes[-(int)((a_pTb)->cTbLookupEntries - (a_idx)) * sizeof(PIEMTB)])
1687
1688/**
1689 * Gets the physical address for a TB opcode range.
1690 */
1691DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1692{
1693 Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1694 uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1695 Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1696 if (idxPage == 0)
1697 return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1698 Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1699 return pTb->aGCPhysPages[idxPage - 1];
1700}
1701
1702
1703/**
1704 * A chunk of memory in the TB allocator.
1705 */
1706typedef struct IEMTBCHUNK
1707{
1708 /** Pointer to the translation blocks in this chunk. */
1709 PIEMTB paTbs;
1710#ifdef IN_RING0
1711 /** Allocation handle. */
1712 RTR0MEMOBJ hMemObj;
1713#endif
1714} IEMTBCHUNK;
1715
1716/**
1717 * A per-CPU translation block allocator.
1718 *
1719 * Because of how the IEMTBCACHE uses the lower 6 bits of the TB address to keep
1720 * the length of the collision list, and of course also for cache line alignment
1721 * reasons, the TBs must be allocated with at least 64-byte alignment.
1722 * Memory is there therefore allocated using one of the page aligned allocators.
1723 *
1724 *
1725 * To avoid wasting too much memory, it is allocated piecemeal as needed,
1726 * in chunks (IEMTBCHUNK) of 2 MiB or more. The TB has an 8-bit chunk index
1727 * that enables us to quickly calculate the allocation bitmap position when
1728 * freeing the translation block.
1729 */
1730typedef struct IEMTBALLOCATOR
1731{
1732 /** Magic value (IEMTBALLOCATOR_MAGIC). */
1733 uint32_t uMagic;
1734
1735#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
1736 /** Mask corresponding to cTbsPerChunk - 1. */
1737 uint32_t fChunkMask;
1738 /** Shift count corresponding to cTbsPerChunk. */
1739 uint8_t cChunkShift;
1740#else
1741 uint32_t uUnused;
1742 uint8_t bUnused;
1743#endif
1744 /** Number of chunks we're allowed to allocate. */
1745 uint8_t cMaxChunks;
1746 /** Number of chunks currently populated. */
1747 uint16_t cAllocatedChunks;
1748 /** Number of translation blocks per chunk. */
1749 uint32_t cTbsPerChunk;
1750 /** Chunk size. */
1751 uint32_t cbPerChunk;
1752
1753 /** The maximum number of TBs. */
1754 uint32_t cMaxTbs;
1755 /** Total number of TBs in the populated chunks.
1756 * (cAllocatedChunks * cTbsPerChunk) */
1757 uint32_t cTotalTbs;
1758 /** The current number of TBs in use.
1759 * The number of free TBs: cAllocatedTbs - cInUseTbs; */
1760 uint32_t cInUseTbs;
1761 /** Statistics: Number of the cInUseTbs that are native ones. */
1762 uint32_t cNativeTbs;
1763 /** Statistics: Number of the cInUseTbs that are threaded ones. */
1764 uint32_t cThreadedTbs;
1765
1766 /** Where to start pruning TBs from when we're out.
1767 * See iemTbAllocatorAllocSlow for details. */
1768 uint32_t iPruneFrom;
1769 /** Where to start pruning native TBs from when we're out of executable memory.
1770 * See iemTbAllocatorFreeupNativeSpace for details. */
1771 uint32_t iPruneNativeFrom;
1772 uint64_t u64Padding;
1773
1774 /** Statistics: Number of TB allocation calls. */
1775 STAMCOUNTER StatAllocs;
1776 /** Statistics: Number of TB free calls. */
1777 STAMCOUNTER StatFrees;
1778 /** Statistics: Time spend pruning. */
1779 STAMPROFILE StatPrune;
1780 /** Statistics: Time spend pruning native TBs. */
1781 STAMPROFILE StatPruneNative;
1782
1783 /** The delayed free list (see iemTbAlloctorScheduleForFree). */
1784 PIEMTB pDelayedFreeHead;
1785 /* Head of the list of free TBs. */
1786 PIEMTB pTbsFreeHead;
1787
1788 /** Allocation chunks. */
1789 IEMTBCHUNK aChunks[256];
1790} IEMTBALLOCATOR;
1791/** Pointer to a TB allocator. */
1792typedef struct IEMTBALLOCATOR *PIEMTBALLOCATOR;
1793
1794/** Magic value for the TB allocator (Emmet Harley Cohen). */
1795#define IEMTBALLOCATOR_MAGIC UINT32_C(0x19900525)
1796
1797
1798/**
1799 * A per-CPU translation block cache (hash table).
1800 *
1801 * The hash table is allocated once during IEM initialization and size double
1802 * the max TB count, rounded up to the nearest power of two (so we can use and
1803 * AND mask rather than a rest division when hashing).
1804 */
1805typedef struct IEMTBCACHE
1806{
1807 /** Magic value (IEMTBCACHE_MAGIC). */
1808 uint32_t uMagic;
1809 /** Size of the hash table. This is a power of two. */
1810 uint32_t cHash;
1811 /** The mask corresponding to cHash. */
1812 uint32_t uHashMask;
1813 uint32_t uPadding;
1814
1815 /** @name Statistics
1816 * @{ */
1817 /** Number of collisions ever. */
1818 STAMCOUNTER cCollisions;
1819
1820 /** Statistics: Number of TB lookup misses. */
1821 STAMCOUNTER cLookupMisses;
1822 /** Statistics: Number of TB lookup hits via hash table (debug only). */
1823 STAMCOUNTER cLookupHits;
1824 /** Statistics: Number of TB lookup hits via TB associated lookup table (debug only). */
1825 STAMCOUNTER cLookupHitsViaTbLookupTable;
1826 STAMCOUNTER auPadding2[2];
1827 /** Statistics: Collision list length pruning. */
1828 STAMPROFILE StatPrune;
1829 /** @} */
1830
1831 /** The hash table itself.
1832 * @note The lower 6 bits of the pointer is used for keeping the collision
1833 * list length, so we can take action when it grows too long.
1834 * This works because TBs are allocated using a 64 byte (or
1835 * higher) alignment from page aligned chunks of memory, so the lower
1836 * 6 bits of the address will always be zero.
1837 * See IEMTBCACHE_PTR_COUNT_MASK, IEMTBCACHE_PTR_MAKE and friends.
1838 */
1839 RT_FLEXIBLE_ARRAY_EXTENSION
1840 PIEMTB apHash[RT_FLEXIBLE_ARRAY];
1841} IEMTBCACHE;
1842/** Pointer to a per-CPU translation block cache. */
1843typedef IEMTBCACHE *PIEMTBCACHE;
1844
1845/** Magic value for IEMTBCACHE (Johnny O'Neal). */
1846#define IEMTBCACHE_MAGIC UINT32_C(0x19561010)
1847
1848/** The collision count mask for IEMTBCACHE::apHash entries. */
1849#define IEMTBCACHE_PTR_COUNT_MASK ((uintptr_t)0x3f)
1850/** The max collision count for IEMTBCACHE::apHash entries before pruning. */
1851#define IEMTBCACHE_PTR_MAX_COUNT ((uintptr_t)0x30)
1852/** Combine a TB pointer and a collision list length into a value for an
1853 * IEMTBCACHE::apHash entry. */
1854#define IEMTBCACHE_PTR_MAKE(a_pTb, a_cCount) (PIEMTB)((uintptr_t)(a_pTb) | (a_cCount))
1855/** Combine a TB pointer and a collision list length into a value for an
1856 * IEMTBCACHE::apHash entry. */
1857#define IEMTBCACHE_PTR_GET_TB(a_pHashEntry) (PIEMTB)((uintptr_t)(a_pHashEntry) & ~IEMTBCACHE_PTR_COUNT_MASK)
1858/** Combine a TB pointer and a collision list length into a value for an
1859 * IEMTBCACHE::apHash entry. */
1860#define IEMTBCACHE_PTR_GET_COUNT(a_pHashEntry) ((uintptr_t)(a_pHashEntry) & IEMTBCACHE_PTR_COUNT_MASK)
1861
1862/**
1863 * Calculates the hash table slot for a TB from physical PC address and TB flags.
1864 */
1865#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
1866 IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, (a_fTbFlags) & IEMTB_F_KEY_MASK, a_GCPhysPc)
1867
1868/**
1869 * Calculates the hash table slot for a TB from physical PC address and TB
1870 * flags, ASSUMING the caller has applied IEMTB_F_KEY_MASK to @a a_fTbFlags.
1871 */
1872#define IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, a_fTbFlags, a_GCPhysPc) \
1873 (((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
1874
1875
1876/** @name IEMBRANCHED_F_XXX - Branched indicator (IEMCPU::fTbBranched).
1877 *
1878 * These flags parallels the main IEM_CIMPL_F_BRANCH_XXX flags.
1879 *
1880 * @{ */
1881/** Value if no branching happened recently. */
1882#define IEMBRANCHED_F_NO UINT8_C(0x00)
1883/** Flag set if direct branch, clear if absolute or indirect. */
1884#define IEMBRANCHED_F_DIRECT UINT8_C(0x01)
1885/** Flag set if indirect branch, clear if direct or relative. */
1886#define IEMBRANCHED_F_INDIRECT UINT8_C(0x02)
1887/** Flag set if relative branch, clear if absolute or indirect. */
1888#define IEMBRANCHED_F_RELATIVE UINT8_C(0x04)
1889/** Flag set if conditional branch, clear if unconditional. */
1890#define IEMBRANCHED_F_CONDITIONAL UINT8_C(0x08)
1891/** Flag set if it's a far branch.
1892 * @note x86 specific */
1893#define IEMBRANCHED_F_FAR UINT8_C(0x10)
1894/** Flag set if the stack pointer is modified. */
1895#define IEMBRANCHED_F_STACK UINT8_C(0x20)
1896/** Flag set if the stack pointer and (maybe) the stack segment are modified.
1897 * @note x86 specific */
1898#define IEMBRANCHED_F_STACK_FAR UINT8_C(0x40)
1899/** Flag set (by IEM_MC_REL_JMP_XXX) if it's a zero bytes relative jump. */
1900#define IEMBRANCHED_F_ZERO UINT8_C(0x80)
1901/** @} */
1902
1903
1904/** @def IEM_MAX_MEM_MAPPINGS
1905 * Maximum number of concurrent memory needed by the target architecture.
1906 * @x86 There are a few instructions with two memory operands (push/pop [mem],
1907 * string instructions). We add another entry for safety.
1908 * @arm Except for the recently specified memcpy/move instructions,
1909 * ARM instruction takes at most one memory operand. We use 1 and add
1910 * another entry for safety, ignoring the memcpy instructions for now. */
1911#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING) /* for now: */ || defined(VBOX_VMM_TARGET_AGNOSTIC)
1912# define IEM_MAX_MEM_MAPPINGS 3
1913#elif defined(VBOX_VMM_TARGET_ARMV8)
1914# define IEM_MAX_MEM_MAPPINGS 2
1915#else
1916# error "port me"
1917#endif
1918
1919/** @def IEM_BOUNCE_BUFFER_SIZE
1920 * The size of the bounce buffers. This is dictated by the largest memory
1921 * operand of the target architecture.
1922 * @x86 fxsave/fxrstor takes a 512 byte operand. Whether we actually need a
1923 * 512 byte bounce buffer for it is questionable...
1924 * @arm Currently we shouldn't need more than 64 bytes here (ld64b, ld4). */
1925#if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING) /* for now: */ || defined(VBOX_VMM_TARGET_AGNOSTIC)
1926# define IEM_BOUNCE_BUFFER_SIZE 512
1927#elif defined(VBOX_VMM_TARGET_ARMV8)
1928# define IEM_BOUNCE_BUFFER_SIZE 64
1929#else
1930# error "port me"
1931#endif
1932
1933
1934/**
1935 * The per-CPU IEM state.
1936 */
1937typedef struct IEMCPU
1938{
1939 /** Info status code that needs to be propagated to the IEM caller.
1940 * This cannot be passed internally, as it would complicate all success
1941 * checks within the interpreter making the code larger and almost impossible
1942 * to get right. Instead, we'll store status codes to pass on here. Each
1943 * source of these codes will perform appropriate sanity checks. */
1944 int32_t rcPassUp; /* 0x00 */
1945 /** Execution flag, IEM_F_XXX. */
1946 uint32_t fExec; /* 0x04 */
1947
1948 /** @name Decoder state.
1949 * @{ */
1950#ifdef IEM_WITH_CODE_TLB
1951 /** The offset of the next instruction byte. */
1952 uint32_t offInstrNextByte; /* 0x08 */
1953# if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1954 /** X86: The number of bytes available at pbInstrBuf for the current
1955 * instruction. This takes the max opcode length into account so that doesn't
1956 * need to be checked separately. */
1957 uint32_t cbInstrBuf; /* x86: 0x0c */
1958# else
1959 /** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1960 * @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1961 uint32_t cbInstrBufTotal; /* !x86: 0x0c */
1962# endif
1963 /** Pointer to the page containing PC, user specified buffer or abOpcode.
1964 * This can be NULL if the page isn't mappable for some reason, in which
1965 * case we'll do fallback stuff.
1966 *
1967 * If we're executing an instruction from a user specified buffer,
1968 * IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
1969 * aligned pointer but pointer to the user data.
1970 *
1971 * X86: For instructions crossing pages, this will start on the first page and
1972 * be advanced to the next page by the time we've decoded the instruction. This
1973 * therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
1974 */
1975 uint8_t const *pbInstrBuf; /* 0x10 */
1976# if ARCH_BITS == 32
1977 uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
1978# endif
1979 /** The program counter corresponding to pbInstrBuf.
1980 * This is set to a non-canonical address when we need to invalidate it. */
1981 uint64_t uInstrBufPc; /* 0x18 */
1982 /** The guest physical address corresponding to pbInstrBuf. */
1983 RTGCPHYS GCPhysInstrBuf; /* 0x20 */
1984# if defined(VBOX_VMM_TARGET_X86) || defined(DOXYGEN_RUNNING)
1985 /** X86: The number of bytes available at pbInstrBuf in total (for IEMExecLots).
1986 * This takes the CS segment limit into account.
1987 * @note Set to zero when the code TLB is flushed to trigger TLB reload. */
1988 uint16_t cbInstrBufTotal; /* x86: 0x28 */
1989 /** X86: Offset into pbInstrBuf of the first byte of the current instruction.
1990 * Can be negative to efficiently handle cross page instructions. */
1991 int16_t offCurInstrStart; /* x86: 0x2a */
1992# endif
1993
1994# if (!defined(IEM_WITH_OPAQUE_DECODER_STATE) && defined(VBOX_VMM_TARGET_X86)) || defined(DOXYGEN_RUNNING)
1995 /** X86: The prefix mask (IEM_OP_PRF_XXX). */
1996 uint32_t fPrefixes; /* x86: 0x2c */
1997 /** X86: The extra REX ModR/M register field bit (REX.R << 3). */
1998 uint8_t uRexReg; /* x86: 0x30 */
1999 /** X86: The extra REX ModR/M r/m field, SIB base and opcode reg bit
2000 * (REX.B << 3). */
2001 uint8_t uRexB; /* x86: 0x31 */
2002 /** X86: The extra REX SIB index field bit (REX.X << 3). */
2003 uint8_t uRexIndex; /* x86: 0x32 */
2004
2005 /** X86: The effective segment register (X86_SREG_XXX). */
2006 uint8_t iEffSeg; /* x86: 0x33 */
2007
2008 /** X86: The offset of the ModR/M byte relative to the start of the instruction. */
2009 uint8_t offModRm; /* x86: 0x34 */
2010
2011# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
2012 /** X86: The current offset into abOpcode. */
2013 uint8_t offOpcode; /* x86: 0x35 */
2014# else
2015 uint8_t bUnused; /* x86: 0x35 */
2016# endif
2017# else /* IEM_WITH_OPAQUE_DECODER_STATE || !X86 */
2018# ifdef VBOX_VMM_TARGET_X86
2019 uint8_t abOpaqueDecoderPart1[0x36 - 0x2c];
2020# endif
2021# endif /* IEM_WITH_OPAQUE_DECODER_STATE || !X86 */
2022
2023#else /* !IEM_WITH_CODE_TLB */
2024# ifndef IEM_WITH_OPAQUE_DECODER_STATE
2025 /** The size of what has currently been fetched into abOpcode. */
2026 uint8_t cbOpcode; /* 0x08 */
2027
2028# ifdef VBOX_VMM_TARGET_X86
2029 /** X86: The current offset into abOpcode. */
2030 uint8_t offOpcode; /* x86: 0x09 */
2031 /** X86: The offset of the ModR/M byte relative to the start of the
2032 * instruction. */
2033 uint8_t offModRm; /* x86: 0x0a */
2034
2035 /** X86: The effective segment register (X86_SREG_XXX). */
2036 uint8_t iEffSeg; /* x86: 0x0b */
2037
2038 /** X86: The prefix mask (IEM_OP_PRF_XXX). */
2039 uint32_t fPrefixes; /* x86: 0x0c */
2040 /** X86: The extra REX ModR/M register field bit (REX.R << 3). */
2041 uint8_t uRexReg; /* x86: 0x10 */
2042 /** X86: The extra REX ModR/M r/m field, SIB base and opcode reg bit
2043 * (REX.B << 3). */
2044 uint8_t uRexB; /* x86: 0x11 */
2045 /** X86: The extra REX SIB index field bit (REX.X << 3). */
2046 uint8_t uRexIndex; /* x86: 0x12 */
2047# endif
2048# else /* IEM_WITH_OPAQUE_DECODER_STATE */
2049# ifndef VBOX_VMM_TARGET_X86
2050 uint8_t abOpaqueDecoderPart1[1];
2051# else
2052 uint8_t abOpaqueDecoderPart1[0x13 - 0x08];
2053# endif
2054# endif /* IEM_WITH_OPAQUE_DECODER_STATE */
2055#endif /* !IEM_WITH_CODE_TLB */
2056
2057#if (!defined(IEM_WITH_OPAQUE_DECODER_STATE) && (defined(VBOX_VMM_TARGET_X86) || !defined(IEM_WITH_CODE_TLB))) \
2058 || defined(DOXGYEN_RUNNING)
2059# ifdef VBOX_VMM_TARGET_X86
2060 /** X86: The effective operand mode. */
2061 IEMMODE enmEffOpSize; /* x86: 0x36, 0x13 */
2062 /** X86: The default addressing mode. */
2063 IEMMODE enmDefAddrMode; /* x86: 0x37, 0x14 */
2064 /** X86: The effective addressing mode. */
2065 IEMMODE enmEffAddrMode; /* x86: 0x38, 0x15 */
2066 /** X86: The default operand mode. */
2067 IEMMODE enmDefOpSize; /* x86: 0x39, 0x16 */
2068
2069 /** X86: Prefix index (VEX.pp) for two byte and three byte tables. */
2070 uint8_t idxPrefix; /* x86: 0x3a, 0x17 */
2071 /** X86: 3rd VEX/EVEX/XOP register.
2072 * Please use IEM_GET_EFFECTIVE_VVVV to access. */
2073 uint8_t uVex3rdReg; /* x86: 0x3b, 0x18 */
2074 /** X86: The VEX/EVEX/XOP length field. */
2075 uint8_t uVexLength; /* x86: 0x3c, 0x19 */
2076 /** X86: Additional EVEX stuff. */
2077 uint8_t fEvexStuff; /* x86: 0x3d, 0x1a */
2078
2079# ifndef IEM_WITH_CODE_TLB
2080 /** Explicit alignment padding. */
2081 uint8_t abAlignment2a[1]; /* x86: 0x1b */
2082# endif
2083 /** X86: The FPU opcode (FOP). */
2084 uint16_t uFpuOpcode; /* x86: 0x3e, 0x1c */
2085# ifndef IEM_WITH_CODE_TLB
2086 /** Opcode buffer alignment padding. */
2087 uint8_t abAlignment2b[2]; /* x86: 0x1e */
2088# endif
2089# else /* !VBOX_VMM_TARGET_X86 */
2090 /** Opcode buffer alignment padding. */
2091 uint8_t abAlignment2b[3+4]; /* !x86: 0x09 */
2092# endif /* !VBOX_VMM_TARGET_X86 */
2093
2094 /** The opcode bytes. */
2095# ifdef VBOX_VMM_TARGET_X86
2096 uint8_t abOpcode[15]; /* x86: 0x40, 0x20 */
2097# else
2098 union
2099 {
2100 uint8_t abOpcode[ 32]; /* !x86: 0x10 */
2101 uint16_t au16Opcode[16];
2102 uint32_t au32Opcode[ 8];
2103 };
2104# endif
2105 /** Explicit alignment padding. */
2106# ifdef VBOX_VMM_TARGET_X86
2107# ifdef IEM_WITH_CODE_TLB
2108 //uint8_t abAlignment2c[0x4f - 0x4f]; /* x86: 0x4f */
2109# else
2110 uint8_t abAlignment2c[0x4f - 0x2f]; /* x86: 0x2f */
2111# endif
2112# else
2113 uint8_t abAlignment2c[0x4f - 0x30]; /* !x86: 0x30 */
2114# endif
2115
2116#else /* IEM_WITH_OPAQUE_DECODER_STATE || (!x86 && TLB) */
2117# ifdef IEM_WITH_CODE_TLB
2118# ifdef VBOX_VMM_TARGET_X86
2119 uint8_t abOpaqueDecoderPart2[0x4f - 0x36];
2120# else
2121 uint8_t abOpaqueDecoderPart2[0x4f - 0x28];
2122# endif
2123# else
2124# ifdef VBOX_VMM_TARGET_X86
2125 uint8_t abOpaqueDecoderPart2[0x4f - 0x13];
2126# else
2127 uint8_t abOpaqueDecoderPart2[0x4f - 0x09];
2128# endif
2129# endif
2130#endif /* IEM_WITH_OPAQUE_DECODER_STATE */
2131 /** @} */
2132
2133
2134 /** The number of active guest memory mappings. */
2135 uint8_t cActiveMappings; /* 0x4f, 0x4f */
2136
2137 /** Records for tracking guest memory mappings. */
2138 struct
2139 {
2140 /** The address of the mapped bytes. */
2141 R3R0PTRTYPE(void *) pv;
2142 /** The access flags (IEM_ACCESS_XXX).
2143 * IEM_ACCESS_INVALID if the entry is unused. */
2144 uint32_t fAccess;
2145#if HC_ARCH_BITS == 64
2146 uint32_t u32Alignment4; /**< Alignment padding. */
2147#endif
2148 } aMemMappings[IEM_MAX_MEM_MAPPINGS]; /* arm: 0x50 LB 0x20 x86: 0x50 LB 0x30 */
2149
2150 /** Locking records for the mapped memory. */
2151 union
2152 {
2153 PGMPAGEMAPLOCK Lock;
2154 uint64_t au64Padding[2];
2155 } aMemMappingLocks[IEM_MAX_MEM_MAPPINGS]; /* arm: 0x70 LB 0x20 x86: 0x80 LB 0x30 */
2156
2157 /** Bounce buffer info.
2158 * This runs in parallel to aMemMappings. */
2159 struct
2160 {
2161 /** The physical address of the first byte. */
2162 RTGCPHYS GCPhysFirst;
2163 /** The physical address of the second page. */
2164 RTGCPHYS GCPhysSecond;
2165 /** The number of bytes in the first page. */
2166 uint16_t cbFirst;
2167 /** The number of bytes in the second page. */
2168 uint16_t cbSecond;
2169 /** Whether it's unassigned memory. */
2170 bool fUnassigned;
2171 /** Explicit alignment padding. */
2172 bool afAlignment5[3];
2173 } aMemBbMappings[IEM_MAX_MEM_MAPPINGS]; /* arm: 0x90 LB 0x30 x86: 0xb0 LB 0x48 */
2174
2175 /** The flags of the current exception / interrupt.
2176 * @note X86 specific? */
2177 uint32_t fCurXcpt; /* arm: 0xc0 x86: 0xf8 */
2178 /** The current exception / interrupt.
2179 *@note X86 specific? */
2180 uint8_t uCurXcpt; /* arm: 0xc4 x86: 0xfc */
2181 /** Exception / interrupt recursion depth.
2182 *@note X86 specific? */
2183 int8_t cXcptRecursions; /* arm: 0xc5 x86: 0xfb */
2184
2185 /** The next unused mapping index.
2186 * @todo try find room for this up with cActiveMappings. */
2187 uint8_t iNextMapping; /* arm: 0xc6 x86: 0xfd */
2188 uint8_t abAlignment7[IEM_MAX_MEM_MAPPINGS == 3 ? 1 : 0x39];
2189
2190 /** Bounce buffer storage.
2191 * This runs in parallel to aMemMappings and aMemBbMappings. */
2192 struct
2193 {
2194 uint8_t ab[IEM_BOUNCE_BUFFER_SIZE];
2195 } aBounceBuffers[IEM_MAX_MEM_MAPPINGS]; /* arm: 0x100 LB 0x80 x86: 0x100 LB 0x600 */
2196
2197
2198 /** Pointer set jump buffer - ring-3 context. */
2199 R3PTRTYPE(jmp_buf *) pJmpBufR3;
2200 /** Pointer set jump buffer - ring-0 context. */
2201 R0PTRTYPE(jmp_buf *) pJmpBufR0;
2202
2203 /** @todo Should move this near @a fCurXcpt later. */
2204 /** The CR2 for the current exception / interrupt. */
2205 uint64_t uCurXcptCr2;
2206 /** The error code for the current exception / interrupt. */
2207 uint32_t uCurXcptErr;
2208
2209 /** @name Statistics
2210 * @{ */
2211 /** The number of instructions we've executed. */
2212 uint32_t cInstructions;
2213 /** The number of potential exits. */
2214 uint32_t cPotentialExits;
2215 /** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
2216 uint32_t cRetInstrNotImplemented;
2217 /** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
2218 uint32_t cRetAspectNotImplemented;
2219 /** Counts informational statuses returned (other than VINF_SUCCESS). */
2220 uint32_t cRetInfStatuses;
2221 /** Counts other error statuses returned. */
2222 uint32_t cRetErrStatuses;
2223 /** Number of times rcPassUp has been used. */
2224 uint32_t cRetPassUpStatus;
2225 /** Number of times RZ left with instruction commit pending for ring-3. */
2226 uint32_t cPendingCommit;
2227 /** Number of misaligned (host sense) atomic instruction accesses. */
2228 uint32_t cMisalignedAtomics;
2229 /** Number of long jumps. */
2230 uint32_t cLongJumps;
2231 /** @} */
2232
2233 /** @name Target CPU information.
2234 * @{ */
2235#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
2236 /** The target CPU. */
2237 uint8_t uTargetCpu;
2238#else
2239 uint8_t bTargetCpuPadding;
2240#endif
2241 /** For selecting assembly works matching the target CPU EFLAGS behaviour, see
2242 * IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
2243 * native host support and the 2nd for when there is.
2244 *
2245 * The two values are typically indexed by a g_CpumHostFeatures bit.
2246 *
2247 * This is for instance used for the BSF & BSR instructions where AMD and
2248 * Intel CPUs produce different EFLAGS. */
2249 uint8_t aidxTargetCpuEflFlavour[2];
2250
2251 /** The CPU vendor. */
2252 CPUMCPUVENDOR enmCpuVendor;
2253 /** @} */
2254
2255 /** Counts RDMSR \#GP(0) LogRel(). */
2256 uint8_t cLogRelRdMsr;
2257 /** Counts WRMSR \#GP(0) LogRel(). */
2258 uint8_t cLogRelWrMsr;
2259 /** Alignment padding. */
2260 uint8_t abAlignment9[50];
2261
2262
2263 /** @name Recompiled Exection
2264 * @{ */
2265 /** Pointer to the current translation block.
2266 * This can either be one being executed or one being compiled. */
2267 R3PTRTYPE(PIEMTB) pCurTbR3;
2268#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER_LONGJMP
2269 /** Frame pointer for the last native TB to execute. */
2270 R3PTRTYPE(void *) pvTbFramePointerR3;
2271#else
2272 R3PTRTYPE(void *) pvUnusedR3;
2273#endif
2274#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2275 /** The saved host floating point control register (MXCSR on x86, FPCR on arm64)
2276 * needing restore when the TB finished, IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED indicates the TB
2277 * didn't modify it so we don't need to restore it. */
2278# ifdef RT_ARCH_AMD64
2279 uint32_t uRegFpCtrl;
2280 /** Temporary copy of MXCSR for stmxcsr/ldmxcsr (so we don't have to fiddle with stack pointers). */
2281 uint32_t uRegMxcsrTmp;
2282# elif defined(RT_ARCH_ARM64)
2283 uint64_t uRegFpCtrl;
2284# else
2285# error "Port me"
2286# endif
2287#else
2288 uint64_t u64Unused;
2289#endif
2290 /** Pointer to the ring-3 TB cache for this EMT. */
2291 R3PTRTYPE(PIEMTBCACHE) pTbCacheR3;
2292 /** Pointer to the ring-3 TB lookup entry.
2293 * This either points to pTbLookupEntryDummyR3 or an actually lookuptable
2294 * entry, thus it can always safely be used w/o NULL checking. */
2295 R3PTRTYPE(PIEMTB *) ppTbLookupEntryR3;
2296#if 0 /* unused */
2297 /** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
2298 * The TBs are based on physical addresses, so this is needed to correleated
2299 * RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
2300 uint64_t uCurTbStartPc;
2301#endif
2302
2303 /** Number of threaded TBs executed. */
2304 uint64_t cTbExecThreaded;
2305 /** Number of native TBs executed. */
2306 uint64_t cTbExecNative;
2307
2308 /** The number of IRQ/FF checks till the next timer poll call. */
2309 uint32_t cTbsTillNextTimerPoll;
2310 /** The virtual sync time at the last timer poll call in milliseconds. */
2311 uint32_t msRecompilerPollNow;
2312 /** The virtual sync time at the last timer poll call in nanoseconds. */
2313 uint64_t nsRecompilerPollNow;
2314 /** The previous cTbsTillNextTimerPoll value. */
2315 uint32_t cTbsTillNextTimerPollPrev;
2316
2317 /** The current instruction number in a native TB.
2318 * This is set by code that may trigger an unexpected TB exit (throw/longjmp)
2319 * and will be picked up by the TB execution loop. Only used when
2320 * IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
2321 uint8_t idxTbCurInstr;
2322 /** @} */
2323
2324 /** @name Recompilation
2325 * @{ */
2326 /** Whether we need to check the opcode bytes for the current instruction.
2327 * This is set by a previous instruction if it modified memory or similar. */
2328 bool fTbCheckOpcodes;
2329 /** Indicates whether and how we just branched - IEMBRANCHED_F_XXX. */
2330 uint8_t fTbBranched;
2331 /** Set when GCPhysInstrBuf is updated because of a page crossing. */
2332 bool fTbCrossedPage;
2333 /** Whether to end the current TB. */
2334 bool fEndTb;
2335 /** Indicates that the current instruction is an STI. This is set by the
2336 * iemCImpl_sti code and subsequently cleared by the recompiler. */
2337 bool fTbCurInstrIsSti;
2338 /** Spaced reserved for recompiler data / alignment. */
2339 bool afRecompilerStuff1[1];
2340 /** Number of instructions before we need emit an IRQ check call again.
2341 * This helps making sure we don't execute too long w/o checking for
2342 * interrupts and immediately following instructions that may enable
2343 * interrupts (e.g. POPF, IRET, STI). With STI an additional hack is
2344 * required to make sure we check following the next instruction as well, see
2345 * fTbCurInstrIsSti. */
2346 uint8_t cInstrTillIrqCheck;
2347 /** The index of the last CheckIrq call during threaded recompilation. */
2348 uint16_t idxLastCheckIrqCallNo;
2349 /** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
2350 uint16_t cbOpcodesAllocated;
2351 /** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
2352 uint32_t uTbNativeRecompileAtUsedCount;
2353 /** The IEM_CIMPL_F_XXX mask for the current instruction. */
2354 uint32_t fTbCurInstr;
2355 /** The IEM_CIMPL_F_XXX mask for the previous instruction. */
2356 uint32_t fTbPrevInstr;
2357 /** Strict: Tracking skipped EFLAGS calculations. Any bits set here are
2358 * currently not up to date in EFLAGS. */
2359 uint32_t fSkippingEFlags;
2360#if 0 /* unused */
2361 /** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set. */
2362 RTGCPHYS GCPhysInstrBufPrev;
2363#endif
2364
2365 /** Fixed TB used for threaded recompilation.
2366 * This is allocated once with maxed-out sizes and re-used afterwards. */
2367 R3PTRTYPE(PIEMTB) pThrdCompileTbR3;
2368 /** Pointer to the ring-3 TB allocator for this EMT. */
2369 R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
2370 /** Pointer to the ring-3 executable memory allocator for this EMT. */
2371 R3PTRTYPE(struct IEMEXECMEMALLOCATOR *) pExecMemAllocatorR3;
2372 /** Pointer to the native recompiler state for ring-3. */
2373 R3PTRTYPE(struct IEMRECOMPILERSTATE *) pNativeRecompilerStateR3;
2374 /** Dummy entry for ppTbLookupEntryR3. */
2375 R3PTRTYPE(PIEMTB) pTbLookupEntryDummyR3;
2376#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2377 /** The debug code advances this register as if it was CPUMCTX::rip and we
2378 * didn't do delayed PC updating. When CPUMCTX::rip is finally updated,
2379 * the result is compared with this value. */
2380 uint64_t uPcUpdatingDebug;
2381#elif defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
2382 /** The SSM handle used for saving threaded TBs for recompiler profiling. */
2383 R3PTRTYPE(PSSMHANDLE) pSsmThreadedTbsForProfiling;
2384#else
2385 uint64_t u64Placeholder;
2386#endif
2387 /**
2388 * Whether we should use the host instruction invalidation APIs of the
2389 * host OS or our own version of it (macOS). */
2390 uint8_t fHostICacheInvalidation;
2391#define IEMNATIVE_ICACHE_F_USE_HOST_API UINT8_C(0x01) /**< Use the host API (macOS) instead of our code. */
2392#define IEMNATIVE_ICACHE_F_END_WITH_ISH UINT8_C(0x02) /**< Whether to end with a ISH barrier (arm). */
2393 bool afRecompilerStuff2[7];
2394 /** @} */
2395
2396 /** Dummy TLB entry used for accesses to pages with databreakpoints. */
2397 IEMTLBENTRY DataBreakpointTlbe;
2398
2399 /** Threaded TB statistics: Times TB execution was broken off before reaching the end. */
2400 STAMCOUNTER StatTbThreadedExecBreaks;
2401 /** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
2402 STAMCOUNTER StatCheckIrqBreaks;
2403 /** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
2404 STAMCOUNTER StatCheckTimersBreaks;
2405 /** Statistics: Times BltIn_CheckMode breaks out of the TB. */
2406 STAMCOUNTER StatCheckModeBreaks;
2407 /** Threaded TB statistics: Times execution break on call with lookup entries. */
2408 STAMCOUNTER StatTbThreadedExecBreaksWithLookup;
2409 /** Threaded TB statistics: Times execution break on call without lookup entries. */
2410 STAMCOUNTER StatTbThreadedExecBreaksWithoutLookup;
2411 /** Statistics: Times a post jump target check missed and had to find new TB. */
2412 STAMCOUNTER StatCheckBranchMisses;
2413 /** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
2414 STAMCOUNTER StatCheckNeedCsLimChecking;
2415 /** Statistics: Times a loop was detected within a TB. */
2416 STAMCOUNTER StatTbLoopInTbDetected;
2417 /** Statistics: Times a loop back to the start of the TB was detected. */
2418 STAMCOUNTER StatTbLoopFullTbDetected;
2419 /** Statistics: Times a loop back to the start of the TB was detected, var 2. */
2420 STAMCOUNTER StatTbLoopFullTbDetected2;
2421 /** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
2422 STAMCOUNTER StatNativeExecMemInstrBufAllocFailed;
2423 /** Native TB statistics: Number of fully recompiled TBs. */
2424 STAMCOUNTER StatNativeFullyRecompiledTbs;
2425 /** TB statistics: Number of instructions per TB. */
2426 STAMPROFILE StatTbInstr;
2427 /** TB statistics: Number of TB lookup table entries per TB. */
2428 STAMPROFILE StatTbLookupEntries;
2429 /** Threaded TB statistics: Number of calls per TB. */
2430 STAMPROFILE StatTbThreadedCalls;
2431 /** Native TB statistics: Native code size per TB. */
2432 STAMPROFILE StatTbNativeCode;
2433 /** Native TB statistics: Profiling native recompilation. */
2434 STAMPROFILE StatNativeRecompilation;
2435 /** Native TB statistics: Number of calls per TB that were recompiled properly. */
2436 STAMPROFILE StatNativeCallsRecompiled;
2437 /** Native TB statistics: Number of threaded calls per TB that weren't recompiled. */
2438 STAMPROFILE StatNativeCallsThreaded;
2439 /** Native recompiled execution: TLB hits for data fetches. */
2440 STAMCOUNTER StatNativeTlbHitsForFetch;
2441 /** Native recompiled execution: TLB hits for data stores. */
2442 STAMCOUNTER StatNativeTlbHitsForStore;
2443 /** Native recompiled execution: TLB hits for stack accesses. */
2444 STAMCOUNTER StatNativeTlbHitsForStack;
2445 /** Native recompiled execution: TLB hits for mapped accesses. */
2446 STAMCOUNTER StatNativeTlbHitsForMapped;
2447 /** Native recompiled execution: Code TLB misses for new page. */
2448 STAMCOUNTER StatNativeCodeTlbMissesNewPage;
2449 /** Native recompiled execution: Code TLB hits for new page. */
2450 STAMCOUNTER StatNativeCodeTlbHitsForNewPage;
2451 /** Native recompiled execution: Code TLB misses for new page with offset. */
2452 STAMCOUNTER StatNativeCodeTlbMissesNewPageWithOffset;
2453 /** Native recompiled execution: Code TLB hits for new page with offset. */
2454 STAMCOUNTER StatNativeCodeTlbHitsForNewPageWithOffset;
2455
2456 /** Native recompiler: Number of calls to iemNativeRegAllocFindFree. */
2457 STAMCOUNTER StatNativeRegFindFree;
2458 /** Native recompiler: Number of times iemNativeRegAllocFindFree needed
2459 * to free a variable. */
2460 STAMCOUNTER StatNativeRegFindFreeVar;
2461 /** Native recompiler: Number of times iemNativeRegAllocFindFree did
2462 * not need to free any variables. */
2463 STAMCOUNTER StatNativeRegFindFreeNoVar;
2464 /** Native recompiler: Liveness info freed shadowed guest registers in
2465 * iemNativeRegAllocFindFree. */
2466 STAMCOUNTER StatNativeRegFindFreeLivenessUnshadowed;
2467 /** Native recompiler: Liveness info helped with the allocation in
2468 * iemNativeRegAllocFindFree. */
2469 STAMCOUNTER StatNativeRegFindFreeLivenessHelped;
2470
2471 /** Native recompiler: Number of times status flags calc has been skipped. */
2472 STAMCOUNTER StatNativeEflSkippedArithmetic;
2473 /** Native recompiler: Number of times status flags calc has been postponed. */
2474 STAMCOUNTER StatNativeEflPostponedArithmetic;
2475 /** Native recompiler: Total number instructions in this category. */
2476 STAMCOUNTER StatNativeEflTotalArithmetic;
2477
2478 /** Native recompiler: Number of times status flags calc has been skipped. */
2479 STAMCOUNTER StatNativeEflSkippedLogical;
2480 /** Native recompiler: Number of times status flags calc has been postponed. */
2481 STAMCOUNTER StatNativeEflPostponedLogical;
2482 /** Native recompiler: Total number instructions in this category. */
2483 STAMCOUNTER StatNativeEflTotalLogical;
2484
2485 /** Native recompiler: Number of times status flags calc has been skipped. */
2486 STAMCOUNTER StatNativeEflSkippedShift;
2487 /** Native recompiler: Number of times status flags calc has been postponed. */
2488 STAMCOUNTER StatNativeEflPostponedShift;
2489 /** Native recompiler: Total number instructions in this category. */
2490 STAMCOUNTER StatNativeEflTotalShift;
2491
2492 /** Native recompiler: Number of emits per postponement. */
2493 STAMPROFILE StatNativeEflPostponedEmits;
2494
2495 /** Native recompiler: Number of opportunities to skip EFLAGS.CF updating. */
2496 STAMCOUNTER StatNativeLivenessEflCfSkippable;
2497 /** Native recompiler: Number of opportunities to skip EFLAGS.PF updating. */
2498 STAMCOUNTER StatNativeLivenessEflPfSkippable;
2499 /** Native recompiler: Number of opportunities to skip EFLAGS.AF updating. */
2500 STAMCOUNTER StatNativeLivenessEflAfSkippable;
2501 /** Native recompiler: Number of opportunities to skip EFLAGS.ZF updating. */
2502 STAMCOUNTER StatNativeLivenessEflZfSkippable;
2503 /** Native recompiler: Number of opportunities to skip EFLAGS.SF updating. */
2504 STAMCOUNTER StatNativeLivenessEflSfSkippable;
2505 /** Native recompiler: Number of opportunities to skip EFLAGS.OF updating. */
2506 STAMCOUNTER StatNativeLivenessEflOfSkippable;
2507 /** Native recompiler: Number of required EFLAGS.CF updates. */
2508 STAMCOUNTER StatNativeLivenessEflCfRequired;
2509 /** Native recompiler: Number of required EFLAGS.PF updates. */
2510 STAMCOUNTER StatNativeLivenessEflPfRequired;
2511 /** Native recompiler: Number of required EFLAGS.AF updates. */
2512 STAMCOUNTER StatNativeLivenessEflAfRequired;
2513 /** Native recompiler: Number of required EFLAGS.ZF updates. */
2514 STAMCOUNTER StatNativeLivenessEflZfRequired;
2515 /** Native recompiler: Number of required EFLAGS.SF updates. */
2516 STAMCOUNTER StatNativeLivenessEflSfRequired;
2517 /** Native recompiler: Number of required EFLAGS.OF updates. */
2518 STAMCOUNTER StatNativeLivenessEflOfRequired;
2519 /** Native recompiler: Number of potentially delayable EFLAGS.CF updates. */
2520 STAMCOUNTER StatNativeLivenessEflCfDelayable;
2521 /** Native recompiler: Number of potentially delayable EFLAGS.PF updates. */
2522 STAMCOUNTER StatNativeLivenessEflPfDelayable;
2523 /** Native recompiler: Number of potentially delayable EFLAGS.AF updates. */
2524 STAMCOUNTER StatNativeLivenessEflAfDelayable;
2525 /** Native recompiler: Number of potentially delayable EFLAGS.ZF updates. */
2526 STAMCOUNTER StatNativeLivenessEflZfDelayable;
2527 /** Native recompiler: Number of potentially delayable EFLAGS.SF updates. */
2528 STAMCOUNTER StatNativeLivenessEflSfDelayable;
2529 /** Native recompiler: Number of potentially delayable EFLAGS.OF updates. */
2530 STAMCOUNTER StatNativeLivenessEflOfDelayable;
2531
2532 /** Native recompiler: Number of potential PC updates in total. */
2533 STAMCOUNTER StatNativePcUpdateTotal;
2534 /** Native recompiler: Number of PC updates which could be delayed. */
2535 STAMCOUNTER StatNativePcUpdateDelayed;
2536
2537 /** Native recompiler: Number of time we had complicated dirty shadow
2538 * register situations with the other branch in IEM_MC_ENDIF. */
2539 STAMCOUNTER StatNativeEndIfOtherBranchDirty;
2540
2541 /** Native recompiler: Number of calls to iemNativeSimdRegAllocFindFree. */
2542 STAMCOUNTER StatNativeSimdRegFindFree;
2543 /** Native recompiler: Number of times iemNativeSimdRegAllocFindFree needed
2544 * to free a variable. */
2545 STAMCOUNTER StatNativeSimdRegFindFreeVar;
2546 /** Native recompiler: Number of times iemNativeSimdRegAllocFindFree did
2547 * not need to free any variables. */
2548 STAMCOUNTER StatNativeSimdRegFindFreeNoVar;
2549 /** Native recompiler: Liveness info freed shadowed guest registers in
2550 * iemNativeSimdRegAllocFindFree. */
2551 STAMCOUNTER StatNativeSimdRegFindFreeLivenessUnshadowed;
2552 /** Native recompiler: Liveness info helped with the allocation in
2553 * iemNativeSimdRegAllocFindFree. */
2554 STAMCOUNTER StatNativeSimdRegFindFreeLivenessHelped;
2555
2556 /** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks. */
2557 STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckPotential;
2558 /** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks. */
2559 STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckPotential;
2560 /** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks. */
2561 STAMCOUNTER StatNativeMaybeSseXcptCheckPotential;
2562 /** Native recompiler: Number of potential IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks. */
2563 STAMCOUNTER StatNativeMaybeAvxXcptCheckPotential;
2564
2565 /** Native recompiler: Number of IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() checks omitted. */
2566 STAMCOUNTER StatNativeMaybeDeviceNotAvailXcptCheckOmitted;
2567 /** Native recompiler: Number of IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() checks omitted. */
2568 STAMCOUNTER StatNativeMaybeWaitDeviceNotAvailXcptCheckOmitted;
2569 /** Native recompiler: Number of IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() checks omitted. */
2570 STAMCOUNTER StatNativeMaybeSseXcptCheckOmitted;
2571 /** Native recompiler: Number of IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() checks omitted. */
2572 STAMCOUNTER StatNativeMaybeAvxXcptCheckOmitted;
2573
2574 /** Native recompiler: The TB finished executing completely without jumping to a an exit label.
2575 * Not availabe in release builds. */
2576 STAMCOUNTER StatNativeTbFinished;
2577 /** Native recompiler: The TB finished executing jumping to the ReturnBreak label. */
2578 STAMCOUNTER StatNativeTbExitReturnBreak;
2579 /** Native recompiler: The TB finished executing jumping to the ReturnBreakFF label. */
2580 STAMCOUNTER StatNativeTbExitReturnBreakFF;
2581 /** Native recompiler: The TB finished executing jumping to the ReturnWithFlags label. */
2582 STAMCOUNTER StatNativeTbExitReturnWithFlags;
2583 /** Native recompiler: The TB finished executing with other non-zero status. */
2584 STAMCOUNTER StatNativeTbExitReturnOtherStatus;
2585 /** Native recompiler: The TB finished executing via throw / long jump. */
2586 STAMCOUNTER StatNativeTbExitLongJump;
2587 /** Native recompiler: The TB finished executing jumping to the ReturnBreak
2588 * label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2589 STAMCOUNTER StatNativeTbExitDirectLinking1NoIrq;
2590 /** Native recompiler: The TB finished executing jumping to the ReturnBreak
2591 * label, but directly jumped to the next TB, scenario \#1 with IRQ checks. */
2592 STAMCOUNTER StatNativeTbExitDirectLinking1Irq;
2593 /** Native recompiler: The TB finished executing jumping to the ReturnBreak
2594 * label, but directly jumped to the next TB, scenario \#1 w/o IRQ checks. */
2595 STAMCOUNTER StatNativeTbExitDirectLinking2NoIrq;
2596 /** Native recompiler: The TB finished executing jumping to the ReturnBreak
2597 * label, but directly jumped to the next TB, scenario \#2 with IRQ checks. */
2598 STAMCOUNTER StatNativeTbExitDirectLinking2Irq;
2599
2600 /** Native recompiler: The TB finished executing jumping to the RaiseDe label. */
2601 STAMCOUNTER StatNativeTbExitRaiseDe;
2602 /** Native recompiler: The TB finished executing jumping to the RaiseUd label. */
2603 STAMCOUNTER StatNativeTbExitRaiseUd;
2604 /** Native recompiler: The TB finished executing jumping to the RaiseSseRelated label. */
2605 STAMCOUNTER StatNativeTbExitRaiseSseRelated;
2606 /** Native recompiler: The TB finished executing jumping to the RaiseAvxRelated label. */
2607 STAMCOUNTER StatNativeTbExitRaiseAvxRelated;
2608 /** Native recompiler: The TB finished executing jumping to the RaiseSseAvxFpRelated label. */
2609 STAMCOUNTER StatNativeTbExitRaiseSseAvxFpRelated;
2610 /** Native recompiler: The TB finished executing jumping to the RaiseNm label. */
2611 STAMCOUNTER StatNativeTbExitRaiseNm;
2612 /** Native recompiler: The TB finished executing jumping to the RaiseGp0 label. */
2613 STAMCOUNTER StatNativeTbExitRaiseGp0;
2614 /** Native recompiler: The TB finished executing jumping to the RaiseMf label. */
2615 STAMCOUNTER StatNativeTbExitRaiseMf;
2616 /** Native recompiler: The TB finished executing jumping to the RaiseXf label. */
2617 STAMCOUNTER StatNativeTbExitRaiseXf;
2618 /** Native recompiler: The TB finished executing jumping to the ObsoleteTb label. */
2619 STAMCOUNTER StatNativeTbExitObsoleteTb;
2620
2621 /** Native recompiler: Number of full TB loops (jumps from end to start). */
2622 STAMCOUNTER StatNativeTbExitLoopFullTb;
2623
2624 /** Native recompiler: Failure situations with direct linking scenario \#1.
2625 * Counter with StatNativeTbExitReturnBreak. Not in release builds.
2626 * @{ */
2627 STAMCOUNTER StatNativeTbExitDirectLinking1NoTb;
2628 STAMCOUNTER StatNativeTbExitDirectLinking1MismatchGCPhysPc;
2629 STAMCOUNTER StatNativeTbExitDirectLinking1MismatchFlags;
2630 STAMCOUNTER StatNativeTbExitDirectLinking1PendingIrq;
2631 /** @} */
2632
2633 /** Native recompiler: Failure situations with direct linking scenario \#2.
2634 * Counter with StatNativeTbExitReturnBreak. Not in release builds.
2635 * @{ */
2636 STAMCOUNTER StatNativeTbExitDirectLinking2NoTb;
2637 STAMCOUNTER StatNativeTbExitDirectLinking2MismatchGCPhysPc;
2638 STAMCOUNTER StatNativeTbExitDirectLinking2MismatchFlags;
2639 STAMCOUNTER StatNativeTbExitDirectLinking2PendingIrq;
2640 /** @} */
2641
2642 /** iemMemMap and iemMemMapJmp statistics.
2643 * @{ */
2644 STAMCOUNTER StatMemMapJmp;
2645 STAMCOUNTER StatMemMapNoJmp;
2646 STAMCOUNTER StatMemBounceBufferCrossPage;
2647 STAMCOUNTER StatMemBounceBufferMapPhys;
2648 /** @} */
2649
2650 /** Timer polling statistics (debug only).
2651 * @{ */
2652 STAMPROFILE StatTimerPoll;
2653 STAMPROFILE StatTimerPollPoll;
2654 STAMPROFILE StatTimerPollRun;
2655 STAMCOUNTER StatTimerPollUnchanged;
2656 STAMCOUNTER StatTimerPollTiny;
2657 STAMCOUNTER StatTimerPollDefaultCalc;
2658 STAMCOUNTER StatTimerPollMax;
2659 STAMPROFILE StatTimerPollFactorDivision;
2660 STAMPROFILE StatTimerPollFactorMultiplication;
2661 /** @} */
2662
2663
2664 STAMCOUNTER aStatAdHoc[8];
2665
2666#ifdef IEM_WITH_TLB_TRACE
2667 /*uint64_t au64Padding[0];*/
2668#else
2669 uint64_t au64Padding[2];
2670#endif
2671
2672#ifdef IEM_WITH_TLB_TRACE
2673 /** The end (next) trace entry. */
2674 uint32_t idxTlbTraceEntry;
2675 /** Number of trace entries allocated expressed as a power of two. */
2676 uint32_t cTlbTraceEntriesShift;
2677 /** The trace entries. */
2678 PIEMTLBTRACEENTRY paTlbTraceEntries;
2679#endif
2680
2681 /** Data TLB.
2682 * @remarks Must be 64-byte aligned. */
2683 IEMTLB DataTlb;
2684 /** Instruction TLB.
2685 * @remarks Must be 64-byte aligned. */
2686 IEMTLB CodeTlb;
2687
2688 /** Exception statistics. */
2689 STAMCOUNTER aStatXcpts[32];
2690 /** Interrupt statistics. */
2691 uint32_t aStatInts[256];
2692
2693#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING) && !defined(IEM_WITHOUT_INSTRUCTION_STATS)
2694 /** Instruction statistics for ring-0/raw-mode. */
2695 IEMINSTRSTATS StatsRZ;
2696 /** Instruction statistics for ring-3. */
2697 IEMINSTRSTATS StatsR3;
2698# ifdef VBOX_WITH_IEM_RECOMPILER
2699 /** Statistics per threaded function call.
2700 * Updated by both the threaded and native recompilers. */
2701 uint32_t acThreadedFuncStats[0x6000 /*24576*/];
2702# endif
2703#endif
2704} IEMCPU;
2705AssertCompileMemberOffset(IEMCPU, cActiveMappings, 0x4f);
2706AssertCompileMemberAlignment(IEMCPU, aMemMappings, 16);
2707AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
2708AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
2709AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
2710AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
2711AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
2712
2713/** Pointer to the per-CPU IEM state. */
2714typedef IEMCPU *PIEMCPU;
2715/** Pointer to the const per-CPU IEM state. */
2716typedef IEMCPU const *PCIEMCPU;
2717
2718/** @def IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED
2719 * Value indicating the TB didn't modified the floating point control register.
2720 * @note Neither FPCR nor MXCSR accept this as a valid value (MXCSR is not fully populated,
2721 * FPCR has the upper 32-bit reserved), so this is safe. */
2722#if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) || defined(DOXYGEN_RUNNING)
2723# ifdef RT_ARCH_AMD64
2724# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT32_MAX
2725# elif defined(RT_ARCH_ARM64)
2726# define IEMNATIVE_SIMD_FP_CTRL_REG_NOT_MODIFIED UINT64_MAX
2727# else
2728# error "Port me"
2729# endif
2730#endif
2731
2732/** @def IEM_GET_CTX
2733 * Gets the guest CPU context for the calling EMT.
2734 * @returns PCPUMCTX
2735 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2736 */
2737#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
2738
2739/** @def IEM_CTX_ASSERT
2740 * Asserts that the @a a_fExtrnMbz is present in the CPU context.
2741 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2742 * @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
2743 */
2744#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) \
2745 AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
2746 ("fExtrn=%#RX64 & fExtrnMbz=%#RX64 -> %#RX64\n", \
2747 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz), (a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz) ))
2748
2749/** @def IEM_CTX_IMPORT_RET
2750 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2751 *
2752 * Will call the keep to import the bits as needed.
2753 *
2754 * Returns on import failure.
2755 *
2756 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2757 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2758 */
2759#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
2760 do { \
2761 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2762 { /* likely */ } \
2763 else \
2764 { \
2765 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2766 AssertRCReturn(rcCtxImport, rcCtxImport); \
2767 } \
2768 } while (0)
2769
2770/** @def IEM_CTX_IMPORT_NORET
2771 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2772 *
2773 * Will call the keep to import the bits as needed.
2774 *
2775 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2776 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2777 */
2778#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
2779 do { \
2780 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2781 { /* likely */ } \
2782 else \
2783 { \
2784 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2785 AssertLogRelRC(rcCtxImport); \
2786 } \
2787 } while (0)
2788
2789/** @def IEM_CTX_IMPORT_JMP
2790 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
2791 *
2792 * Will call the keep to import the bits as needed.
2793 *
2794 * Jumps on import failure.
2795 *
2796 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2797 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
2798 */
2799#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
2800 do { \
2801 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
2802 { /* likely */ } \
2803 else \
2804 { \
2805 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
2806 AssertRCStmt(rcCtxImport, IEM_DO_LONGJMP(pVCpu, rcCtxImport)); \
2807 } \
2808 } while (0)
2809
2810
2811
2812/** @def IEM_GET_TARGET_CPU
2813 * Gets the current IEMTARGETCPU value.
2814 * @returns IEMTARGETCPU value.
2815 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2816 */
2817#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
2818# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
2819#else
2820# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
2821#endif
2822
2823
2824/** @def IEM_TRY_SETJMP
2825 * Wrapper around setjmp / try, hiding all the ugly differences.
2826 *
2827 * @note Use with extreme care as this is a fragile macro.
2828 * @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2829 * @param a_rcTarget The variable that should receive the status code in case
2830 * of a longjmp/throw.
2831 */
2832/** @def IEM_TRY_SETJMP_AGAIN
2833 * For when setjmp / try is used again in the same variable scope as a previous
2834 * IEM_TRY_SETJMP invocation.
2835 */
2836/** @def IEM_CATCH_LONGJMP_BEGIN
2837 * Start wrapper for catch / setjmp-else.
2838 *
2839 * This will set up a scope.
2840 *
2841 * @note Use with extreme care as this is a fragile macro.
2842 * @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2843 * @param a_rcTarget The variable that should receive the status code in case
2844 * of a longjmp/throw.
2845 */
2846/** @def IEM_CATCH_LONGJMP_END
2847 * End wrapper for catch / setjmp-else.
2848 *
2849 * This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the
2850 * state.
2851 *
2852 * @note Use with extreme care as this is a fragile macro.
2853 * @param a_pVCpu The cross context virtual CPU structure of the calling EMT.
2854 */
2855#ifdef IEM_WITH_THROW_CATCH
2856# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2857 a_rcTarget = VINF_SUCCESS; \
2858 try
2859# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2860 IEM_TRY_SETJMP(a_pVCpu, a_rcTarget)
2861# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2862 catch (int rcThrown) \
2863 { \
2864 a_rcTarget = rcThrown
2865# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2866 } \
2867 ((void)0)
2868#else /* !IEM_WITH_THROW_CATCH */
2869# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
2870 jmp_buf JmpBuf; \
2871 jmp_buf * volatile pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2872 (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2873 if ((rcStrict = setjmp(JmpBuf)) == 0)
2874# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
2875 pSavedJmpBuf = (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf); \
2876 (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
2877 if ((rcStrict = setjmp(JmpBuf)) == 0)
2878# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
2879 else \
2880 { \
2881 ((void)0)
2882# define IEM_CATCH_LONGJMP_END(a_pVCpu) \
2883 } \
2884 (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf
2885#endif /* !IEM_WITH_THROW_CATCH */
2886
2887
2888/**
2889 * Shared per-VM IEM data.
2890 */
2891typedef struct IEM
2892{
2893 /** The VMX APIC-access page handler type. */
2894 PGMPHYSHANDLERTYPE hVmxApicAccessPage;
2895#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
2896 /** Set if the CPUID host call functionality is enabled. */
2897 bool fCpuIdHostCall;
2898#endif
2899} IEM;
2900
2901
2902
2903/** @name IEM_ACCESS_XXX - Access details.
2904 * @{ */
2905#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
2906#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
2907#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
2908#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
2909#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
2910#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
2911#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
2912#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
2913#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
2914#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
2915/** The writes are partial, so if initialize the bounce buffer with the
2916 * orignal RAM content. */
2917#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
2918/** Used in aMemMappings to indicate that the entry is bounce buffered. */
2919#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
2920/** Bounce buffer with ring-3 write pending, first page. */
2921#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
2922/** Bounce buffer with ring-3 write pending, second page. */
2923#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
2924/** Not locked, accessed via the TLB. */
2925#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
2926/** Atomic access.
2927 * This enables special alignment checks and the VINF_EM_EMULATE_SPLIT_LOCK
2928 * fallback for misaligned stuff. See @bugref{10547}. */
2929#define IEM_ACCESS_ATOMIC UINT32_C(0x00002000)
2930/** Valid bit mask. */
2931#define IEM_ACCESS_VALID_MASK UINT32_C(0x00003fff)
2932/** Shift count for the TLB flags (upper word). */
2933#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
2934
2935/** Atomic read+write data alias. */
2936#define IEM_ACCESS_DATA_ATOMIC (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA | IEM_ACCESS_ATOMIC)
2937/** Read+write data alias. */
2938#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
2939/** Write data alias. */
2940#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
2941/** Read data alias. */
2942#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA)
2943/** Instruction fetch alias. */
2944#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_WHAT_CODE)
2945/** Stack write alias. */
2946#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
2947/** Stack read alias. */
2948#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_STACK)
2949/** Stack read+write alias. */
2950#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
2951/** Read system table alias. */
2952#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_SYS)
2953/** Read+write system table alias. */
2954#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_SYS)
2955/** @} */
2956
2957
2958/** @def IEM_DECL_MSC_GUARD_IGNORE
2959 * Disables control flow guards checks inside a method and any function pointers
2960 * referenced by it. */
2961#if defined(_MSC_VER) && !defined(IN_RING0)
2962# define IEM_DECL_MSC_GUARD_IGNORE __declspec(guard(ignore))
2963#else
2964# define IEM_DECL_MSC_GUARD_IGNORE
2965#endif
2966
2967/** @def IEM_DECL_MSC_GUARD_NONE
2968 * Disables control flow guards checks inside a method and but continue track
2969 * function pointers references by it. */
2970#if defined(_MSC_VER) && !defined(IN_RING0)
2971# define IEM_DECL_MSC_GUARD_NONE __declspec(guard(nocf))
2972#else
2973# define IEM_DECL_MSC_GUARD_NONE
2974#endif
2975
2976
2977/** @def IEM_DECL_IMPL_TYPE
2978 * For typedef'ing an instruction implementation function.
2979 *
2980 * @param a_RetType The return type.
2981 * @param a_Name The name of the type.
2982 * @param a_ArgList The argument list enclosed in parentheses.
2983 */
2984
2985/** @def IEM_DECL_IMPL_DEF
2986 * For defining an instruction implementation function.
2987 *
2988 * @param a_RetType The return type.
2989 * @param a_Name The name of the type.
2990 * @param a_ArgList The argument list enclosed in parentheses.
2991 */
2992
2993#if defined(__GNUC__) && defined(RT_ARCH_X86)
2994# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
2995 __attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
2996# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
2997 __attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
2998# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
2999 __attribute__((__fastcall__, __nothrow__)) DECL_HIDDEN_ONLY(a_RetType) a_Name a_ArgList
3000
3001#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
3002# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
3003 a_RetType (__fastcall a_Name) a_ArgList
3004# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
3005 IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
3006# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
3007 IEM_DECL_MSC_GUARD_IGNORE a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
3008
3009#elif __cplusplus >= 201700 /* P0012R1 support */
3010# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
3011 a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
3012# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
3013 IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
3014# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
3015 IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList RT_NOEXCEPT
3016
3017#else
3018# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
3019 a_RetType (VBOXCALL a_Name) a_ArgList
3020# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
3021 IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
3022# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
3023 IEM_DECL_MSC_GUARD_IGNORE DECL_HIDDEN_ONLY(a_RetType) VBOXCALL a_Name a_ArgList
3024
3025#endif
3026
3027
3028/** @name C instruction implementations for anything slightly complicated.
3029 * @{ */
3030
3031/**
3032 * For typedef'ing or declaring a C instruction implementation function taking
3033 * no extra arguments.
3034 *
3035 * @param a_Name The name of the type.
3036 */
3037# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
3038 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
3039/**
3040 * For defining a C instruction implementation function taking no extra
3041 * arguments.
3042 *
3043 * @param a_Name The name of the function
3044 */
3045# define IEM_CIMPL_DEF_0(a_Name) \
3046 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
3047/**
3048 * Prototype version of IEM_CIMPL_DEF_0.
3049 */
3050# define IEM_CIMPL_PROTO_0(a_Name) \
3051 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
3052/**
3053 * For calling a C instruction implementation function taking no extra
3054 * arguments.
3055 *
3056 * This special call macro adds default arguments to the call and allow us to
3057 * change these later.
3058 *
3059 * @param a_fn The name of the function.
3060 */
3061# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
3062
3063/** Type for a C instruction implementation function taking no extra
3064 * arguments. */
3065typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
3066/** Function pointer type for a C instruction implementation function taking
3067 * no extra arguments. */
3068typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
3069
3070/**
3071 * For typedef'ing or declaring a C instruction implementation function taking
3072 * one extra argument.
3073 *
3074 * @param a_Name The name of the type.
3075 * @param a_Type0 The argument type.
3076 * @param a_Arg0 The argument name.
3077 */
3078# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
3079 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
3080/**
3081 * For defining a C instruction implementation function taking one extra
3082 * argument.
3083 *
3084 * @param a_Name The name of the function
3085 * @param a_Type0 The argument type.
3086 * @param a_Arg0 The argument name.
3087 */
3088# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
3089 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
3090/**
3091 * Prototype version of IEM_CIMPL_DEF_1.
3092 */
3093# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
3094 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
3095/**
3096 * For calling a C instruction implementation function taking one extra
3097 * argument.
3098 *
3099 * This special call macro adds default arguments to the call and allow us to
3100 * change these later.
3101 *
3102 * @param a_fn The name of the function.
3103 * @param a0 The name of the 1st argument.
3104 */
3105# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
3106
3107/**
3108 * For typedef'ing or declaring a C instruction implementation function taking
3109 * two extra arguments.
3110 *
3111 * @param a_Name The name of the type.
3112 * @param a_Type0 The type of the 1st argument
3113 * @param a_Arg0 The name of the 1st argument.
3114 * @param a_Type1 The type of the 2nd argument.
3115 * @param a_Arg1 The name of the 2nd argument.
3116 */
3117# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
3118 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
3119/**
3120 * For defining a C instruction implementation function taking two extra
3121 * arguments.
3122 *
3123 * @param a_Name The name of the function.
3124 * @param a_Type0 The type of the 1st argument
3125 * @param a_Arg0 The name of the 1st argument.
3126 * @param a_Type1 The type of the 2nd argument.
3127 * @param a_Arg1 The name of the 2nd argument.
3128 */
3129# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
3130 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
3131/**
3132 * Prototype version of IEM_CIMPL_DEF_2.
3133 */
3134# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
3135 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
3136/**
3137 * For calling a C instruction implementation function taking two extra
3138 * arguments.
3139 *
3140 * This special call macro adds default arguments to the call and allow us to
3141 * change these later.
3142 *
3143 * @param a_fn The name of the function.
3144 * @param a0 The name of the 1st argument.
3145 * @param a1 The name of the 2nd argument.
3146 */
3147# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
3148
3149/**
3150 * For typedef'ing or declaring a C instruction implementation function taking
3151 * three extra arguments.
3152 *
3153 * @param a_Name The name of the type.
3154 * @param a_Type0 The type of the 1st argument
3155 * @param a_Arg0 The name of the 1st argument.
3156 * @param a_Type1 The type of the 2nd argument.
3157 * @param a_Arg1 The name of the 2nd argument.
3158 * @param a_Type2 The type of the 3rd argument.
3159 * @param a_Arg2 The name of the 3rd argument.
3160 */
3161# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3162 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3163/**
3164 * For defining a C instruction implementation function taking three extra
3165 * arguments.
3166 *
3167 * @param a_Name The name of the function.
3168 * @param a_Type0 The type of the 1st argument
3169 * @param a_Arg0 The name of the 1st argument.
3170 * @param a_Type1 The type of the 2nd argument.
3171 * @param a_Arg1 The name of the 2nd argument.
3172 * @param a_Type2 The type of the 3rd argument.
3173 * @param a_Arg2 The name of the 3rd argument.
3174 */
3175# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3176 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3177/**
3178 * Prototype version of IEM_CIMPL_DEF_3.
3179 */
3180# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
3181 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
3182/**
3183 * For calling a C instruction implementation function taking three extra
3184 * arguments.
3185 *
3186 * This special call macro adds default arguments to the call and allow us to
3187 * change these later.
3188 *
3189 * @param a_fn The name of the function.
3190 * @param a0 The name of the 1st argument.
3191 * @param a1 The name of the 2nd argument.
3192 * @param a2 The name of the 3rd argument.
3193 */
3194# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
3195
3196
3197/**
3198 * For typedef'ing or declaring a C instruction implementation function taking
3199 * four extra arguments.
3200 *
3201 * @param a_Name The name of the type.
3202 * @param a_Type0 The type of the 1st argument
3203 * @param a_Arg0 The name of the 1st argument.
3204 * @param a_Type1 The type of the 2nd argument.
3205 * @param a_Arg1 The name of the 2nd argument.
3206 * @param a_Type2 The type of the 3rd argument.
3207 * @param a_Arg2 The name of the 3rd argument.
3208 * @param a_Type3 The type of the 4th argument.
3209 * @param a_Arg3 The name of the 4th argument.
3210 */
3211# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3212 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
3213/**
3214 * For defining a C instruction implementation function taking four extra
3215 * arguments.
3216 *
3217 * @param a_Name The name of the function.
3218 * @param a_Type0 The type of the 1st argument
3219 * @param a_Arg0 The name of the 1st argument.
3220 * @param a_Type1 The type of the 2nd argument.
3221 * @param a_Arg1 The name of the 2nd argument.
3222 * @param a_Type2 The type of the 3rd argument.
3223 * @param a_Arg2 The name of the 3rd argument.
3224 * @param a_Type3 The type of the 4th argument.
3225 * @param a_Arg3 The name of the 4th argument.
3226 */
3227# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3228 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3229 a_Type2 a_Arg2, a_Type3 a_Arg3))
3230/**
3231 * Prototype version of IEM_CIMPL_DEF_4.
3232 */
3233# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
3234 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3235 a_Type2 a_Arg2, a_Type3 a_Arg3))
3236/**
3237 * For calling a C instruction implementation function taking four extra
3238 * arguments.
3239 *
3240 * This special call macro adds default arguments to the call and allow us to
3241 * change these later.
3242 *
3243 * @param a_fn The name of the function.
3244 * @param a0 The name of the 1st argument.
3245 * @param a1 The name of the 2nd argument.
3246 * @param a2 The name of the 3rd argument.
3247 * @param a3 The name of the 4th argument.
3248 */
3249# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
3250
3251
3252/**
3253 * For typedef'ing or declaring a C instruction implementation function taking
3254 * five extra arguments.
3255 *
3256 * @param a_Name The name of the type.
3257 * @param a_Type0 The type of the 1st argument
3258 * @param a_Arg0 The name of the 1st argument.
3259 * @param a_Type1 The type of the 2nd argument.
3260 * @param a_Arg1 The name of the 2nd argument.
3261 * @param a_Type2 The type of the 3rd argument.
3262 * @param a_Arg2 The name of the 3rd argument.
3263 * @param a_Type3 The type of the 4th argument.
3264 * @param a_Arg3 The name of the 4th argument.
3265 * @param a_Type4 The type of the 5th argument.
3266 * @param a_Arg4 The name of the 5th argument.
3267 */
3268# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3269 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
3270 a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
3271 a_Type3 a_Arg3, a_Type4 a_Arg4))
3272/**
3273 * For defining a C instruction implementation function taking five extra
3274 * arguments.
3275 *
3276 * @param a_Name The name of the function.
3277 * @param a_Type0 The type of the 1st argument
3278 * @param a_Arg0 The name of the 1st argument.
3279 * @param a_Type1 The type of the 2nd argument.
3280 * @param a_Arg1 The name of the 2nd argument.
3281 * @param a_Type2 The type of the 3rd argument.
3282 * @param a_Arg2 The name of the 3rd argument.
3283 * @param a_Type3 The type of the 4th argument.
3284 * @param a_Arg3 The name of the 4th argument.
3285 * @param a_Type4 The type of the 5th argument.
3286 * @param a_Arg4 The name of the 5th argument.
3287 */
3288# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3289 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3290 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
3291/**
3292 * Prototype version of IEM_CIMPL_DEF_5.
3293 */
3294# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
3295 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
3296 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
3297/**
3298 * For calling a C instruction implementation function taking five extra
3299 * arguments.
3300 *
3301 * This special call macro adds default arguments to the call and allow us to
3302 * change these later.
3303 *
3304 * @param a_fn The name of the function.
3305 * @param a0 The name of the 1st argument.
3306 * @param a1 The name of the 2nd argument.
3307 * @param a2 The name of the 3rd argument.
3308 * @param a3 The name of the 4th argument.
3309 * @param a4 The name of the 5th argument.
3310 */
3311# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
3312
3313/** @} */
3314
3315
3316/** @name Opcode Decoder Function Types.
3317 * @{ */
3318
3319/** @typedef PFNIEMOP
3320 * Pointer to an opcode decoder function.
3321 */
3322
3323/** @def FNIEMOP_DEF
3324 * Define an opcode decoder function.
3325 *
3326 * We're using macors for this so that adding and removing parameters as well as
3327 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
3328 *
3329 * @param a_Name The function name.
3330 */
3331
3332/** @typedef PFNIEMOPRM
3333 * Pointer to an opcode decoder function with RM byte.
3334 */
3335
3336/** @def FNIEMOPRM_DEF
3337 * Define an opcode decoder function with RM byte.
3338 *
3339 * We're using macors for this so that adding and removing parameters as well as
3340 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
3341 *
3342 * @param a_Name The function name.
3343 */
3344
3345#if defined(__GNUC__) && defined(RT_ARCH_X86)
3346typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
3347typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3348# define FNIEMOP_DEF(a_Name) \
3349 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
3350# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3351 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3352# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3353 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3354
3355#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
3356typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
3357typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3358# define FNIEMOP_DEF(a_Name) \
3359 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3360# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3361 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3362# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3363 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3364
3365#elif defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3366typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3367typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3368# define FNIEMOP_DEF(a_Name) \
3369 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
3370# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3371 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
3372# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3373 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
3374
3375#else
3376typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
3377typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
3378# define FNIEMOP_DEF(a_Name) \
3379 IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
3380# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
3381 IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) IEM_NOEXCEPT_MAY_LONGJMP
3382# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
3383 IEM_STATIC IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) IEM_NOEXCEPT_MAY_LONGJMP
3384
3385#endif
3386#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
3387
3388/**
3389 * Call an opcode decoder function.
3390 *
3391 * We're using macors for this so that adding and removing parameters can be
3392 * done as we please. See FNIEMOP_DEF.
3393 */
3394#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
3395
3396/**
3397 * Call a common opcode decoder function taking one extra argument.
3398 *
3399 * We're using macors for this so that adding and removing parameters can be
3400 * done as we please. See FNIEMOP_DEF_1.
3401 */
3402#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
3403
3404/**
3405 * Call a common opcode decoder function taking one extra argument.
3406 *
3407 * We're using macors for this so that adding and removing parameters can be
3408 * done as we please. See FNIEMOP_DEF_1.
3409 */
3410#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
3411/** @} */
3412
3413
3414/** @name Misc Helpers
3415 * @{ */
3416
3417/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
3418 * due to GCC lacking knowledge about the value range of a switch. */
3419#if RT_CPLUSPLUS_PREREQ(202000)
3420# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: [[unlikely]] AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3421#else
3422# define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
3423#endif
3424
3425/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
3426#if RT_CPLUSPLUS_PREREQ(202000)
3427# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: [[unlikely]] AssertFailedReturn(a_RetValue)
3428#else
3429# define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
3430#endif
3431
3432/**
3433 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3434 * occation.
3435 */
3436#ifdef LOG_ENABLED
3437# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3438 do { \
3439 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
3440 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3441 } while (0)
3442#else
3443# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
3444 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3445#endif
3446
3447/**
3448 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
3449 * occation using the supplied logger statement.
3450 *
3451 * @param a_LoggerArgs What to log on failure.
3452 */
3453#ifdef LOG_ENABLED
3454# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3455 do { \
3456 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
3457 /*LogFunc(a_LoggerArgs);*/ \
3458 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
3459 } while (0)
3460#else
3461# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
3462 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
3463#endif
3464
3465
3466/** @} */
3467
3468uint32_t iemCalcExecDbgFlagsSlow(PVMCPUCC pVCpu);
3469VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu);
3470
3471/** @} */
3472
3473
3474/** @name Memory access.
3475 * @{ */
3476VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, uint8_t *pbUnmapInfo,
3477 size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) RT_NOEXCEPT;
3478VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, uint8_t *pbUnmapInfo, size_t cbMem,
3479 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) RT_NOEXCEPT;
3480VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3481#ifndef IN_RING3
3482VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3483#endif
3484void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3485void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3486
3487void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3488void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3489void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3490void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3491void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
3492void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT;
3493
3494VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t cbAccess, uint32_t fAccess,
3495 PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
3496
3497void iemTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3498/** @} */
3499
3500/*
3501 * Recompiler related stuff.
3502 */
3503
3504DECLHIDDEN(int) iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
3505
3506DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
3507 uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
3508void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree);
3509DECLHIDDEN(void) iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
3510void iemTbAllocatorProcessDelayedFrees(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator);
3511void iemTbAllocatorFreeupNativeSpace(PVMCPUCC pVCpu, uint32_t cNeededInstrs);
3512DECLHIDDEN(PIEMTBALLOCATOR) iemTbAllocatorFreeBulkStart(PVMCPUCC pVCpu);
3513DECLHIDDEN(void) iemTbAllocatorFreeBulk(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator, PIEMTB pTb);
3514DECLHIDDEN(const char *) iemTbFlagsToString(uint32_t fFlags, char *pszBuf, size_t cbBuf) RT_NOEXCEPT;
3515DECLHIDDEN(void) iemThreadedDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3516#if defined(VBOX_WITH_IEM_NATIVE_RECOMPILER) && defined(VBOX_WITH_SAVE_THREADED_TBS_FOR_PROFILING)
3517DECLHIDDEN(void) iemThreadedSaveTbForProfilingCleanup(PVMCPU pVCpu);
3518#endif
3519
3520
3521/** @todo FNIEMTHREADEDFUNC and friends may need more work... */
3522#if defined(__GNUC__) && !defined(IEM_WITH_THROW_CATCH)
3523typedef VBOXSTRICTRC /*__attribute__((__nothrow__))*/ FNIEMTHREADEDFUNC(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3524typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3525# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3526 VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3527# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3528 VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2)
3529
3530#else
3531typedef VBOXSTRICTRC (FNIEMTHREADEDFUNC)(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
3532typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;
3533# define IEM_DECL_IEMTHREADEDFUNC_DEF(a_Name) \
3534 IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3535# define IEM_DECL_IEMTHREADEDFUNC_PROTO(a_Name) \
3536 IEM_DECL_MSC_GUARD_IGNORE VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint64_t uParam0, uint64_t uParam1, uint64_t uParam2) IEM_NOEXCEPT_MAY_LONGJMP
3537#endif
3538
3539
3540/* Native recompiler public bits: */
3541
3542DECLHIDDEN(PIEMTB) iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb) RT_NOEXCEPT;
3543DECLHIDDEN(void) iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
3544int iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT;
3545DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, PIEMNATIVEINSTR *ppaExec,
3546 struct IEMNATIVEPERCHUNKCTX const **ppChunkCtx) RT_NOEXCEPT;
3547DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq,
3548 PIEMNATIVEINSTR *ppaExec);
3549DECLHIDDEN(void) iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3550void iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
3551DECLASM(DECL_NO_RETURN(void)) iemNativeTbLongJmp(void *pvFramePointer, int rc) RT_NOEXCEPT;
3552DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb);
3553DECLHIDDEN(int) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk, struct IEMNATIVEPERCHUNKCTX const **ppCtx);
3554
3555# ifdef VBOX_VMM_TARGET_X86
3556# include "VMMAll/target-x86/IEMInternal-x86.h"
3557# elif defined(VBOX_VMM_TARGET_ARMV8)
3558# include "VMMAll/target-armv8/IEMInternal-armv8.h"
3559# endif
3560
3561#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3562
3563
3564/** @} */
3565
3566RT_C_DECLS_END
3567
3568/* ASM-INC: %include "IEMInternalStruct.mac" */
3569
3570#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3571
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette