VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 95560

Last change on this file since 95560 was 95560, checked in by vboxsync, 3 years ago

VMM/IEM: Some crude exception/interrupt stats and history. bugref:9898
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 167.5 KB
Line 
1/* $Id: IEMInternal.h 95560 2022-07-07 23:43:55Z vboxsync $ */
2/** @file
3 * IEM - Internal header file.
4 */
5
6/*
7 * Copyright (C) 2011-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
19#define VMM_INCLUDED_SRC_include_IEMInternal_h
20#ifndef RT_WITHOUT_PRAGMA_ONCE
21# pragma once
22#endif
23
24#include <VBox/vmm/cpum.h>
25#include <VBox/vmm/iem.h>
26#include <VBox/vmm/pgm.h>
27#include <VBox/vmm/stam.h>
28#include <VBox/param.h>
29
30#include <iprt/setjmp-without-sigmask.h>
31
32
33RT_C_DECLS_BEGIN
34
35
36/** @defgroup grp_iem_int Internals
37 * @ingroup grp_iem
38 * @internal
39 * @{
40 */
41
42/** For expanding symbol in slickedit and other products tagging and
43 * crossreferencing IEM symbols. */
44#ifndef IEM_STATIC
45# define IEM_STATIC static
46#endif
47
48/** @def IEM_WITH_SETJMP
49 * Enables alternative status code handling using setjmps.
50 *
51 * This adds a bit of expense via the setjmp() call since it saves all the
52 * non-volatile registers. However, it eliminates return code checks and allows
53 * for more optimal return value passing (return regs instead of stack buffer).
54 */
55#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
56# define IEM_WITH_SETJMP
57#endif
58
59#define IEM_IMPLEMENTS_TASKSWITCH
60
61/** @def IEM_WITH_3DNOW
62 * Includes the 3DNow decoding. */
63#define IEM_WITH_3DNOW
64
65/** @def IEM_WITH_THREE_0F_38
66 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
67#define IEM_WITH_THREE_0F_38
68
69/** @def IEM_WITH_THREE_0F_3A
70 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
71#define IEM_WITH_THREE_0F_3A
72
73/** @def IEM_WITH_VEX
74 * Includes the VEX decoding. */
75#define IEM_WITH_VEX
76
77/** @def IEM_CFG_TARGET_CPU
78 * The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
79 *
80 * By default we allow this to be configured by the user via the
81 * CPUM/GuestCpuName config string, but this comes at a slight cost during
82 * decoding. So, for applications of this code where there is no need to
83 * be dynamic wrt target CPU, just modify this define.
84 */
85#if !defined(IEM_CFG_TARGET_CPU) || defined(DOXYGEN_RUNNING)
86# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
87#endif
88
89//#define IEM_WITH_CODE_TLB // - work in progress
90//#define IEM_WITH_DATA_TLB // - work in progress
91
92
93/** @def IEM_USE_UNALIGNED_DATA_ACCESS
94 * Use unaligned accesses instead of elaborate byte assembly. */
95#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
96# define IEM_USE_UNALIGNED_DATA_ACCESS
97#endif
98
99//#define IEM_LOG_MEMORY_WRITES
100
101#if !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
102/** Instruction statistics. */
103typedef struct IEMINSTRSTATS
104{
105# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
106# include "IEMInstructionStatisticsTmpl.h"
107# undef IEM_DO_INSTR_STAT
108} IEMINSTRSTATS;
109#else
110struct IEMINSTRSTATS;
111typedef struct IEMINSTRSTATS IEMINSTRSTATS;
112#endif
113/** Pointer to IEM instruction statistics. */
114typedef IEMINSTRSTATS *PIEMINSTRSTATS;
115
116
117/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
118 * @{ */
119#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /**< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. */
120#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /**< Intel EFLAGS result. */
121#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /**< AMD EFLAGS result */
122#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /**< Reserved/dummy entry slot that's the same as 0. */
123#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /**< For masking the index before use. */
124/** Selects the right variant from a_aArray.
125 * pVCpu is implicit in the caller context. */
126#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
127 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
128/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
129 * be used because the host CPU does not support the operation. */
130#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
131 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
132/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
133 * array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
134 * into the two.
135 * @sa IEM_SELECT_NATIVE_OR_FALLBACK */
136#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
137# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
138 (a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
139#else
140# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
141 (a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
142#endif
143/** @} */
144
145/**
146 * Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
147 * indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
148 *
149 * On non-x86 hosts, this will shortcut to the fallback w/o checking the
150 * indicator.
151 *
152 * @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
153 */
154#if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
155# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
156 (g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
157#else
158# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
159#endif
160
161
162/**
163 * Extended operand mode that includes a representation of 8-bit.
164 *
165 * This is used for packing down modes when invoking some C instruction
166 * implementations.
167 */
168typedef enum IEMMODEX
169{
170 IEMMODEX_16BIT = IEMMODE_16BIT,
171 IEMMODEX_32BIT = IEMMODE_32BIT,
172 IEMMODEX_64BIT = IEMMODE_64BIT,
173 IEMMODEX_8BIT
174} IEMMODEX;
175AssertCompileSize(IEMMODEX, 4);
176
177
178/**
179 * Branch types.
180 */
181typedef enum IEMBRANCH
182{
183 IEMBRANCH_JUMP = 1,
184 IEMBRANCH_CALL,
185 IEMBRANCH_TRAP,
186 IEMBRANCH_SOFTWARE_INT,
187 IEMBRANCH_HARDWARE_INT
188} IEMBRANCH;
189AssertCompileSize(IEMBRANCH, 4);
190
191
192/**
193 * INT instruction types.
194 */
195typedef enum IEMINT
196{
197 /** INT n instruction (opcode 0xcd imm). */
198 IEMINT_INTN = 0,
199 /** Single byte INT3 instruction (opcode 0xcc). */
200 IEMINT_INT3 = IEM_XCPT_FLAGS_BP_INSTR,
201 /** Single byte INTO instruction (opcode 0xce). */
202 IEMINT_INTO = IEM_XCPT_FLAGS_OF_INSTR,
203 /** Single byte INT1 (ICEBP) instruction (opcode 0xf1). */
204 IEMINT_INT1 = IEM_XCPT_FLAGS_ICEBP_INSTR
205} IEMINT;
206AssertCompileSize(IEMINT, 4);
207
208
209/**
210 * A FPU result.
211 */
212typedef struct IEMFPURESULT
213{
214 /** The output value. */
215 RTFLOAT80U r80Result;
216 /** The output status. */
217 uint16_t FSW;
218} IEMFPURESULT;
219AssertCompileMemberOffset(IEMFPURESULT, FSW, 10);
220/** Pointer to a FPU result. */
221typedef IEMFPURESULT *PIEMFPURESULT;
222/** Pointer to a const FPU result. */
223typedef IEMFPURESULT const *PCIEMFPURESULT;
224
225
226/**
227 * A FPU result consisting of two output values and FSW.
228 */
229typedef struct IEMFPURESULTTWO
230{
231 /** The first output value. */
232 RTFLOAT80U r80Result1;
233 /** The output status. */
234 uint16_t FSW;
235 /** The second output value. */
236 RTFLOAT80U r80Result2;
237} IEMFPURESULTTWO;
238AssertCompileMemberOffset(IEMFPURESULTTWO, FSW, 10);
239AssertCompileMemberOffset(IEMFPURESULTTWO, r80Result2, 12);
240/** Pointer to a FPU result consisting of two output values and FSW. */
241typedef IEMFPURESULTTWO *PIEMFPURESULTTWO;
242/** Pointer to a const FPU result consisting of two output values and FSW. */
243typedef IEMFPURESULTTWO const *PCIEMFPURESULTTWO;
244
245
246/**
247 * IEM TLB entry.
248 *
249 * Lookup assembly:
250 * @code{.asm}
251 ; Calculate tag.
252 mov rax, [VA]
253 shl rax, 16
254 shr rax, 16 + X86_PAGE_SHIFT
255 or rax, [uTlbRevision]
256
257 ; Do indexing.
258 movzx ecx, al
259 lea rcx, [pTlbEntries + rcx]
260
261 ; Check tag.
262 cmp [rcx + IEMTLBENTRY.uTag], rax
263 jne .TlbMiss
264
265 ; Check access.
266 mov rax, ACCESS_FLAGS | MAPPING_R3_NOT_VALID | 0xffffff00
267 and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
268 cmp rax, [uTlbPhysRev]
269 jne .TlbMiss
270
271 ; Calc address and we're done.
272 mov eax, X86_PAGE_OFFSET_MASK
273 and eax, [VA]
274 or rax, [rcx + IEMTLBENTRY.pMappingR3]
275 %ifdef VBOX_WITH_STATISTICS
276 inc qword [cTlbHits]
277 %endif
278 jmp .Done
279
280 .TlbMiss:
281 mov r8d, ACCESS_FLAGS
282 mov rdx, [VA]
283 mov rcx, [pVCpu]
284 call iemTlbTypeMiss
285 .Done:
286
287 @endcode
288 *
289 */
290typedef struct IEMTLBENTRY
291{
292 /** The TLB entry tag.
293 * Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
294 * is ASSUMING a virtual address width of 48 bits.
295 *
296 * Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
297 *
298 * The TLB lookup code uses the current TLB revision, which won't ever be zero,
299 * enabling an extremely cheap TLB invalidation most of the time. When the TLB
300 * revision wraps around though, the tags needs to be zeroed.
301 *
302 * @note Try use SHRD instruction? After seeing
303 * https://gmplib.org/~tege/x86-timing.pdf, maybe not.
304 *
305 * @todo This will need to be reorganized for 57-bit wide virtual address and
306 * PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
307 * have to move the TLB entry versioning entirely to the
308 * fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
309 * 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
310 * consumed by PCID and ASID (12 + 6 = 18).
311 */
312 uint64_t uTag;
313 /** Access flags and physical TLB revision.
314 *
315 * - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
316 * - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
317 * - Bit 2 - page tables - not user (complemented X86_PTE_US).
318 * - Bit 3 - pgm phys/virt - not directly writable.
319 * - Bit 4 - pgm phys page - not directly readable.
320 * - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
321 * - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
322 * - Bit 7 - tlb entry - pMappingR3 member not valid.
323 * - Bits 63 thru 8 are used for the physical TLB revision number.
324 *
325 * We're using complemented bit meanings here because it makes it easy to check
326 * whether special action is required. For instance a user mode write access
327 * would do a "TEST fFlags, (X86_PTE_RW | X86_PTE_US | X86_PTE_D)" and a
328 * non-zero result would mean special handling needed because either it wasn't
329 * writable, or it wasn't user, or the page wasn't dirty. A user mode read
330 * access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
331 * need to check any PTE flag.
332 */
333 uint64_t fFlagsAndPhysRev;
334 /** The guest physical page address. */
335 uint64_t GCPhys;
336 /** Pointer to the ring-3 mapping. */
337 R3PTRTYPE(uint8_t *) pbMappingR3;
338#if HC_ARCH_BITS == 32
339 uint32_t u32Padding1;
340#endif
341} IEMTLBENTRY;
342AssertCompileSize(IEMTLBENTRY, 32);
343/** Pointer to an IEM TLB entry. */
344typedef IEMTLBENTRY *PIEMTLBENTRY;
345
346/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
347 * @{ */
348#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /**< Page tables: Not executable. */
349#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /**< Page tables: Not writable. */
350#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /**< Page tables: Not user accessible (supervisor only). */
351#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /**< Phys page: Not writable (access handler, ROM, whatever). */
352#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /**< Phys page: Not readable (MMIO / access handler, ROM) */
353#define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /**< Phys tables: Not accessed (need to be marked accessed). */
354#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /**< Page tables: Not dirty (needs to be made dirty on write). */
355#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(7) /**< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. */
356#define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(8) /**< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). */
357#define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffffe00) /**< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR */
358/** @} */
359
360
361/**
362 * An IEM TLB.
363 *
364 * We've got two of these, one for data and one for instructions.
365 */
366typedef struct IEMTLB
367{
368 /** The TLB entries.
369 * We've choosen 256 because that way we can obtain the result directly from a
370 * 8-bit register without an additional AND instruction. */
371 IEMTLBENTRY aEntries[256];
372 /** The TLB revision.
373 * This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
374 * by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
375 * the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
376 * (The revision zero indicates an invalid TLB entry.)
377 *
378 * The initial value is choosen to cause an early wraparound. */
379 uint64_t uTlbRevision;
380 /** The TLB physical address revision - shadow of PGM variable.
381 *
382 * This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
383 * incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
384 * a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
385 * as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
386 *
387 * The initial value is choosen to cause an early wraparound. */
388 uint64_t volatile uTlbPhysRev;
389
390 /* Statistics: */
391
392 /** TLB hits (VBOX_WITH_STATISTICS only). */
393 uint64_t cTlbHits;
394 /** TLB misses. */
395 uint32_t cTlbMisses;
396 /** Slow read path. */
397 uint32_t cTlbSlowReadPath;
398#if 0
399 /** TLB misses because of tag mismatch. */
400 uint32_t cTlbMissesTag;
401 /** TLB misses because of virtual access violation. */
402 uint32_t cTlbMissesVirtAccess;
403 /** TLB misses because of dirty bit. */
404 uint32_t cTlbMissesDirty;
405 /** TLB misses because of MMIO */
406 uint32_t cTlbMissesMmio;
407 /** TLB misses because of write access handlers. */
408 uint32_t cTlbMissesWriteHandler;
409 /** TLB misses because no r3(/r0) mapping. */
410 uint32_t cTlbMissesMapping;
411#endif
412 /** Alignment padding. */
413 uint32_t au32Padding[3+5];
414} IEMTLB;
415AssertCompileSizeAlignment(IEMTLB, 64);
416/** IEMTLB::uTlbRevision increment. */
417#define IEMTLB_REVISION_INCR RT_BIT_64(36)
418/** IEMTLB::uTlbRevision mask. */
419#define IEMTLB_REVISION_MASK (~(RT_BIT_64(36) - 1))
420/** IEMTLB::uTlbPhysRev increment.
421 * @sa IEMTLBE_F_PHYS_REV */
422#define IEMTLB_PHYS_REV_INCR RT_BIT_64(9)
423/**
424 * Calculates the TLB tag for a virtual address.
425 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
426 * @param a_pTlb The TLB.
427 * @param a_GCPtr The virtual address.
428 */
429#define IEMTLB_CALC_TAG(a_pTlb, a_GCPtr) ( IEMTLB_CALC_TAG_NO_REV(a_GCPtr) | (a_pTlb)->uTlbRevision )
430/**
431 * Calculates the TLB tag for a virtual address but without TLB revision.
432 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
433 * @param a_GCPtr The virtual address.
434 */
435#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
436/**
437 * Converts a TLB tag value into a TLB index.
438 * @returns Index into IEMTLB::aEntries.
439 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
440 */
441#define IEMTLB_TAG_TO_INDEX(a_uTag) ( (uint8_t)(a_uTag) )
442/**
443 * Converts a TLB tag value into a TLB index.
444 * @returns Index into IEMTLB::aEntries.
445 * @param a_pTlb The TLB.
446 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
447 */
448#define IEMTLB_TAG_TO_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_INDEX(a_uTag)] )
449
450
451/**
452 * The per-CPU IEM state.
453 */
454typedef struct IEMCPU
455{
456 /** Info status code that needs to be propagated to the IEM caller.
457 * This cannot be passed internally, as it would complicate all success
458 * checks within the interpreter making the code larger and almost impossible
459 * to get right. Instead, we'll store status codes to pass on here. Each
460 * source of these codes will perform appropriate sanity checks. */
461 int32_t rcPassUp; /* 0x00 */
462
463 /** The current CPU execution mode (CS). */
464 IEMMODE enmCpuMode; /* 0x04 */
465 /** The CPL. */
466 uint8_t uCpl; /* 0x05 */
467
468 /** Whether to bypass access handlers or not. */
469 bool fBypassHandlers; /* 0x06 */
470 /** Whether to disregard the lock prefix (implied or not). */
471 bool fDisregardLock; /* 0x07 */
472
473 /** @name Decoder state.
474 * @{ */
475#ifdef IEM_WITH_CODE_TLB
476 /** The offset of the next instruction byte. */
477 uint32_t offInstrNextByte; /* 0x08 */
478 /** The number of bytes available at pbInstrBuf for the current instruction.
479 * This takes the max opcode length into account so that doesn't need to be
480 * checked separately. */
481 uint32_t cbInstrBuf; /* 0x0c */
482 /** Pointer to the page containing RIP, user specified buffer or abOpcode.
483 * This can be NULL if the page isn't mappable for some reason, in which
484 * case we'll do fallback stuff.
485 *
486 * If we're executing an instruction from a user specified buffer,
487 * IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
488 * aligned pointer but pointer to the user data.
489 *
490 * For instructions crossing pages, this will start on the first page and be
491 * advanced to the next page by the time we've decoded the instruction. This
492 * therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
493 */
494 uint8_t const *pbInstrBuf; /* 0x10 */
495# if ARCH_BITS == 32
496 uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
497# endif
498 /** The program counter corresponding to pbInstrBuf.
499 * This is set to a non-canonical address when we need to invalidate it. */
500 uint64_t uInstrBufPc; /* 0x18 */
501 /** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
502 * This takes the CS segment limit into account. */
503 uint16_t cbInstrBufTotal; /* 0x20 */
504 /** Offset into pbInstrBuf of the first byte of the current instruction.
505 * Can be negative to efficiently handle cross page instructions. */
506 int16_t offCurInstrStart; /* 0x22 */
507
508 /** The prefix mask (IEM_OP_PRF_XXX). */
509 uint32_t fPrefixes; /* 0x24 */
510 /** The extra REX ModR/M register field bit (REX.R << 3). */
511 uint8_t uRexReg; /* 0x28 */
512 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
513 * (REX.B << 3). */
514 uint8_t uRexB; /* 0x29 */
515 /** The extra REX SIB index field bit (REX.X << 3). */
516 uint8_t uRexIndex; /* 0x2a */
517
518 /** The effective segment register (X86_SREG_XXX). */
519 uint8_t iEffSeg; /* 0x2b */
520
521 /** The offset of the ModR/M byte relative to the start of the instruction. */
522 uint8_t offModRm; /* 0x2c */
523#else
524 /** The size of what has currently been fetched into abOpcode. */
525 uint8_t cbOpcode; /* 0x08 */
526 /** The current offset into abOpcode. */
527 uint8_t offOpcode; /* 0x09 */
528 /** The offset of the ModR/M byte relative to the start of the instruction. */
529 uint8_t offModRm; /* 0x0a */
530
531 /** The effective segment register (X86_SREG_XXX). */
532 uint8_t iEffSeg; /* 0x0b */
533
534 /** The prefix mask (IEM_OP_PRF_XXX). */
535 uint32_t fPrefixes; /* 0x0c */
536 /** The extra REX ModR/M register field bit (REX.R << 3). */
537 uint8_t uRexReg; /* 0x10 */
538 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
539 * (REX.B << 3). */
540 uint8_t uRexB; /* 0x11 */
541 /** The extra REX SIB index field bit (REX.X << 3). */
542 uint8_t uRexIndex; /* 0x12 */
543
544#endif
545
546 /** The effective operand mode. */
547 IEMMODE enmEffOpSize; /* 0x2d, 0x13 */
548 /** The default addressing mode. */
549 IEMMODE enmDefAddrMode; /* 0x2e, 0x14 */
550 /** The effective addressing mode. */
551 IEMMODE enmEffAddrMode; /* 0x2f, 0x15 */
552 /** The default operand mode. */
553 IEMMODE enmDefOpSize; /* 0x30, 0x16 */
554
555 /** Prefix index (VEX.pp) for two byte and three byte tables. */
556 uint8_t idxPrefix; /* 0x31, 0x17 */
557 /** 3rd VEX/EVEX/XOP register.
558 * Please use IEM_GET_EFFECTIVE_VVVV to access. */
559 uint8_t uVex3rdReg; /* 0x32, 0x18 */
560 /** The VEX/EVEX/XOP length field. */
561 uint8_t uVexLength; /* 0x33, 0x19 */
562 /** Additional EVEX stuff. */
563 uint8_t fEvexStuff; /* 0x34, 0x1a */
564
565 /** Explicit alignment padding. */
566 uint8_t abAlignment2a[1]; /* 0x35, 0x1b */
567 /** The FPU opcode (FOP). */
568 uint16_t uFpuOpcode; /* 0x36, 0x1c */
569#ifndef IEM_WITH_CODE_TLB
570 /** Explicit alignment padding. */
571 uint8_t abAlignment2b[2]; /* 0x1e */
572#endif
573
574 /** The opcode bytes. */
575 uint8_t abOpcode[15]; /* 0x48, 0x20 */
576 /** Explicit alignment padding. */
577#ifdef IEM_WITH_CODE_TLB
578 uint8_t abAlignment2c[0x48 - 0x47]; /* 0x37 */
579#else
580 uint8_t abAlignment2c[0x48 - 0x2f]; /* 0x2f */
581#endif
582 /** @} */
583
584
585 /** The flags of the current exception / interrupt. */
586 uint32_t fCurXcpt; /* 0x48, 0x48 */
587 /** The current exception / interrupt. */
588 uint8_t uCurXcpt;
589 /** Exception / interrupt recursion depth. */
590 int8_t cXcptRecursions;
591
592 /** The number of active guest memory mappings. */
593 uint8_t cActiveMappings;
594 /** The next unused mapping index. */
595 uint8_t iNextMapping;
596 /** Records for tracking guest memory mappings. */
597 struct
598 {
599 /** The address of the mapped bytes. */
600 void *pv;
601 /** The access flags (IEM_ACCESS_XXX).
602 * IEM_ACCESS_INVALID if the entry is unused. */
603 uint32_t fAccess;
604#if HC_ARCH_BITS == 64
605 uint32_t u32Alignment4; /**< Alignment padding. */
606#endif
607 } aMemMappings[3];
608
609 /** Locking records for the mapped memory. */
610 union
611 {
612 PGMPAGEMAPLOCK Lock;
613 uint64_t au64Padding[2];
614 } aMemMappingLocks[3];
615
616 /** Bounce buffer info.
617 * This runs in parallel to aMemMappings. */
618 struct
619 {
620 /** The physical address of the first byte. */
621 RTGCPHYS GCPhysFirst;
622 /** The physical address of the second page. */
623 RTGCPHYS GCPhysSecond;
624 /** The number of bytes in the first page. */
625 uint16_t cbFirst;
626 /** The number of bytes in the second page. */
627 uint16_t cbSecond;
628 /** Whether it's unassigned memory. */
629 bool fUnassigned;
630 /** Explicit alignment padding. */
631 bool afAlignment5[3];
632 } aMemBbMappings[3];
633
634 /** Bounce buffer storage.
635 * This runs in parallel to aMemMappings and aMemBbMappings. */
636 struct
637 {
638 uint8_t ab[512];
639 } aBounceBuffers[3];
640
641
642 /** Pointer set jump buffer - ring-3 context. */
643 R3PTRTYPE(jmp_buf *) pJmpBufR3;
644 /** Pointer set jump buffer - ring-0 context. */
645 R0PTRTYPE(jmp_buf *) pJmpBufR0;
646
647 /** @todo Should move this near @a fCurXcpt later. */
648 /** The CR2 for the current exception / interrupt. */
649 uint64_t uCurXcptCr2;
650 /** The error code for the current exception / interrupt. */
651 uint32_t uCurXcptErr;
652
653 /** @name Statistics
654 * @{ */
655 /** The number of instructions we've executed. */
656 uint32_t cInstructions;
657 /** The number of potential exits. */
658 uint32_t cPotentialExits;
659 /** The number of bytes data or stack written (mostly for IEMExecOneEx).
660 * This may contain uncommitted writes. */
661 uint32_t cbWritten;
662 /** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
663 uint32_t cRetInstrNotImplemented;
664 /** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
665 uint32_t cRetAspectNotImplemented;
666 /** Counts informational statuses returned (other than VINF_SUCCESS). */
667 uint32_t cRetInfStatuses;
668 /** Counts other error statuses returned. */
669 uint32_t cRetErrStatuses;
670 /** Number of times rcPassUp has been used. */
671 uint32_t cRetPassUpStatus;
672 /** Number of times RZ left with instruction commit pending for ring-3. */
673 uint32_t cPendingCommit;
674 /** Number of long jumps. */
675 uint32_t cLongJumps;
676 /** @} */
677
678 /** @name Target CPU information.
679 * @{ */
680#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
681 /** The target CPU. */
682 uint8_t uTargetCpu;
683#else
684 uint8_t bTargetCpuPadding;
685#endif
686 /** For selecting assembly works matching the target CPU EFLAGS behaviour, see
687 * IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
688 * native host support and the 2nd for when there is.
689 *
690 * The two values are typically indexed by a g_CpumHostFeatures bit.
691 *
692 * This is for instance used for the BSF & BSR instructions where AMD and
693 * Intel CPUs produce different EFLAGS. */
694 uint8_t aidxTargetCpuEflFlavour[2];
695
696 /** The CPU vendor. */
697 CPUMCPUVENDOR enmCpuVendor;
698 /** @} */
699
700 /** @name Host CPU information.
701 * @{ */
702 /** The CPU vendor. */
703 CPUMCPUVENDOR enmHostCpuVendor;
704 /** @} */
705
706 /** Counts RDMSR \#GP(0) LogRel(). */
707 uint8_t cLogRelRdMsr;
708 /** Counts WRMSR \#GP(0) LogRel(). */
709 uint8_t cLogRelWrMsr;
710 /** Alignment padding. */
711 uint8_t abAlignment8[50];
712
713 /** Data TLB.
714 * @remarks Must be 64-byte aligned. */
715 IEMTLB DataTlb;
716 /** Instruction TLB.
717 * @remarks Must be 64-byte aligned. */
718 IEMTLB CodeTlb;
719
720 /** Exception and interrupt statistics. */
721 STAMCOUNTER aStatXcpts[256];
722
723#if defined(VBOX_WITH_STATISTICS) && !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
724 /** Instruction statistics for ring-0/raw-mode. */
725 IEMINSTRSTATS StatsRZ;
726 /** Instruction statistics for ring-3. */
727 IEMINSTRSTATS StatsR3;
728#endif
729} IEMCPU;
730AssertCompileMemberOffset(IEMCPU, fCurXcpt, 0x48);
731AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
732AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
733/** Pointer to the per-CPU IEM state. */
734typedef IEMCPU *PIEMCPU;
735/** Pointer to the const per-CPU IEM state. */
736typedef IEMCPU const *PCIEMCPU;
737
738
739/** @def IEM_GET_CTX
740 * Gets the guest CPU context for the calling EMT.
741 * @returns PCPUMCTX
742 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
743 */
744#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
745
746/** @def IEM_CTX_ASSERT
747 * Asserts that the @a a_fExtrnMbz is present in the CPU context.
748 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
749 * @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
750 */
751#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
752 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (a_pVCpu)->cpum.GstCtx.fExtrn, \
753 (a_fExtrnMbz)))
754
755/** @def IEM_CTX_IMPORT_RET
756 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
757 *
758 * Will call the keep to import the bits as needed.
759 *
760 * Returns on import failure.
761 *
762 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
763 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
764 */
765#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
766 do { \
767 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
768 { /* likely */ } \
769 else \
770 { \
771 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
772 AssertRCReturn(rcCtxImport, rcCtxImport); \
773 } \
774 } while (0)
775
776/** @def IEM_CTX_IMPORT_NORET
777 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
778 *
779 * Will call the keep to import the bits as needed.
780 *
781 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
782 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
783 */
784#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
785 do { \
786 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
787 { /* likely */ } \
788 else \
789 { \
790 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
791 AssertLogRelRC(rcCtxImport); \
792 } \
793 } while (0)
794
795/** @def IEM_CTX_IMPORT_JMP
796 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
797 *
798 * Will call the keep to import the bits as needed.
799 *
800 * Jumps on import failure.
801 *
802 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
803 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
804 */
805#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
806 do { \
807 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
808 { /* likely */ } \
809 else \
810 { \
811 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
812 AssertRCStmt(rcCtxImport, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), rcCtxImport)); \
813 } \
814 } while (0)
815
816
817
818/** @def IEM_GET_TARGET_CPU
819 * Gets the current IEMTARGETCPU value.
820 * @returns IEMTARGETCPU value.
821 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
822 */
823#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
824# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
825#else
826# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
827#endif
828
829/** @def IEM_GET_INSTR_LEN
830 * Gets the instruction length. */
831#ifdef IEM_WITH_CODE_TLB
832# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
833#else
834# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
835#endif
836
837
838/**
839 * Shared per-VM IEM data.
840 */
841typedef struct IEM
842{
843 /** The VMX APIC-access page handler type. */
844 PGMPHYSHANDLERTYPE hVmxApicAccessPage;
845} IEM;
846
847
848
849/** @name IEM_ACCESS_XXX - Access details.
850 * @{ */
851#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
852#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
853#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
854#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
855#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
856#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
857#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
858#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
859#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
860#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
861/** The writes are partial, so if initialize the bounce buffer with the
862 * orignal RAM content. */
863#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
864/** Used in aMemMappings to indicate that the entry is bounce buffered. */
865#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
866/** Bounce buffer with ring-3 write pending, first page. */
867#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
868/** Bounce buffer with ring-3 write pending, second page. */
869#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
870/** Not locked, accessed via the TLB. */
871#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
872/** Valid bit mask. */
873#define IEM_ACCESS_VALID_MASK UINT32_C(0x00001fff)
874/** Shift count for the TLB flags (upper word). */
875#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
876
877/** Read+write data alias. */
878#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
879/** Write data alias. */
880#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
881/** Read data alias. */
882#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA)
883/** Instruction fetch alias. */
884#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_WHAT_CODE)
885/** Stack write alias. */
886#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
887/** Stack read alias. */
888#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_STACK)
889/** Stack read+write alias. */
890#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
891/** Read system table alias. */
892#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_SYS)
893/** Read+write system table alias. */
894#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_SYS)
895/** @} */
896
897/** @name Prefix constants (IEMCPU::fPrefixes)
898 * @{ */
899#define IEM_OP_PRF_SEG_CS RT_BIT_32(0) /**< CS segment prefix (0x2e). */
900#define IEM_OP_PRF_SEG_SS RT_BIT_32(1) /**< SS segment prefix (0x36). */
901#define IEM_OP_PRF_SEG_DS RT_BIT_32(2) /**< DS segment prefix (0x3e). */
902#define IEM_OP_PRF_SEG_ES RT_BIT_32(3) /**< ES segment prefix (0x26). */
903#define IEM_OP_PRF_SEG_FS RT_BIT_32(4) /**< FS segment prefix (0x64). */
904#define IEM_OP_PRF_SEG_GS RT_BIT_32(5) /**< GS segment prefix (0x65). */
905#define IEM_OP_PRF_SEG_MASK UINT32_C(0x3f)
906
907#define IEM_OP_PRF_SIZE_OP RT_BIT_32(8) /**< Operand size prefix (0x66). */
908#define IEM_OP_PRF_SIZE_REX_W RT_BIT_32(9) /**< REX.W prefix (0x48-0x4f). */
909#define IEM_OP_PRF_SIZE_ADDR RT_BIT_32(10) /**< Address size prefix (0x67). */
910
911#define IEM_OP_PRF_LOCK RT_BIT_32(16) /**< Lock prefix (0xf0). */
912#define IEM_OP_PRF_REPNZ RT_BIT_32(17) /**< Repeat-not-zero prefix (0xf2). */
913#define IEM_OP_PRF_REPZ RT_BIT_32(18) /**< Repeat-if-zero prefix (0xf3). */
914
915#define IEM_OP_PRF_REX RT_BIT_32(24) /**< Any REX prefix (0x40-0x4f). */
916#define IEM_OP_PRF_REX_R RT_BIT_32(25) /**< REX.R prefix (0x44,0x45,0x46,0x47,0x4c,0x4d,0x4e,0x4f). */
917#define IEM_OP_PRF_REX_B RT_BIT_32(26) /**< REX.B prefix (0x41,0x43,0x45,0x47,0x49,0x4b,0x4d,0x4f). */
918#define IEM_OP_PRF_REX_X RT_BIT_32(27) /**< REX.X prefix (0x42,0x43,0x46,0x47,0x4a,0x4b,0x4e,0x4f). */
919/** Mask with all the REX prefix flags.
920 * This is generally for use when needing to undo the REX prefixes when they
921 * are followed legacy prefixes and therefore does not immediately preceed
922 * the first opcode byte.
923 * For testing whether any REX prefix is present, use IEM_OP_PRF_REX instead. */
924#define IEM_OP_PRF_REX_MASK (IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W )
925
926#define IEM_OP_PRF_VEX RT_BIT_32(28) /**< Indiciates VEX prefix. */
927#define IEM_OP_PRF_EVEX RT_BIT_32(29) /**< Indiciates EVEX prefix. */
928#define IEM_OP_PRF_XOP RT_BIT_32(30) /**< Indiciates XOP prefix. */
929/** @} */
930
931/** @name IEMOPFORM_XXX - Opcode forms
932 * @note These are ORed together with IEMOPHINT_XXX.
933 * @{ */
934/** ModR/M: reg, r/m */
935#define IEMOPFORM_RM 0
936/** ModR/M: reg, r/m (register) */
937#define IEMOPFORM_RM_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
938/** ModR/M: reg, r/m (memory) */
939#define IEMOPFORM_RM_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
940/** ModR/M: reg, r/m */
941#define IEMOPFORM_RMI 1
942/** ModR/M: reg, r/m (register) */
943#define IEMOPFORM_RMI_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
944/** ModR/M: reg, r/m (memory) */
945#define IEMOPFORM_RMI_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
946/** ModR/M: r/m, reg */
947#define IEMOPFORM_MR 2
948/** ModR/M: r/m (register), reg */
949#define IEMOPFORM_MR_REG (IEMOPFORM_MR | IEMOPFORM_MOD3)
950/** ModR/M: r/m (memory), reg */
951#define IEMOPFORM_MR_MEM (IEMOPFORM_MR | IEMOPFORM_NOT_MOD3)
952/** ModR/M: r/m only */
953#define IEMOPFORM_M 3
954/** ModR/M: r/m only (register). */
955#define IEMOPFORM_M_REG (IEMOPFORM_M | IEMOPFORM_MOD3)
956/** ModR/M: r/m only (memory). */
957#define IEMOPFORM_M_MEM (IEMOPFORM_M | IEMOPFORM_NOT_MOD3)
958/** ModR/M: reg only */
959#define IEMOPFORM_R 4
960
961/** VEX+ModR/M: reg, r/m */
962#define IEMOPFORM_VEX_RM 8
963/** VEX+ModR/M: reg, r/m (register) */
964#define IEMOPFORM_VEX_RM_REG (IEMOPFORM_VEX_RM | IEMOPFORM_MOD3)
965/** VEX+ModR/M: reg, r/m (memory) */
966#define IEMOPFORM_VEX_RM_MEM (IEMOPFORM_VEX_RM | IEMOPFORM_NOT_MOD3)
967/** VEX+ModR/M: r/m, reg */
968#define IEMOPFORM_VEX_MR 9
969/** VEX+ModR/M: r/m (register), reg */
970#define IEMOPFORM_VEX_MR_REG (IEMOPFORM_VEX_MR | IEMOPFORM_MOD3)
971/** VEX+ModR/M: r/m (memory), reg */
972#define IEMOPFORM_VEX_MR_MEM (IEMOPFORM_VEX_MR | IEMOPFORM_NOT_MOD3)
973/** VEX+ModR/M: r/m only */
974#define IEMOPFORM_VEX_M 10
975/** VEX+ModR/M: r/m only (register). */
976#define IEMOPFORM_VEX_M_REG (IEMOPFORM_VEX_M | IEMOPFORM_MOD3)
977/** VEX+ModR/M: r/m only (memory). */
978#define IEMOPFORM_VEX_M_MEM (IEMOPFORM_VEX_M | IEMOPFORM_NOT_MOD3)
979/** VEX+ModR/M: reg only */
980#define IEMOPFORM_VEX_R 11
981/** VEX+ModR/M: reg, vvvv, r/m */
982#define IEMOPFORM_VEX_RVM 12
983/** VEX+ModR/M: reg, vvvv, r/m (register). */
984#define IEMOPFORM_VEX_RVM_REG (IEMOPFORM_VEX_RVM | IEMOPFORM_MOD3)
985/** VEX+ModR/M: reg, vvvv, r/m (memory). */
986#define IEMOPFORM_VEX_RVM_MEM (IEMOPFORM_VEX_RVM | IEMOPFORM_NOT_MOD3)
987/** VEX+ModR/M: reg, r/m, vvvv */
988#define IEMOPFORM_VEX_RMV 13
989/** VEX+ModR/M: reg, r/m, vvvv (register). */
990#define IEMOPFORM_VEX_RMV_REG (IEMOPFORM_VEX_RMV | IEMOPFORM_MOD3)
991/** VEX+ModR/M: reg, r/m, vvvv (memory). */
992#define IEMOPFORM_VEX_RMV_MEM (IEMOPFORM_VEX_RMV | IEMOPFORM_NOT_MOD3)
993/** VEX+ModR/M: reg, r/m, imm8 */
994#define IEMOPFORM_VEX_RMI 14
995/** VEX+ModR/M: reg, r/m, imm8 (register). */
996#define IEMOPFORM_VEX_RMI_REG (IEMOPFORM_VEX_RMI | IEMOPFORM_MOD3)
997/** VEX+ModR/M: reg, r/m, imm8 (memory). */
998#define IEMOPFORM_VEX_RMI_MEM (IEMOPFORM_VEX_RMI | IEMOPFORM_NOT_MOD3)
999/** VEX+ModR/M: r/m, vvvv, reg */
1000#define IEMOPFORM_VEX_MVR 15
1001/** VEX+ModR/M: r/m, vvvv, reg (register) */
1002#define IEMOPFORM_VEX_MVR_REG (IEMOPFORM_VEX_MVR | IEMOPFORM_MOD3)
1003/** VEX+ModR/M: r/m, vvvv, reg (memory) */
1004#define IEMOPFORM_VEX_MVR_MEM (IEMOPFORM_VEX_MVR | IEMOPFORM_NOT_MOD3)
1005/** VEX+ModR/M+/n: vvvv, r/m */
1006#define IEMOPFORM_VEX_VM 16
1007/** VEX+ModR/M+/n: vvvv, r/m (register) */
1008#define IEMOPFORM_VEX_VM_REG (IEMOPFORM_VEX_VM | IEMOPFORM_MOD3)
1009/** VEX+ModR/M+/n: vvvv, r/m (memory) */
1010#define IEMOPFORM_VEX_VM_MEM (IEMOPFORM_VEX_VM | IEMOPFORM_NOT_MOD3)
1011
1012/** Fixed register instruction, no R/M. */
1013#define IEMOPFORM_FIXED 32
1014
1015/** The r/m is a register. */
1016#define IEMOPFORM_MOD3 RT_BIT_32(8)
1017/** The r/m is a memory access. */
1018#define IEMOPFORM_NOT_MOD3 RT_BIT_32(9)
1019/** @} */
1020
1021/** @name IEMOPHINT_XXX - Additional Opcode Hints
1022 * @note These are ORed together with IEMOPFORM_XXX.
1023 * @{ */
1024/** Ignores the operand size prefix (66h). */
1025#define IEMOPHINT_IGNORES_OZ_PFX RT_BIT_32(10)
1026/** Ignores REX.W (aka WIG). */
1027#define IEMOPHINT_IGNORES_REXW RT_BIT_32(11)
1028/** Both the operand size prefixes (66h + REX.W) are ignored. */
1029#define IEMOPHINT_IGNORES_OP_SIZES (IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW)
1030/** Allowed with the lock prefix. */
1031#define IEMOPHINT_LOCK_ALLOWED RT_BIT_32(11)
1032/** The VEX.L value is ignored (aka LIG). */
1033#define IEMOPHINT_VEX_L_IGNORED RT_BIT_32(12)
1034/** The VEX.L value must be zero (i.e. 128-bit width only). */
1035#define IEMOPHINT_VEX_L_ZERO RT_BIT_32(13)
1036/** The VEX.V value must be zero. */
1037#define IEMOPHINT_VEX_V_ZERO RT_BIT_32(14)
1038
1039/** Hint to IEMAllInstructionPython.py that this macro should be skipped. */
1040#define IEMOPHINT_SKIP_PYTHON RT_BIT_32(31)
1041/** @} */
1042
1043/**
1044 * Possible hardware task switch sources.
1045 */
1046typedef enum IEMTASKSWITCH
1047{
1048 /** Task switch caused by an interrupt/exception. */
1049 IEMTASKSWITCH_INT_XCPT = 1,
1050 /** Task switch caused by a far CALL. */
1051 IEMTASKSWITCH_CALL,
1052 /** Task switch caused by a far JMP. */
1053 IEMTASKSWITCH_JUMP,
1054 /** Task switch caused by an IRET. */
1055 IEMTASKSWITCH_IRET
1056} IEMTASKSWITCH;
1057AssertCompileSize(IEMTASKSWITCH, 4);
1058
1059/**
1060 * Possible CrX load (write) sources.
1061 */
1062typedef enum IEMACCESSCRX
1063{
1064 /** CrX access caused by 'mov crX' instruction. */
1065 IEMACCESSCRX_MOV_CRX,
1066 /** CrX (CR0) write caused by 'lmsw' instruction. */
1067 IEMACCESSCRX_LMSW,
1068 /** CrX (CR0) write caused by 'clts' instruction. */
1069 IEMACCESSCRX_CLTS,
1070 /** CrX (CR0) read caused by 'smsw' instruction. */
1071 IEMACCESSCRX_SMSW
1072} IEMACCESSCRX;
1073
1074#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1075/** @name IEM_SLAT_FAIL_XXX - Second-level address translation failure information.
1076 *
1077 * These flags provide further context to SLAT page-walk failures that could not be
1078 * determined by PGM (e.g, PGM is not privy to memory access permissions).
1079 *
1080 * @{
1081 */
1082/** Translating a nested-guest linear address failed accessing a nested-guest
1083 * physical address. */
1084# define IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR RT_BIT_32(0)
1085/** Translating a nested-guest linear address failed accessing a
1086 * paging-structure entry or updating accessed/dirty bits. */
1087# define IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE RT_BIT_32(1)
1088/** @} */
1089
1090DECLCALLBACK(FNPGMPHYSHANDLER) iemVmxApicAccessPageHandler;
1091# ifndef IN_RING3
1092DECLCALLBACK(FNPGMRZPHYSPFHANDLER) iemVmxApicAccessPagePfHandler;
1093# endif
1094#endif
1095
1096/**
1097 * Indicates to the verifier that the given flag set is undefined.
1098 *
1099 * Can be invoked again to add more flags.
1100 *
1101 * This is a NOOP if the verifier isn't compiled in.
1102 *
1103 * @note We're temporarily keeping this until code is converted to new
1104 * disassembler style opcode handling.
1105 */
1106#define IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fEfl) do { } while (0)
1107
1108
1109/** @def IEM_DECL_IMPL_TYPE
1110 * For typedef'ing an instruction implementation function.
1111 *
1112 * @param a_RetType The return type.
1113 * @param a_Name The name of the type.
1114 * @param a_ArgList The argument list enclosed in parentheses.
1115 */
1116
1117/** @def IEM_DECL_IMPL_DEF
1118 * For defining an instruction implementation function.
1119 *
1120 * @param a_RetType The return type.
1121 * @param a_Name The name of the type.
1122 * @param a_ArgList The argument list enclosed in parentheses.
1123 */
1124
1125#if defined(__GNUC__) && defined(RT_ARCH_X86)
1126# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1127 __attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
1128# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1129 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1130# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1131 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1132
1133#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
1134# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1135 a_RetType (__fastcall a_Name) a_ArgList
1136# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1137 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1138# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1139 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1140
1141#elif __cplusplus >= 201700 /* P0012R1 support */
1142# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1143 a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
1144# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1145 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1146# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1147 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1148
1149#else
1150# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1151 a_RetType (VBOXCALL a_Name) a_ArgList
1152# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1153 a_RetType VBOXCALL a_Name a_ArgList
1154# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1155 a_RetType VBOXCALL a_Name a_ArgList
1156
1157#endif
1158
1159/** Defined in IEMAllAImplC.cpp but also used by IEMAllAImplA.asm. */
1160RT_C_DECLS_BEGIN
1161extern uint8_t const g_afParity[256];
1162RT_C_DECLS_END
1163
1164
1165/** @name Arithmetic assignment operations on bytes (binary).
1166 * @{ */
1167typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU8, (uint8_t *pu8Dst, uint8_t u8Src, uint32_t *pEFlags));
1168typedef FNIEMAIMPLBINU8 *PFNIEMAIMPLBINU8;
1169FNIEMAIMPLBINU8 iemAImpl_add_u8, iemAImpl_add_u8_locked;
1170FNIEMAIMPLBINU8 iemAImpl_adc_u8, iemAImpl_adc_u8_locked;
1171FNIEMAIMPLBINU8 iemAImpl_sub_u8, iemAImpl_sub_u8_locked;
1172FNIEMAIMPLBINU8 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked;
1173FNIEMAIMPLBINU8 iemAImpl_or_u8, iemAImpl_or_u8_locked;
1174FNIEMAIMPLBINU8 iemAImpl_xor_u8, iemAImpl_xor_u8_locked;
1175FNIEMAIMPLBINU8 iemAImpl_and_u8, iemAImpl_and_u8_locked;
1176/** @} */
1177
1178/** @name Arithmetic assignment operations on words (binary).
1179 * @{ */
1180typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU16, (uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pEFlags));
1181typedef FNIEMAIMPLBINU16 *PFNIEMAIMPLBINU16;
1182FNIEMAIMPLBINU16 iemAImpl_add_u16, iemAImpl_add_u16_locked;
1183FNIEMAIMPLBINU16 iemAImpl_adc_u16, iemAImpl_adc_u16_locked;
1184FNIEMAIMPLBINU16 iemAImpl_sub_u16, iemAImpl_sub_u16_locked;
1185FNIEMAIMPLBINU16 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked;
1186FNIEMAIMPLBINU16 iemAImpl_or_u16, iemAImpl_or_u16_locked;
1187FNIEMAIMPLBINU16 iemAImpl_xor_u16, iemAImpl_xor_u16_locked;
1188FNIEMAIMPLBINU16 iemAImpl_and_u16, iemAImpl_and_u16_locked;
1189/** @} */
1190
1191/** @name Arithmetic assignment operations on double words (binary).
1192 * @{ */
1193typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU32, (uint32_t *pu32Dst, uint32_t u32Src, uint32_t *pEFlags));
1194typedef FNIEMAIMPLBINU32 *PFNIEMAIMPLBINU32;
1195FNIEMAIMPLBINU32 iemAImpl_add_u32, iemAImpl_add_u32_locked;
1196FNIEMAIMPLBINU32 iemAImpl_adc_u32, iemAImpl_adc_u32_locked;
1197FNIEMAIMPLBINU32 iemAImpl_sub_u32, iemAImpl_sub_u32_locked;
1198FNIEMAIMPLBINU32 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked;
1199FNIEMAIMPLBINU32 iemAImpl_or_u32, iemAImpl_or_u32_locked;
1200FNIEMAIMPLBINU32 iemAImpl_xor_u32, iemAImpl_xor_u32_locked;
1201FNIEMAIMPLBINU32 iemAImpl_and_u32, iemAImpl_and_u32_locked;
1202FNIEMAIMPLBINU32 iemAImpl_blsi_u32, iemAImpl_blsi_u32_fallback;
1203FNIEMAIMPLBINU32 iemAImpl_blsr_u32, iemAImpl_blsr_u32_fallback;
1204FNIEMAIMPLBINU32 iemAImpl_blsmsk_u32, iemAImpl_blsmsk_u32_fallback;
1205/** @} */
1206
1207/** @name Arithmetic assignment operations on quad words (binary).
1208 * @{ */
1209typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU64, (uint64_t *pu64Dst, uint64_t u64Src, uint32_t *pEFlags));
1210typedef FNIEMAIMPLBINU64 *PFNIEMAIMPLBINU64;
1211FNIEMAIMPLBINU64 iemAImpl_add_u64, iemAImpl_add_u64_locked;
1212FNIEMAIMPLBINU64 iemAImpl_adc_u64, iemAImpl_adc_u64_locked;
1213FNIEMAIMPLBINU64 iemAImpl_sub_u64, iemAImpl_sub_u64_locked;
1214FNIEMAIMPLBINU64 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked;
1215FNIEMAIMPLBINU64 iemAImpl_or_u64, iemAImpl_or_u64_locked;
1216FNIEMAIMPLBINU64 iemAImpl_xor_u64, iemAImpl_xor_u64_locked;
1217FNIEMAIMPLBINU64 iemAImpl_and_u64, iemAImpl_and_u64_locked;
1218FNIEMAIMPLBINU64 iemAImpl_blsi_u64, iemAImpl_blsi_u64_fallback;
1219FNIEMAIMPLBINU64 iemAImpl_blsr_u64, iemAImpl_blsr_u64_fallback;
1220FNIEMAIMPLBINU64 iemAImpl_blsmsk_u64, iemAImpl_blsmsk_u64_fallback;
1221/** @} */
1222
1223/** @name Compare operations (thrown in with the binary ops).
1224 * @{ */
1225FNIEMAIMPLBINU8 iemAImpl_cmp_u8;
1226FNIEMAIMPLBINU16 iemAImpl_cmp_u16;
1227FNIEMAIMPLBINU32 iemAImpl_cmp_u32;
1228FNIEMAIMPLBINU64 iemAImpl_cmp_u64;
1229/** @} */
1230
1231/** @name Test operations (thrown in with the binary ops).
1232 * @{ */
1233FNIEMAIMPLBINU8 iemAImpl_test_u8;
1234FNIEMAIMPLBINU16 iemAImpl_test_u16;
1235FNIEMAIMPLBINU32 iemAImpl_test_u32;
1236FNIEMAIMPLBINU64 iemAImpl_test_u64;
1237/** @} */
1238
1239/** @name Bit operations operations (thrown in with the binary ops).
1240 * @{ */
1241FNIEMAIMPLBINU16 iemAImpl_bt_u16;
1242FNIEMAIMPLBINU32 iemAImpl_bt_u32;
1243FNIEMAIMPLBINU64 iemAImpl_bt_u64;
1244FNIEMAIMPLBINU16 iemAImpl_btc_u16, iemAImpl_btc_u16_locked;
1245FNIEMAIMPLBINU32 iemAImpl_btc_u32, iemAImpl_btc_u32_locked;
1246FNIEMAIMPLBINU64 iemAImpl_btc_u64, iemAImpl_btc_u64_locked;
1247FNIEMAIMPLBINU16 iemAImpl_btr_u16, iemAImpl_btr_u16_locked;
1248FNIEMAIMPLBINU32 iemAImpl_btr_u32, iemAImpl_btr_u32_locked;
1249FNIEMAIMPLBINU64 iemAImpl_btr_u64, iemAImpl_btr_u64_locked;
1250FNIEMAIMPLBINU16 iemAImpl_bts_u16, iemAImpl_bts_u16_locked;
1251FNIEMAIMPLBINU32 iemAImpl_bts_u32, iemAImpl_bts_u32_locked;
1252FNIEMAIMPLBINU64 iemAImpl_bts_u64, iemAImpl_bts_u64_locked;
1253/** @} */
1254
1255/** @name Arithmetic three operand operations on double words (binary).
1256 * @{ */
1257typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2, uint32_t *pEFlags));
1258typedef FNIEMAIMPLBINVEXU32 *PFNIEMAIMPLBINVEXU32;
1259FNIEMAIMPLBINVEXU32 iemAImpl_andn_u32, iemAImpl_andn_u32_fallback;
1260FNIEMAIMPLBINVEXU32 iemAImpl_bextr_u32, iemAImpl_bextr_u32_fallback;
1261FNIEMAIMPLBINVEXU32 iemAImpl_bzhi_u32, iemAImpl_bzhi_u32_fallback;
1262/** @} */
1263
1264/** @name Arithmetic three operand operations on quad words (binary).
1265 * @{ */
1266typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2, uint32_t *pEFlags));
1267typedef FNIEMAIMPLBINVEXU64 *PFNIEMAIMPLBINVEXU64;
1268FNIEMAIMPLBINVEXU64 iemAImpl_andn_u64, iemAImpl_andn_u64_fallback;
1269FNIEMAIMPLBINVEXU64 iemAImpl_bextr_u64, iemAImpl_bextr_u64_fallback;
1270FNIEMAIMPLBINVEXU64 iemAImpl_bzhi_u64, iemAImpl_bzhi_u64_fallback;
1271/** @} */
1272
1273/** @name Arithmetic three operand operations on double words w/o EFLAGS (binary).
1274 * @{ */
1275typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32NOEFL, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2));
1276typedef FNIEMAIMPLBINVEXU32NOEFL *PFNIEMAIMPLBINVEXU32NOEFL;
1277FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pdep_u32, iemAImpl_pdep_u32_fallback;
1278FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pext_u32, iemAImpl_pext_u32_fallback;
1279FNIEMAIMPLBINVEXU32NOEFL iemAImpl_sarx_u32, iemAImpl_sarx_u32_fallback;
1280FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shlx_u32, iemAImpl_shlx_u32_fallback;
1281FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shrx_u32, iemAImpl_shrx_u32_fallback;
1282FNIEMAIMPLBINVEXU32NOEFL iemAImpl_rorx_u32;
1283/** @} */
1284
1285/** @name Arithmetic three operand operations on quad words w/o EFLAGS (binary).
1286 * @{ */
1287typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64NOEFL, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2));
1288typedef FNIEMAIMPLBINVEXU64NOEFL *PFNIEMAIMPLBINVEXU64NOEFL;
1289FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pdep_u64, iemAImpl_pdep_u64_fallback;
1290FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pext_u64, iemAImpl_pext_u64_fallback;
1291FNIEMAIMPLBINVEXU64NOEFL iemAImpl_sarx_u64, iemAImpl_sarx_u64_fallback;
1292FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shlx_u64, iemAImpl_shlx_u64_fallback;
1293FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shrx_u64, iemAImpl_shrx_u64_fallback;
1294FNIEMAIMPLBINVEXU64NOEFL iemAImpl_rorx_u64;
1295/** @} */
1296
1297/** @name MULX 32-bit and 64-bit.
1298 * @{ */
1299typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU32, (uint32_t *puDst1, uint32_t *puDst2, uint32_t uSrc1, uint32_t uSrc2));
1300typedef FNIEMAIMPLMULXVEXU32 *PFNIEMAIMPLMULXVEXU32;
1301FNIEMAIMPLMULXVEXU32 iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback;
1302
1303typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU64, (uint64_t *puDst1, uint64_t *puDst2, uint64_t uSrc1, uint64_t uSrc2));
1304typedef FNIEMAIMPLMULXVEXU64 *PFNIEMAIMPLMULXVEXU64;
1305FNIEMAIMPLMULXVEXU64 iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback;
1306/** @} */
1307
1308
1309/** @name Exchange memory with register operations.
1310 * @{ */
1311IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1312IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1313IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1314IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1315IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1316IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1317IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1318IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1319/** @} */
1320
1321/** @name Exchange and add operations.
1322 * @{ */
1323IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1324IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1325IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1326IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1327IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8_locked, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1328IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16_locked,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1329IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32_locked,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1330IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1331/** @} */
1332
1333/** @name Compare and exchange.
1334 * @{ */
1335IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1336IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1337IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1338IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1339IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1340IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1341#if ARCH_BITS == 32
1342IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1343IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1344#else
1345IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1346IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1347#endif
1348IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1349 uint32_t *pEFlags));
1350IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1351 uint32_t *pEFlags));
1352IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1353 uint32_t *pEFlags));
1354IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1355 uint32_t *pEFlags));
1356#ifndef RT_ARCH_ARM64
1357IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1358 PRTUINT128U pu128RbxRcx, uint32_t *pEFlags));
1359#endif
1360/** @} */
1361
1362/** @name Memory ordering
1363 * @{ */
1364typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEMFENCE,(void));
1365typedef FNIEMAIMPLMEMFENCE *PFNIEMAIMPLMEMFENCE;
1366IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void));
1367IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void));
1368IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void));
1369#ifndef RT_ARCH_ARM64
1370IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void));
1371#endif
1372/** @} */
1373
1374/** @name Double precision shifts
1375 * @{ */
1376typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU16,(uint16_t *pu16Dst, uint16_t u16Src, uint8_t cShift, uint32_t *pEFlags));
1377typedef FNIEMAIMPLSHIFTDBLU16 *PFNIEMAIMPLSHIFTDBLU16;
1378typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU32,(uint32_t *pu32Dst, uint32_t u32Src, uint8_t cShift, uint32_t *pEFlags));
1379typedef FNIEMAIMPLSHIFTDBLU32 *PFNIEMAIMPLSHIFTDBLU32;
1380typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU64,(uint64_t *pu64Dst, uint64_t u64Src, uint8_t cShift, uint32_t *pEFlags));
1381typedef FNIEMAIMPLSHIFTDBLU64 *PFNIEMAIMPLSHIFTDBLU64;
1382FNIEMAIMPLSHIFTDBLU16 iemAImpl_shld_u16, iemAImpl_shld_u16_amd, iemAImpl_shld_u16_intel;
1383FNIEMAIMPLSHIFTDBLU32 iemAImpl_shld_u32, iemAImpl_shld_u32_amd, iemAImpl_shld_u32_intel;
1384FNIEMAIMPLSHIFTDBLU64 iemAImpl_shld_u64, iemAImpl_shld_u64_amd, iemAImpl_shld_u64_intel;
1385FNIEMAIMPLSHIFTDBLU16 iemAImpl_shrd_u16, iemAImpl_shrd_u16_amd, iemAImpl_shrd_u16_intel;
1386FNIEMAIMPLSHIFTDBLU32 iemAImpl_shrd_u32, iemAImpl_shrd_u32_amd, iemAImpl_shrd_u32_intel;
1387FNIEMAIMPLSHIFTDBLU64 iemAImpl_shrd_u64, iemAImpl_shrd_u64_amd, iemAImpl_shrd_u64_intel;
1388/** @} */
1389
1390
1391/** @name Bit search operations (thrown in with the binary ops).
1392 * @{ */
1393FNIEMAIMPLBINU16 iemAImpl_bsf_u16, iemAImpl_bsf_u16_amd, iemAImpl_bsf_u16_intel;
1394FNIEMAIMPLBINU32 iemAImpl_bsf_u32, iemAImpl_bsf_u32_amd, iemAImpl_bsf_u32_intel;
1395FNIEMAIMPLBINU64 iemAImpl_bsf_u64, iemAImpl_bsf_u64_amd, iemAImpl_bsf_u64_intel;
1396FNIEMAIMPLBINU16 iemAImpl_bsr_u16, iemAImpl_bsr_u16_amd, iemAImpl_bsr_u16_intel;
1397FNIEMAIMPLBINU32 iemAImpl_bsr_u32, iemAImpl_bsr_u32_amd, iemAImpl_bsr_u32_intel;
1398FNIEMAIMPLBINU64 iemAImpl_bsr_u64, iemAImpl_bsr_u64_amd, iemAImpl_bsr_u64_intel;
1399FNIEMAIMPLBINU16 iemAImpl_lzcnt_u16, iemAImpl_lzcnt_u16_amd, iemAImpl_lzcnt_u16_intel;
1400FNIEMAIMPLBINU32 iemAImpl_lzcnt_u32, iemAImpl_lzcnt_u32_amd, iemAImpl_lzcnt_u32_intel;
1401FNIEMAIMPLBINU64 iemAImpl_lzcnt_u64, iemAImpl_lzcnt_u64_amd, iemAImpl_lzcnt_u64_intel;
1402FNIEMAIMPLBINU16 iemAImpl_tzcnt_u16, iemAImpl_tzcnt_u16_amd, iemAImpl_tzcnt_u16_intel;
1403FNIEMAIMPLBINU32 iemAImpl_tzcnt_u32, iemAImpl_tzcnt_u32_amd, iemAImpl_tzcnt_u32_intel;
1404FNIEMAIMPLBINU64 iemAImpl_tzcnt_u64, iemAImpl_tzcnt_u64_amd, iemAImpl_tzcnt_u64_intel;
1405FNIEMAIMPLBINU16 iemAImpl_popcnt_u16, iemAImpl_popcnt_u16_fallback;
1406FNIEMAIMPLBINU32 iemAImpl_popcnt_u32, iemAImpl_popcnt_u32_fallback;
1407FNIEMAIMPLBINU64 iemAImpl_popcnt_u64, iemAImpl_popcnt_u64_fallback;
1408/** @} */
1409
1410/** @name Signed multiplication operations (thrown in with the binary ops).
1411 * @{ */
1412FNIEMAIMPLBINU16 iemAImpl_imul_two_u16, iemAImpl_imul_two_u16_amd, iemAImpl_imul_two_u16_intel;
1413FNIEMAIMPLBINU32 iemAImpl_imul_two_u32, iemAImpl_imul_two_u32_amd, iemAImpl_imul_two_u32_intel;
1414FNIEMAIMPLBINU64 iemAImpl_imul_two_u64, iemAImpl_imul_two_u64_amd, iemAImpl_imul_two_u64_intel;
1415/** @} */
1416
1417/** @name Arithmetic assignment operations on bytes (unary).
1418 * @{ */
1419typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU8, (uint8_t *pu8Dst, uint32_t *pEFlags));
1420typedef FNIEMAIMPLUNARYU8 *PFNIEMAIMPLUNARYU8;
1421FNIEMAIMPLUNARYU8 iemAImpl_inc_u8, iemAImpl_inc_u8_locked;
1422FNIEMAIMPLUNARYU8 iemAImpl_dec_u8, iemAImpl_dec_u8_locked;
1423FNIEMAIMPLUNARYU8 iemAImpl_not_u8, iemAImpl_not_u8_locked;
1424FNIEMAIMPLUNARYU8 iemAImpl_neg_u8, iemAImpl_neg_u8_locked;
1425/** @} */
1426
1427/** @name Arithmetic assignment operations on words (unary).
1428 * @{ */
1429typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU16, (uint16_t *pu16Dst, uint32_t *pEFlags));
1430typedef FNIEMAIMPLUNARYU16 *PFNIEMAIMPLUNARYU16;
1431FNIEMAIMPLUNARYU16 iemAImpl_inc_u16, iemAImpl_inc_u16_locked;
1432FNIEMAIMPLUNARYU16 iemAImpl_dec_u16, iemAImpl_dec_u16_locked;
1433FNIEMAIMPLUNARYU16 iemAImpl_not_u16, iemAImpl_not_u16_locked;
1434FNIEMAIMPLUNARYU16 iemAImpl_neg_u16, iemAImpl_neg_u16_locked;
1435/** @} */
1436
1437/** @name Arithmetic assignment operations on double words (unary).
1438 * @{ */
1439typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU32, (uint32_t *pu32Dst, uint32_t *pEFlags));
1440typedef FNIEMAIMPLUNARYU32 *PFNIEMAIMPLUNARYU32;
1441FNIEMAIMPLUNARYU32 iemAImpl_inc_u32, iemAImpl_inc_u32_locked;
1442FNIEMAIMPLUNARYU32 iemAImpl_dec_u32, iemAImpl_dec_u32_locked;
1443FNIEMAIMPLUNARYU32 iemAImpl_not_u32, iemAImpl_not_u32_locked;
1444FNIEMAIMPLUNARYU32 iemAImpl_neg_u32, iemAImpl_neg_u32_locked;
1445/** @} */
1446
1447/** @name Arithmetic assignment operations on quad words (unary).
1448 * @{ */
1449typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU64, (uint64_t *pu64Dst, uint32_t *pEFlags));
1450typedef FNIEMAIMPLUNARYU64 *PFNIEMAIMPLUNARYU64;
1451FNIEMAIMPLUNARYU64 iemAImpl_inc_u64, iemAImpl_inc_u64_locked;
1452FNIEMAIMPLUNARYU64 iemAImpl_dec_u64, iemAImpl_dec_u64_locked;
1453FNIEMAIMPLUNARYU64 iemAImpl_not_u64, iemAImpl_not_u64_locked;
1454FNIEMAIMPLUNARYU64 iemAImpl_neg_u64, iemAImpl_neg_u64_locked;
1455/** @} */
1456
1457
1458/** @name Shift operations on bytes (Group 2).
1459 * @{ */
1460typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU8,(uint8_t *pu8Dst, uint8_t cShift, uint32_t *pEFlags));
1461typedef FNIEMAIMPLSHIFTU8 *PFNIEMAIMPLSHIFTU8;
1462FNIEMAIMPLSHIFTU8 iemAImpl_rol_u8, iemAImpl_rol_u8_amd, iemAImpl_rol_u8_intel;
1463FNIEMAIMPLSHIFTU8 iemAImpl_ror_u8, iemAImpl_ror_u8_amd, iemAImpl_ror_u8_intel;
1464FNIEMAIMPLSHIFTU8 iemAImpl_rcl_u8, iemAImpl_rcl_u8_amd, iemAImpl_rcl_u8_intel;
1465FNIEMAIMPLSHIFTU8 iemAImpl_rcr_u8, iemAImpl_rcr_u8_amd, iemAImpl_rcr_u8_intel;
1466FNIEMAIMPLSHIFTU8 iemAImpl_shl_u8, iemAImpl_shl_u8_amd, iemAImpl_shl_u8_intel;
1467FNIEMAIMPLSHIFTU8 iemAImpl_shr_u8, iemAImpl_shr_u8_amd, iemAImpl_shr_u8_intel;
1468FNIEMAIMPLSHIFTU8 iemAImpl_sar_u8, iemAImpl_sar_u8_amd, iemAImpl_sar_u8_intel;
1469/** @} */
1470
1471/** @name Shift operations on words (Group 2).
1472 * @{ */
1473typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU16,(uint16_t *pu16Dst, uint8_t cShift, uint32_t *pEFlags));
1474typedef FNIEMAIMPLSHIFTU16 *PFNIEMAIMPLSHIFTU16;
1475FNIEMAIMPLSHIFTU16 iemAImpl_rol_u16, iemAImpl_rol_u16_amd, iemAImpl_rol_u16_intel;
1476FNIEMAIMPLSHIFTU16 iemAImpl_ror_u16, iemAImpl_ror_u16_amd, iemAImpl_ror_u16_intel;
1477FNIEMAIMPLSHIFTU16 iemAImpl_rcl_u16, iemAImpl_rcl_u16_amd, iemAImpl_rcl_u16_intel;
1478FNIEMAIMPLSHIFTU16 iemAImpl_rcr_u16, iemAImpl_rcr_u16_amd, iemAImpl_rcr_u16_intel;
1479FNIEMAIMPLSHIFTU16 iemAImpl_shl_u16, iemAImpl_shl_u16_amd, iemAImpl_shl_u16_intel;
1480FNIEMAIMPLSHIFTU16 iemAImpl_shr_u16, iemAImpl_shr_u16_amd, iemAImpl_shr_u16_intel;
1481FNIEMAIMPLSHIFTU16 iemAImpl_sar_u16, iemAImpl_sar_u16_amd, iemAImpl_sar_u16_intel;
1482/** @} */
1483
1484/** @name Shift operations on double words (Group 2).
1485 * @{ */
1486typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU32,(uint32_t *pu32Dst, uint8_t cShift, uint32_t *pEFlags));
1487typedef FNIEMAIMPLSHIFTU32 *PFNIEMAIMPLSHIFTU32;
1488FNIEMAIMPLSHIFTU32 iemAImpl_rol_u32, iemAImpl_rol_u32_amd, iemAImpl_rol_u32_intel;
1489FNIEMAIMPLSHIFTU32 iemAImpl_ror_u32, iemAImpl_ror_u32_amd, iemAImpl_ror_u32_intel;
1490FNIEMAIMPLSHIFTU32 iemAImpl_rcl_u32, iemAImpl_rcl_u32_amd, iemAImpl_rcl_u32_intel;
1491FNIEMAIMPLSHIFTU32 iemAImpl_rcr_u32, iemAImpl_rcr_u32_amd, iemAImpl_rcr_u32_intel;
1492FNIEMAIMPLSHIFTU32 iemAImpl_shl_u32, iemAImpl_shl_u32_amd, iemAImpl_shl_u32_intel;
1493FNIEMAIMPLSHIFTU32 iemAImpl_shr_u32, iemAImpl_shr_u32_amd, iemAImpl_shr_u32_intel;
1494FNIEMAIMPLSHIFTU32 iemAImpl_sar_u32, iemAImpl_sar_u32_amd, iemAImpl_sar_u32_intel;
1495/** @} */
1496
1497/** @name Shift operations on words (Group 2).
1498 * @{ */
1499typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU64,(uint64_t *pu64Dst, uint8_t cShift, uint32_t *pEFlags));
1500typedef FNIEMAIMPLSHIFTU64 *PFNIEMAIMPLSHIFTU64;
1501FNIEMAIMPLSHIFTU64 iemAImpl_rol_u64, iemAImpl_rol_u64_amd, iemAImpl_rol_u64_intel;
1502FNIEMAIMPLSHIFTU64 iemAImpl_ror_u64, iemAImpl_ror_u64_amd, iemAImpl_ror_u64_intel;
1503FNIEMAIMPLSHIFTU64 iemAImpl_rcl_u64, iemAImpl_rcl_u64_amd, iemAImpl_rcl_u64_intel;
1504FNIEMAIMPLSHIFTU64 iemAImpl_rcr_u64, iemAImpl_rcr_u64_amd, iemAImpl_rcr_u64_intel;
1505FNIEMAIMPLSHIFTU64 iemAImpl_shl_u64, iemAImpl_shl_u64_amd, iemAImpl_shl_u64_intel;
1506FNIEMAIMPLSHIFTU64 iemAImpl_shr_u64, iemAImpl_shr_u64_amd, iemAImpl_shr_u64_intel;
1507FNIEMAIMPLSHIFTU64 iemAImpl_sar_u64, iemAImpl_sar_u64_amd, iemAImpl_sar_u64_intel;
1508/** @} */
1509
1510/** @name Multiplication and division operations.
1511 * @{ */
1512typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU8,(uint16_t *pu16AX, uint8_t u8FactorDivisor, uint32_t *pEFlags));
1513typedef FNIEMAIMPLMULDIVU8 *PFNIEMAIMPLMULDIVU8;
1514FNIEMAIMPLMULDIVU8 iemAImpl_mul_u8, iemAImpl_mul_u8_amd, iemAImpl_mul_u8_intel;
1515FNIEMAIMPLMULDIVU8 iemAImpl_imul_u8, iemAImpl_imul_u8_amd, iemAImpl_imul_u8_intel;
1516FNIEMAIMPLMULDIVU8 iemAImpl_div_u8, iemAImpl_div_u8_amd, iemAImpl_div_u8_intel;
1517FNIEMAIMPLMULDIVU8 iemAImpl_idiv_u8, iemAImpl_idiv_u8_amd, iemAImpl_idiv_u8_intel;
1518
1519typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU16,(uint16_t *pu16AX, uint16_t *pu16DX, uint16_t u16FactorDivisor, uint32_t *pEFlags));
1520typedef FNIEMAIMPLMULDIVU16 *PFNIEMAIMPLMULDIVU16;
1521FNIEMAIMPLMULDIVU16 iemAImpl_mul_u16, iemAImpl_mul_u16_amd, iemAImpl_mul_u16_intel;
1522FNIEMAIMPLMULDIVU16 iemAImpl_imul_u16, iemAImpl_imul_u16_amd, iemAImpl_imul_u16_intel;
1523FNIEMAIMPLMULDIVU16 iemAImpl_div_u16, iemAImpl_div_u16_amd, iemAImpl_div_u16_intel;
1524FNIEMAIMPLMULDIVU16 iemAImpl_idiv_u16, iemAImpl_idiv_u16_amd, iemAImpl_idiv_u16_intel;
1525
1526typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU32,(uint32_t *pu32EAX, uint32_t *pu32EDX, uint32_t u32FactorDivisor, uint32_t *pEFlags));
1527typedef FNIEMAIMPLMULDIVU32 *PFNIEMAIMPLMULDIVU32;
1528FNIEMAIMPLMULDIVU32 iemAImpl_mul_u32, iemAImpl_mul_u32_amd, iemAImpl_mul_u32_intel;
1529FNIEMAIMPLMULDIVU32 iemAImpl_imul_u32, iemAImpl_imul_u32_amd, iemAImpl_imul_u32_intel;
1530FNIEMAIMPLMULDIVU32 iemAImpl_div_u32, iemAImpl_div_u32_amd, iemAImpl_div_u32_intel;
1531FNIEMAIMPLMULDIVU32 iemAImpl_idiv_u32, iemAImpl_idiv_u32_amd, iemAImpl_idiv_u32_intel;
1532
1533typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64FactorDivisor, uint32_t *pEFlags));
1534typedef FNIEMAIMPLMULDIVU64 *PFNIEMAIMPLMULDIVU64;
1535FNIEMAIMPLMULDIVU64 iemAImpl_mul_u64, iemAImpl_mul_u64_amd, iemAImpl_mul_u64_intel;
1536FNIEMAIMPLMULDIVU64 iemAImpl_imul_u64, iemAImpl_imul_u64_amd, iemAImpl_imul_u64_intel;
1537FNIEMAIMPLMULDIVU64 iemAImpl_div_u64, iemAImpl_div_u64_amd, iemAImpl_div_u64_intel;
1538FNIEMAIMPLMULDIVU64 iemAImpl_idiv_u64, iemAImpl_idiv_u64_amd, iemAImpl_idiv_u64_intel;
1539/** @} */
1540
1541/** @name Byte Swap.
1542 * @{ */
1543IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u16,(uint32_t *pu32Dst)); /* Yes, 32-bit register access. */
1544IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u32,(uint32_t *pu32Dst));
1545IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u64,(uint64_t *pu64Dst));
1546/** @} */
1547
1548/** @name Misc.
1549 * @{ */
1550FNIEMAIMPLBINU16 iemAImpl_arpl;
1551/** @} */
1552
1553
1554/** @name FPU operations taking a 32-bit float argument
1555 * @{ */
1556typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1557 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1558typedef FNIEMAIMPLFPUR32FSW *PFNIEMAIMPLFPUR32FSW;
1559
1560typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1561 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1562typedef FNIEMAIMPLFPUR32 *PFNIEMAIMPLFPUR32;
1563
1564FNIEMAIMPLFPUR32FSW iemAImpl_fcom_r80_by_r32;
1565FNIEMAIMPLFPUR32 iemAImpl_fadd_r80_by_r32;
1566FNIEMAIMPLFPUR32 iemAImpl_fmul_r80_by_r32;
1567FNIEMAIMPLFPUR32 iemAImpl_fsub_r80_by_r32;
1568FNIEMAIMPLFPUR32 iemAImpl_fsubr_r80_by_r32;
1569FNIEMAIMPLFPUR32 iemAImpl_fdiv_r80_by_r32;
1570FNIEMAIMPLFPUR32 iemAImpl_fdivr_r80_by_r32;
1571
1572IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val));
1573IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1574 PRTFLOAT32U pr32Val, PCRTFLOAT80U pr80Val));
1575/** @} */
1576
1577/** @name FPU operations taking a 64-bit float argument
1578 * @{ */
1579typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1580 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1581typedef FNIEMAIMPLFPUR64FSW *PFNIEMAIMPLFPUR64FSW;
1582
1583typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1584 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1585typedef FNIEMAIMPLFPUR64 *PFNIEMAIMPLFPUR64;
1586
1587FNIEMAIMPLFPUR64FSW iemAImpl_fcom_r80_by_r64;
1588FNIEMAIMPLFPUR64 iemAImpl_fadd_r80_by_r64;
1589FNIEMAIMPLFPUR64 iemAImpl_fmul_r80_by_r64;
1590FNIEMAIMPLFPUR64 iemAImpl_fsub_r80_by_r64;
1591FNIEMAIMPLFPUR64 iemAImpl_fsubr_r80_by_r64;
1592FNIEMAIMPLFPUR64 iemAImpl_fdiv_r80_by_r64;
1593FNIEMAIMPLFPUR64 iemAImpl_fdivr_r80_by_r64;
1594
1595IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val));
1596IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1597 PRTFLOAT64U pr32Val, PCRTFLOAT80U pr80Val));
1598/** @} */
1599
1600/** @name FPU operations taking a 80-bit float argument
1601 * @{ */
1602typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1603 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1604typedef FNIEMAIMPLFPUR80 *PFNIEMAIMPLFPUR80;
1605FNIEMAIMPLFPUR80 iemAImpl_fadd_r80_by_r80;
1606FNIEMAIMPLFPUR80 iemAImpl_fmul_r80_by_r80;
1607FNIEMAIMPLFPUR80 iemAImpl_fsub_r80_by_r80;
1608FNIEMAIMPLFPUR80 iemAImpl_fsubr_r80_by_r80;
1609FNIEMAIMPLFPUR80 iemAImpl_fdiv_r80_by_r80;
1610FNIEMAIMPLFPUR80 iemAImpl_fdivr_r80_by_r80;
1611FNIEMAIMPLFPUR80 iemAImpl_fprem_r80_by_r80;
1612FNIEMAIMPLFPUR80 iemAImpl_fprem1_r80_by_r80;
1613FNIEMAIMPLFPUR80 iemAImpl_fscale_r80_by_r80;
1614
1615FNIEMAIMPLFPUR80 iemAImpl_fpatan_r80_by_r80, iemAImpl_fpatan_r80_by_r80_amd, iemAImpl_fpatan_r80_by_r80_intel;
1616FNIEMAIMPLFPUR80 iemAImpl_fyl2x_r80_by_r80, iemAImpl_fyl2x_r80_by_r80_amd, iemAImpl_fyl2x_r80_by_r80_intel;
1617FNIEMAIMPLFPUR80 iemAImpl_fyl2xp1_r80_by_r80, iemAImpl_fyl2xp1_r80_by_r80_amd, iemAImpl_fyl2xp1_r80_by_r80_intel;
1618
1619typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1620 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1621typedef FNIEMAIMPLFPUR80FSW *PFNIEMAIMPLFPUR80FSW;
1622FNIEMAIMPLFPUR80FSW iemAImpl_fcom_r80_by_r80;
1623FNIEMAIMPLFPUR80FSW iemAImpl_fucom_r80_by_r80;
1624
1625typedef IEM_DECL_IMPL_TYPE(uint32_t, FNIEMAIMPLFPUR80EFL,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
1626 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1627typedef FNIEMAIMPLFPUR80EFL *PFNIEMAIMPLFPUR80EFL;
1628FNIEMAIMPLFPUR80EFL iemAImpl_fcomi_r80_by_r80;
1629FNIEMAIMPLFPUR80EFL iemAImpl_fucomi_r80_by_r80;
1630
1631typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARY,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1632typedef FNIEMAIMPLFPUR80UNARY *PFNIEMAIMPLFPUR80UNARY;
1633FNIEMAIMPLFPUR80UNARY iemAImpl_fabs_r80;
1634FNIEMAIMPLFPUR80UNARY iemAImpl_fchs_r80;
1635FNIEMAIMPLFPUR80UNARY iemAImpl_f2xm1_r80, iemAImpl_f2xm1_r80_amd, iemAImpl_f2xm1_r80_intel;
1636FNIEMAIMPLFPUR80UNARY iemAImpl_fsqrt_r80;
1637FNIEMAIMPLFPUR80UNARY iemAImpl_frndint_r80;
1638FNIEMAIMPLFPUR80UNARY iemAImpl_fsin_r80, iemAImpl_fsin_r80_amd, iemAImpl_fsin_r80_intel;
1639FNIEMAIMPLFPUR80UNARY iemAImpl_fcos_r80, iemAImpl_fcos_r80_amd, iemAImpl_fcos_r80_intel;
1640
1641typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYFSW,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val));
1642typedef FNIEMAIMPLFPUR80UNARYFSW *PFNIEMAIMPLFPUR80UNARYFSW;
1643FNIEMAIMPLFPUR80UNARYFSW iemAImpl_ftst_r80;
1644FNIEMAIMPLFPUR80UNARYFSW iemAImpl_fxam_r80;
1645
1646typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80LDCONST,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes));
1647typedef FNIEMAIMPLFPUR80LDCONST *PFNIEMAIMPLFPUR80LDCONST;
1648FNIEMAIMPLFPUR80LDCONST iemAImpl_fld1;
1649FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2t;
1650FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2e;
1651FNIEMAIMPLFPUR80LDCONST iemAImpl_fldpi;
1652FNIEMAIMPLFPUR80LDCONST iemAImpl_fldlg2;
1653FNIEMAIMPLFPUR80LDCONST iemAImpl_fldln2;
1654FNIEMAIMPLFPUR80LDCONST iemAImpl_fldz;
1655
1656typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYTWO,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo,
1657 PCRTFLOAT80U pr80Val));
1658typedef FNIEMAIMPLFPUR80UNARYTWO *PFNIEMAIMPLFPUR80UNARYTWO;
1659FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fptan_r80_r80, iemAImpl_fptan_r80_r80_amd, iemAImpl_fptan_r80_r80_intel;
1660FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fxtract_r80_r80;
1661FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fsincos_r80_r80, iemAImpl_fsincos_r80_r80_amd, iemAImpl_fsincos_r80_r80_intel;
1662
1663IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1664IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1665 PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src));
1666
1667IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val));
1668IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1669 PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src));
1670
1671/** @} */
1672
1673/** @name FPU operations taking a 16-bit signed integer argument
1674 * @{ */
1675typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1676 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1677typedef FNIEMAIMPLFPUI16 *PFNIEMAIMPLFPUI16;
1678typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI16,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1679 int16_t *pi16Dst, PCRTFLOAT80U pr80Src));
1680typedef FNIEMAIMPLFPUSTR80TOI16 *PFNIEMAIMPLFPUSTR80TOI16;
1681
1682FNIEMAIMPLFPUI16 iemAImpl_fiadd_r80_by_i16;
1683FNIEMAIMPLFPUI16 iemAImpl_fimul_r80_by_i16;
1684FNIEMAIMPLFPUI16 iemAImpl_fisub_r80_by_i16;
1685FNIEMAIMPLFPUI16 iemAImpl_fisubr_r80_by_i16;
1686FNIEMAIMPLFPUI16 iemAImpl_fidiv_r80_by_i16;
1687FNIEMAIMPLFPUI16 iemAImpl_fidivr_r80_by_i16;
1688
1689typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1690 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1691typedef FNIEMAIMPLFPUI16FSW *PFNIEMAIMPLFPUI16FSW;
1692FNIEMAIMPLFPUI16FSW iemAImpl_ficom_r80_by_i16;
1693
1694IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val));
1695FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fist_r80_to_i16;
1696FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fistt_r80_to_i16, iemAImpl_fistt_r80_to_i16_amd, iemAImpl_fistt_r80_to_i16_intel;
1697/** @} */
1698
1699/** @name FPU operations taking a 32-bit signed integer argument
1700 * @{ */
1701typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1702 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1703typedef FNIEMAIMPLFPUI32 *PFNIEMAIMPLFPUI32;
1704typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI32,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1705 int32_t *pi32Dst, PCRTFLOAT80U pr80Src));
1706typedef FNIEMAIMPLFPUSTR80TOI32 *PFNIEMAIMPLFPUSTR80TOI32;
1707
1708FNIEMAIMPLFPUI32 iemAImpl_fiadd_r80_by_i32;
1709FNIEMAIMPLFPUI32 iemAImpl_fimul_r80_by_i32;
1710FNIEMAIMPLFPUI32 iemAImpl_fisub_r80_by_i32;
1711FNIEMAIMPLFPUI32 iemAImpl_fisubr_r80_by_i32;
1712FNIEMAIMPLFPUI32 iemAImpl_fidiv_r80_by_i32;
1713FNIEMAIMPLFPUI32 iemAImpl_fidivr_r80_by_i32;
1714
1715typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1716 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1717typedef FNIEMAIMPLFPUI32FSW *PFNIEMAIMPLFPUI32FSW;
1718FNIEMAIMPLFPUI32FSW iemAImpl_ficom_r80_by_i32;
1719
1720IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val));
1721FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fist_r80_to_i32;
1722FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fistt_r80_to_i32;
1723/** @} */
1724
1725/** @name FPU operations taking a 64-bit signed integer argument
1726 * @{ */
1727typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI64,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1728 int64_t *pi64Dst, PCRTFLOAT80U pr80Src));
1729typedef FNIEMAIMPLFPUSTR80TOI64 *PFNIEMAIMPLFPUSTR80TOI64;
1730
1731IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val));
1732FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fist_r80_to_i64;
1733FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fistt_r80_to_i64;
1734/** @} */
1735
1736
1737/** Temporary type representing a 256-bit vector register. */
1738typedef struct { uint64_t au64[4]; } IEMVMM256;
1739/** Temporary type pointing to a 256-bit vector register. */
1740typedef IEMVMM256 *PIEMVMM256;
1741/** Temporary type pointing to a const 256-bit vector register. */
1742typedef IEMVMM256 *PCIEMVMM256;
1743
1744
1745/** @name Media (SSE/MMX/AVX) operations: full1 + full2 -> full1.
1746 * @{ */
1747typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc));
1748typedef FNIEMAIMPLMEDIAF2U64 *PFNIEMAIMPLMEDIAF2U64;
1749typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
1750typedef FNIEMAIMPLMEDIAF2U128 *PFNIEMAIMPLMEDIAF2U128;
1751typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U128,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1752typedef FNIEMAIMPLMEDIAF3U128 *PFNIEMAIMPLMEDIAF3U128;
1753typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U256,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1754typedef FNIEMAIMPLMEDIAF3U256 *PFNIEMAIMPLMEDIAF3U256;
1755typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U64,(uint64_t *puDst, uint64_t const *puSrc));
1756typedef FNIEMAIMPLMEDIAOPTF2U64 *PFNIEMAIMPLMEDIAOPTF2U64;
1757typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U128,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1758typedef FNIEMAIMPLMEDIAOPTF2U128 *PFNIEMAIMPLMEDIAOPTF2U128;
1759typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U128,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1760typedef FNIEMAIMPLMEDIAOPTF3U128 *PFNIEMAIMPLMEDIAOPTF3U128;
1761typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U256,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1762typedef FNIEMAIMPLMEDIAOPTF3U256 *PFNIEMAIMPLMEDIAOPTF3U256;
1763FNIEMAIMPLMEDIAF2U64 iemAImpl_pshufb_u64, iemAImpl_pshufb_u64_fallback;
1764FNIEMAIMPLMEDIAF2U64 iemAImpl_pand_u64, iemAImpl_pandn_u64, iemAImpl_por_u64, iemAImpl_pxor_u64;
1765FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqd_u64;
1766FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpgtb_u64, iemAImpl_pcmpgtw_u64, iemAImpl_pcmpgtd_u64;
1767FNIEMAIMPLMEDIAF2U64 iemAImpl_paddb_u64;
1768FNIEMAIMPLMEDIAF2U64 iemAImpl_paddw_u64;
1769FNIEMAIMPLMEDIAF2U64 iemAImpl_paddd_u64;
1770FNIEMAIMPLMEDIAF2U64 iemAImpl_paddq_u64;
1771FNIEMAIMPLMEDIAF2U64 iemAImpl_psubb_u64;
1772FNIEMAIMPLMEDIAF2U64 iemAImpl_psubw_u64;
1773FNIEMAIMPLMEDIAF2U64 iemAImpl_psubd_u64;
1774FNIEMAIMPLMEDIAF2U64 iemAImpl_psubq_u64;
1775FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packsswb_u64;
1776FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packssdw_u64;
1777FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packuswb_u64;
1778
1779FNIEMAIMPLMEDIAF2U128 iemAImpl_pshufb_u128, iemAImpl_pshufb_u128_fallback;
1780FNIEMAIMPLMEDIAF2U128 iemAImpl_pand_u128, iemAImpl_pandn_u128, iemAImpl_por_u128, iemAImpl_pxor_u128;
1781FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqb_u128, iemAImpl_pcmpeqw_u128, iemAImpl_pcmpeqd_u128;
1782FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqq_u128, iemAImpl_pcmpeqq_u128_fallback;
1783FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtb_u128, iemAImpl_pcmpgtw_u128, iemAImpl_pcmpgtd_u128;
1784FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtq_u128, iemAImpl_pcmpgtq_u128_fallback;
1785FNIEMAIMPLMEDIAF2U128 iemAImpl_paddb_u128;
1786FNIEMAIMPLMEDIAF2U128 iemAImpl_paddw_u128;
1787FNIEMAIMPLMEDIAF2U128 iemAImpl_paddd_u128;
1788FNIEMAIMPLMEDIAF2U128 iemAImpl_paddq_u128;
1789FNIEMAIMPLMEDIAF2U128 iemAImpl_psubb_u128;
1790FNIEMAIMPLMEDIAF2U128 iemAImpl_psubw_u128;
1791FNIEMAIMPLMEDIAF2U128 iemAImpl_psubd_u128;
1792FNIEMAIMPLMEDIAF2U128 iemAImpl_psubq_u128;
1793FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packsswb_u128;
1794FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packssdw_u128;
1795FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packuswb_u128;
1796FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packusdw_u128;
1797
1798FNIEMAIMPLMEDIAF3U128 iemAImpl_vpshufb_u128, iemAImpl_vpshufb_u128_fallback;
1799FNIEMAIMPLMEDIAF3U128 iemAImpl_vpand_u128, iemAImpl_vpand_u128_fallback;
1800FNIEMAIMPLMEDIAF3U128 iemAImpl_vpandn_u128, iemAImpl_vpandn_u128_fallback;
1801FNIEMAIMPLMEDIAF3U128 iemAImpl_vpor_u128, iemAImpl_vpor_u128_fallback;
1802FNIEMAIMPLMEDIAF3U128 iemAImpl_vpxor_u128, iemAImpl_vpxor_u128_fallback;
1803FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqb_u128, iemAImpl_vpcmpeqb_u128_fallback;
1804FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqw_u128, iemAImpl_vpcmpeqw_u128_fallback;
1805FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqd_u128, iemAImpl_vpcmpeqd_u128_fallback;
1806FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqq_u128, iemAImpl_vpcmpeqq_u128_fallback;
1807FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtb_u128, iemAImpl_vpcmpgtb_u128_fallback;
1808FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtw_u128, iemAImpl_vpcmpgtw_u128_fallback;
1809FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtd_u128, iemAImpl_vpcmpgtd_u128_fallback;
1810FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtq_u128, iemAImpl_vpcmpgtq_u128_fallback;
1811FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddb_u128, iemAImpl_vpaddb_u128_fallback;
1812FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddw_u128, iemAImpl_vpaddw_u128_fallback;
1813FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddd_u128, iemAImpl_vpaddd_u128_fallback;
1814FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddq_u128, iemAImpl_vpaddq_u128_fallback;
1815FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubb_u128, iemAImpl_vpsubb_u128_fallback;
1816FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubw_u128, iemAImpl_vpsubw_u128_fallback;
1817FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubd_u128, iemAImpl_vpsubd_u128_fallback;
1818FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubq_u128, iemAImpl_vpsubq_u128_fallback;
1819FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpacksswb_u128, iemAImpl_vpacksswb_u128_fallback;
1820FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackssdw_u128, iemAImpl_vpackssdw_u128_fallback;
1821FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackuswb_u128, iemAImpl_vpackuswb_u128_fallback;
1822FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackusdw_u128, iemAImpl_vpackusdw_u128_fallback;
1823
1824FNIEMAIMPLMEDIAF3U256 iemAImpl_vpshufb_u256, iemAImpl_vpshufb_u256_fallback;
1825FNIEMAIMPLMEDIAF3U256 iemAImpl_vpand_u256, iemAImpl_vpand_u256_fallback;
1826FNIEMAIMPLMEDIAF3U256 iemAImpl_vpandn_u256, iemAImpl_vpandn_u256_fallback;
1827FNIEMAIMPLMEDIAF3U256 iemAImpl_vpor_u256, iemAImpl_vpor_u256_fallback;
1828FNIEMAIMPLMEDIAF3U256 iemAImpl_vpxor_u256, iemAImpl_vpxor_u256_fallback;
1829FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqb_u256, iemAImpl_vpcmpeqb_u256_fallback;
1830FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqw_u256, iemAImpl_vpcmpeqw_u256_fallback;
1831FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqd_u256, iemAImpl_vpcmpeqd_u256_fallback;
1832FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqq_u256, iemAImpl_vpcmpeqq_u256_fallback;
1833FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtb_u256, iemAImpl_vpcmpgtb_u256_fallback;
1834FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtw_u256, iemAImpl_vpcmpgtw_u256_fallback;
1835FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtd_u256, iemAImpl_vpcmpgtd_u256_fallback;
1836FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtq_u256, iemAImpl_vpcmpgtq_u256_fallback;
1837FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddb_u256, iemAImpl_vpaddb_u256_fallback;
1838FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddw_u256, iemAImpl_vpaddw_u256_fallback;
1839FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddd_u256, iemAImpl_vpaddd_u256_fallback;
1840FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddq_u256, iemAImpl_vpaddq_u256_fallback;
1841FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubb_u256, iemAImpl_vpsubb_u256_fallback;
1842FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubw_u256, iemAImpl_vpsubw_u256_fallback;
1843FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubd_u256, iemAImpl_vpsubd_u256_fallback;
1844FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubq_u256, iemAImpl_vpsubq_u256_fallback;
1845FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpacksswb_u256, iemAImpl_vpacksswb_u256_fallback;
1846FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackssdw_u256, iemAImpl_vpackssdw_u256_fallback;
1847FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackuswb_u256, iemAImpl_vpackuswb_u256_fallback;
1848FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackusdw_u256, iemAImpl_vpackusdw_u256_fallback;
1849/** @} */
1850
1851/** @name Media (SSE/MMX/AVX) operations: lowhalf1 + lowhalf1 -> full1.
1852 * @{ */
1853FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpcklbw_u64, iemAImpl_punpcklwd_u64, iemAImpl_punpckldq_u64;
1854FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpcklbw_u128, iemAImpl_punpcklwd_u128, iemAImpl_punpckldq_u128, iemAImpl_punpcklqdq_u128;
1855FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpcklbw_u128, iemAImpl_vpunpcklbw_u128_fallback,
1856 iemAImpl_vpunpcklwd_u128, iemAImpl_vpunpcklwd_u128_fallback,
1857 iemAImpl_vpunpckldq_u128, iemAImpl_vpunpckldq_u128_fallback,
1858 iemAImpl_vpunpcklqdq_u128, iemAImpl_vpunpcklqdq_u128_fallback;
1859FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpcklbw_u256, iemAImpl_vpunpcklbw_u256_fallback,
1860 iemAImpl_vpunpcklwd_u256, iemAImpl_vpunpcklwd_u256_fallback,
1861 iemAImpl_vpunpckldq_u256, iemAImpl_vpunpckldq_u256_fallback,
1862 iemAImpl_vpunpcklqdq_u256, iemAImpl_vpunpcklqdq_u256_fallback;
1863/** @} */
1864
1865/** @name Media (SSE/MMX/AVX) operations: hihalf1 + hihalf2 -> full1.
1866 * @{ */
1867FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpckhbw_u64, iemAImpl_punpckhwd_u64, iemAImpl_punpckhdq_u64;
1868FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpckhbw_u128, iemAImpl_punpckhwd_u128, iemAImpl_punpckhdq_u128, iemAImpl_punpckhqdq_u128;
1869FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpckhbw_u128, iemAImpl_vpunpckhbw_u128_fallback,
1870 iemAImpl_vpunpckhwd_u128, iemAImpl_vpunpckhwd_u128_fallback,
1871 iemAImpl_vpunpckhdq_u128, iemAImpl_vpunpckhdq_u128_fallback,
1872 iemAImpl_vpunpckhqdq_u128, iemAImpl_vpunpckhqdq_u128_fallback;
1873FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpckhbw_u256, iemAImpl_vpunpckhbw_u256_fallback,
1874 iemAImpl_vpunpckhwd_u256, iemAImpl_vpunpckhwd_u256_fallback,
1875 iemAImpl_vpunpckhdq_u256, iemAImpl_vpunpckhdq_u256_fallback,
1876 iemAImpl_vpunpckhqdq_u256, iemAImpl_vpunpckhqdq_u256_fallback;
1877/** @} */
1878
1879/** @name Media (SSE/MMX/AVX) operation: Packed Shuffle Stuff (evil)
1880 * @{ */
1881typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil));
1882typedef FNIEMAIMPLMEDIAPSHUFU128 *PFNIEMAIMPLMEDIAPSHUFU128;
1883typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU256,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil));
1884typedef FNIEMAIMPLMEDIAPSHUFU256 *PFNIEMAIMPLMEDIAPSHUFU256;
1885IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw_u64,(uint64_t *puDst, uint64_t const *puSrc, uint8_t bEvil));
1886FNIEMAIMPLMEDIAPSHUFU128 iemAImpl_pshufhw_u128, iemAImpl_pshuflw_u128, iemAImpl_pshufd_u128;
1887#ifndef IEM_WITHOUT_ASSEMBLY
1888FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256, iemAImpl_vpshuflw_u256, iemAImpl_vpshufd_u256;
1889#endif
1890FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256_fallback, iemAImpl_vpshuflw_u256_fallback, iemAImpl_vpshufd_u256_fallback;
1891/** @} */
1892
1893/** @name Media (SSE/MMX/AVX) operation: Move Byte Mask
1894 * @{ */
1895IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(uint64_t *pu64Dst, uint64_t const *puSrc));
1896IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(uint64_t *pu64Dst, PCRTUINT128U puSrc));
1897#ifndef IEM_WITHOUT_ASSEMBLY
1898IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
1899#endif
1900IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256_fallback,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
1901/** @} */
1902
1903/** @name Media (SSE/MMX/AVX) operation: Sort this later
1904 * @{ */
1905IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1906IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1907IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PRTUINT128U puDst, uint64_t uSrc));
1908
1909IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1910IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1911IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1912IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1913IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1914IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1915
1916/** @} */
1917
1918/** @name Media Odds and Ends
1919 * @{ */
1920typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U8,(uint32_t *puDst, uint8_t uSrc));
1921typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U16,(uint32_t *puDst, uint16_t uSrc));
1922typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U32,(uint32_t *puDst, uint32_t uSrc));
1923typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U64,(uint32_t *puDst, uint64_t uSrc));
1924FNIEMAIMPLCR32U8 iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback;
1925FNIEMAIMPLCR32U16 iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback;
1926FNIEMAIMPLCR32U32 iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback;
1927FNIEMAIMPLCR32U64 iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback;
1928/** @} */
1929
1930
1931/** @name Function tables.
1932 * @{
1933 */
1934
1935/**
1936 * Function table for a binary operator providing implementation based on
1937 * operand size.
1938 */
1939typedef struct IEMOPBINSIZES
1940{
1941 PFNIEMAIMPLBINU8 pfnNormalU8, pfnLockedU8;
1942 PFNIEMAIMPLBINU16 pfnNormalU16, pfnLockedU16;
1943 PFNIEMAIMPLBINU32 pfnNormalU32, pfnLockedU32;
1944 PFNIEMAIMPLBINU64 pfnNormalU64, pfnLockedU64;
1945} IEMOPBINSIZES;
1946/** Pointer to a binary operator function table. */
1947typedef IEMOPBINSIZES const *PCIEMOPBINSIZES;
1948
1949
1950/**
1951 * Function table for a unary operator providing implementation based on
1952 * operand size.
1953 */
1954typedef struct IEMOPUNARYSIZES
1955{
1956 PFNIEMAIMPLUNARYU8 pfnNormalU8, pfnLockedU8;
1957 PFNIEMAIMPLUNARYU16 pfnNormalU16, pfnLockedU16;
1958 PFNIEMAIMPLUNARYU32 pfnNormalU32, pfnLockedU32;
1959 PFNIEMAIMPLUNARYU64 pfnNormalU64, pfnLockedU64;
1960} IEMOPUNARYSIZES;
1961/** Pointer to a unary operator function table. */
1962typedef IEMOPUNARYSIZES const *PCIEMOPUNARYSIZES;
1963
1964
1965/**
1966 * Function table for a shift operator providing implementation based on
1967 * operand size.
1968 */
1969typedef struct IEMOPSHIFTSIZES
1970{
1971 PFNIEMAIMPLSHIFTU8 pfnNormalU8;
1972 PFNIEMAIMPLSHIFTU16 pfnNormalU16;
1973 PFNIEMAIMPLSHIFTU32 pfnNormalU32;
1974 PFNIEMAIMPLSHIFTU64 pfnNormalU64;
1975} IEMOPSHIFTSIZES;
1976/** Pointer to a shift operator function table. */
1977typedef IEMOPSHIFTSIZES const *PCIEMOPSHIFTSIZES;
1978
1979
1980/**
1981 * Function table for a multiplication or division operation.
1982 */
1983typedef struct IEMOPMULDIVSIZES
1984{
1985 PFNIEMAIMPLMULDIVU8 pfnU8;
1986 PFNIEMAIMPLMULDIVU16 pfnU16;
1987 PFNIEMAIMPLMULDIVU32 pfnU32;
1988 PFNIEMAIMPLMULDIVU64 pfnU64;
1989} IEMOPMULDIVSIZES;
1990/** Pointer to a multiplication or division operation function table. */
1991typedef IEMOPMULDIVSIZES const *PCIEMOPMULDIVSIZES;
1992
1993
1994/**
1995 * Function table for a double precision shift operator providing implementation
1996 * based on operand size.
1997 */
1998typedef struct IEMOPSHIFTDBLSIZES
1999{
2000 PFNIEMAIMPLSHIFTDBLU16 pfnNormalU16;
2001 PFNIEMAIMPLSHIFTDBLU32 pfnNormalU32;
2002 PFNIEMAIMPLSHIFTDBLU64 pfnNormalU64;
2003} IEMOPSHIFTDBLSIZES;
2004/** Pointer to a double precision shift function table. */
2005typedef IEMOPSHIFTDBLSIZES const *PCIEMOPSHIFTDBLSIZES;
2006
2007
2008/**
2009 * Function table for media instruction taking two full sized media source
2010 * registers and one full sized destination register (AVX).
2011 */
2012typedef struct IEMOPMEDIAF3
2013{
2014 PFNIEMAIMPLMEDIAF3U128 pfnU128;
2015 PFNIEMAIMPLMEDIAF3U256 pfnU256;
2016} IEMOPMEDIAF3;
2017/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2018typedef IEMOPMEDIAF3 const *PCIEMOPMEDIAF3;
2019
2020/** @def IEMOPMEDIAF3_INIT_VARS_EX
2021 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2022 * given functions as initializers. For use in AVX functions where a pair of
2023 * functions are only used once and the function table need not be public. */
2024#ifndef TST_IEM_CHECK_MC
2025# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2026# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2027 static IEMOPMEDIAF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2028 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2029# else
2030# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2031 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2032# endif
2033#else
2034# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2035#endif
2036/** @def IEMOPMEDIAF3_INIT_VARS
2037 * Generate AVX function tables for the @a a_InstrNm instruction.
2038 * @sa IEMOPMEDIAF3_INIT_VARS_EX */
2039#define IEMOPMEDIAF3_INIT_VARS(a_InstrNm) \
2040 IEMOPMEDIAF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2041 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2042
2043/**
2044 * Function table for media instruction taking two full sized media source
2045 * registers and one full sized destination register, but no additional state
2046 * (AVX).
2047 */
2048typedef struct IEMOPMEDIAOPTF3
2049{
2050 PFNIEMAIMPLMEDIAOPTF3U128 pfnU128;
2051 PFNIEMAIMPLMEDIAOPTF3U256 pfnU256;
2052} IEMOPMEDIAOPTF3;
2053/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2054typedef IEMOPMEDIAOPTF3 const *PCIEMOPMEDIAOPTF3;
2055
2056/** @def IEMOPMEDIAOPTF3_INIT_VARS_EX
2057 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2058 * given functions as initializers. For use in AVX functions where a pair of
2059 * functions are only used once and the function table need not be public. */
2060#ifndef TST_IEM_CHECK_MC
2061# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2062# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2063 static IEMOPMEDIAOPTF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2064 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2065# else
2066# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2067 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2068# endif
2069#else
2070# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2071#endif
2072/** @def IEMOPMEDIAOPTF3_INIT_VARS
2073 * Generate AVX function tables for the @a a_InstrNm instruction.
2074 * @sa IEMOPMEDIAOPTF3_INIT_VARS_EX */
2075#define IEMOPMEDIAOPTF3_INIT_VARS(a_InstrNm) \
2076 IEMOPMEDIAOPTF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2077 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2078
2079
2080/** @} */
2081
2082
2083/** @name C instruction implementations for anything slightly complicated.
2084 * @{ */
2085
2086/**
2087 * For typedef'ing or declaring a C instruction implementation function taking
2088 * no extra arguments.
2089 *
2090 * @param a_Name The name of the type.
2091 */
2092# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2093 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2094/**
2095 * For defining a C instruction implementation function taking no extra
2096 * arguments.
2097 *
2098 * @param a_Name The name of the function
2099 */
2100# define IEM_CIMPL_DEF_0(a_Name) \
2101 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2102/**
2103 * Prototype version of IEM_CIMPL_DEF_0.
2104 */
2105# define IEM_CIMPL_PROTO_0(a_Name) \
2106 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2107/**
2108 * For calling a C instruction implementation function taking no extra
2109 * arguments.
2110 *
2111 * This special call macro adds default arguments to the call and allow us to
2112 * change these later.
2113 *
2114 * @param a_fn The name of the function.
2115 */
2116# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2117
2118/**
2119 * For typedef'ing or declaring a C instruction implementation function taking
2120 * one extra argument.
2121 *
2122 * @param a_Name The name of the type.
2123 * @param a_Type0 The argument type.
2124 * @param a_Arg0 The argument name.
2125 */
2126# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2127 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2128/**
2129 * For defining a C instruction implementation function taking one extra
2130 * argument.
2131 *
2132 * @param a_Name The name of the function
2133 * @param a_Type0 The argument type.
2134 * @param a_Arg0 The argument name.
2135 */
2136# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2137 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2138/**
2139 * Prototype version of IEM_CIMPL_DEF_1.
2140 */
2141# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2142 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2143/**
2144 * For calling a C instruction implementation function taking one extra
2145 * argument.
2146 *
2147 * This special call macro adds default arguments to the call and allow us to
2148 * change these later.
2149 *
2150 * @param a_fn The name of the function.
2151 * @param a0 The name of the 1st argument.
2152 */
2153# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2154
2155/**
2156 * For typedef'ing or declaring a C instruction implementation function taking
2157 * two extra arguments.
2158 *
2159 * @param a_Name The name of the type.
2160 * @param a_Type0 The type of the 1st argument
2161 * @param a_Arg0 The name of the 1st argument.
2162 * @param a_Type1 The type of the 2nd argument.
2163 * @param a_Arg1 The name of the 2nd argument.
2164 */
2165# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2166 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2167/**
2168 * For defining a C instruction implementation function taking two extra
2169 * arguments.
2170 *
2171 * @param a_Name The name of the function.
2172 * @param a_Type0 The type of the 1st argument
2173 * @param a_Arg0 The name of the 1st argument.
2174 * @param a_Type1 The type of the 2nd argument.
2175 * @param a_Arg1 The name of the 2nd argument.
2176 */
2177# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2178 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2179/**
2180 * Prototype version of IEM_CIMPL_DEF_2.
2181 */
2182# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2183 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2184/**
2185 * For calling a C instruction implementation function taking two extra
2186 * arguments.
2187 *
2188 * This special call macro adds default arguments to the call and allow us to
2189 * change these later.
2190 *
2191 * @param a_fn The name of the function.
2192 * @param a0 The name of the 1st argument.
2193 * @param a1 The name of the 2nd argument.
2194 */
2195# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
2196
2197/**
2198 * For typedef'ing or declaring a C instruction implementation function taking
2199 * three extra arguments.
2200 *
2201 * @param a_Name The name of the type.
2202 * @param a_Type0 The type of the 1st argument
2203 * @param a_Arg0 The name of the 1st argument.
2204 * @param a_Type1 The type of the 2nd argument.
2205 * @param a_Arg1 The name of the 2nd argument.
2206 * @param a_Type2 The type of the 3rd argument.
2207 * @param a_Arg2 The name of the 3rd argument.
2208 */
2209# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2210 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2211/**
2212 * For defining a C instruction implementation function taking three extra
2213 * arguments.
2214 *
2215 * @param a_Name The name of the function.
2216 * @param a_Type0 The type of the 1st argument
2217 * @param a_Arg0 The name of the 1st argument.
2218 * @param a_Type1 The type of the 2nd argument.
2219 * @param a_Arg1 The name of the 2nd argument.
2220 * @param a_Type2 The type of the 3rd argument.
2221 * @param a_Arg2 The name of the 3rd argument.
2222 */
2223# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2224 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2225/**
2226 * Prototype version of IEM_CIMPL_DEF_3.
2227 */
2228# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2229 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2230/**
2231 * For calling a C instruction implementation function taking three extra
2232 * arguments.
2233 *
2234 * This special call macro adds default arguments to the call and allow us to
2235 * change these later.
2236 *
2237 * @param a_fn The name of the function.
2238 * @param a0 The name of the 1st argument.
2239 * @param a1 The name of the 2nd argument.
2240 * @param a2 The name of the 3rd argument.
2241 */
2242# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
2243
2244
2245/**
2246 * For typedef'ing or declaring a C instruction implementation function taking
2247 * four extra arguments.
2248 *
2249 * @param a_Name The name of the type.
2250 * @param a_Type0 The type of the 1st argument
2251 * @param a_Arg0 The name of the 1st argument.
2252 * @param a_Type1 The type of the 2nd argument.
2253 * @param a_Arg1 The name of the 2nd argument.
2254 * @param a_Type2 The type of the 3rd argument.
2255 * @param a_Arg2 The name of the 3rd argument.
2256 * @param a_Type3 The type of the 4th argument.
2257 * @param a_Arg3 The name of the 4th argument.
2258 */
2259# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2260 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
2261/**
2262 * For defining a C instruction implementation function taking four extra
2263 * arguments.
2264 *
2265 * @param a_Name The name of the function.
2266 * @param a_Type0 The type of the 1st argument
2267 * @param a_Arg0 The name of the 1st argument.
2268 * @param a_Type1 The type of the 2nd argument.
2269 * @param a_Arg1 The name of the 2nd argument.
2270 * @param a_Type2 The type of the 3rd argument.
2271 * @param a_Arg2 The name of the 3rd argument.
2272 * @param a_Type3 The type of the 4th argument.
2273 * @param a_Arg3 The name of the 4th argument.
2274 */
2275# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2276 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2277 a_Type2 a_Arg2, a_Type3 a_Arg3))
2278/**
2279 * Prototype version of IEM_CIMPL_DEF_4.
2280 */
2281# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2282 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2283 a_Type2 a_Arg2, a_Type3 a_Arg3))
2284/**
2285 * For calling a C instruction implementation function taking four extra
2286 * arguments.
2287 *
2288 * This special call macro adds default arguments to the call and allow us to
2289 * change these later.
2290 *
2291 * @param a_fn The name of the function.
2292 * @param a0 The name of the 1st argument.
2293 * @param a1 The name of the 2nd argument.
2294 * @param a2 The name of the 3rd argument.
2295 * @param a3 The name of the 4th argument.
2296 */
2297# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
2298
2299
2300/**
2301 * For typedef'ing or declaring a C instruction implementation function taking
2302 * five extra arguments.
2303 *
2304 * @param a_Name The name of the type.
2305 * @param a_Type0 The type of the 1st argument
2306 * @param a_Arg0 The name of the 1st argument.
2307 * @param a_Type1 The type of the 2nd argument.
2308 * @param a_Arg1 The name of the 2nd argument.
2309 * @param a_Type2 The type of the 3rd argument.
2310 * @param a_Arg2 The name of the 3rd argument.
2311 * @param a_Type3 The type of the 4th argument.
2312 * @param a_Arg3 The name of the 4th argument.
2313 * @param a_Type4 The type of the 5th argument.
2314 * @param a_Arg4 The name of the 5th argument.
2315 */
2316# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2317 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
2318 a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
2319 a_Type3 a_Arg3, a_Type4 a_Arg4))
2320/**
2321 * For defining a C instruction implementation function taking five extra
2322 * arguments.
2323 *
2324 * @param a_Name The name of the function.
2325 * @param a_Type0 The type of the 1st argument
2326 * @param a_Arg0 The name of the 1st argument.
2327 * @param a_Type1 The type of the 2nd argument.
2328 * @param a_Arg1 The name of the 2nd argument.
2329 * @param a_Type2 The type of the 3rd argument.
2330 * @param a_Arg2 The name of the 3rd argument.
2331 * @param a_Type3 The type of the 4th argument.
2332 * @param a_Arg3 The name of the 4th argument.
2333 * @param a_Type4 The type of the 5th argument.
2334 * @param a_Arg4 The name of the 5th argument.
2335 */
2336# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2337 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2338 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2339/**
2340 * Prototype version of IEM_CIMPL_DEF_5.
2341 */
2342# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2343 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2344 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2345/**
2346 * For calling a C instruction implementation function taking five extra
2347 * arguments.
2348 *
2349 * This special call macro adds default arguments to the call and allow us to
2350 * change these later.
2351 *
2352 * @param a_fn The name of the function.
2353 * @param a0 The name of the 1st argument.
2354 * @param a1 The name of the 2nd argument.
2355 * @param a2 The name of the 3rd argument.
2356 * @param a3 The name of the 4th argument.
2357 * @param a4 The name of the 5th argument.
2358 */
2359# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2360
2361/** @} */
2362
2363
2364/** @name Opcode Decoder Function Types.
2365 * @{ */
2366
2367/** @typedef PFNIEMOP
2368 * Pointer to an opcode decoder function.
2369 */
2370
2371/** @def FNIEMOP_DEF
2372 * Define an opcode decoder function.
2373 *
2374 * We're using macors for this so that adding and removing parameters as well as
2375 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
2376 *
2377 * @param a_Name The function name.
2378 */
2379
2380/** @typedef PFNIEMOPRM
2381 * Pointer to an opcode decoder function with RM byte.
2382 */
2383
2384/** @def FNIEMOPRM_DEF
2385 * Define an opcode decoder function with RM byte.
2386 *
2387 * We're using macors for this so that adding and removing parameters as well as
2388 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
2389 *
2390 * @param a_Name The function name.
2391 */
2392
2393#if defined(__GNUC__) && defined(RT_ARCH_X86)
2394typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
2395typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2396# define FNIEMOP_DEF(a_Name) \
2397 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
2398# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2399 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2400# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2401 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2402
2403#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2404typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
2405typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2406# define FNIEMOP_DEF(a_Name) \
2407 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2408# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2409 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2410# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2411 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2412
2413#elif defined(__GNUC__)
2414typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2415typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2416# define FNIEMOP_DEF(a_Name) \
2417 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
2418# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2419 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2420# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2421 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2422
2423#else
2424typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2425typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2426# define FNIEMOP_DEF(a_Name) \
2427 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2428# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2429 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2430# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2431 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2432
2433#endif
2434#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
2435
2436/**
2437 * Call an opcode decoder function.
2438 *
2439 * We're using macors for this so that adding and removing parameters can be
2440 * done as we please. See FNIEMOP_DEF.
2441 */
2442#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
2443
2444/**
2445 * Call a common opcode decoder function taking one extra argument.
2446 *
2447 * We're using macors for this so that adding and removing parameters can be
2448 * done as we please. See FNIEMOP_DEF_1.
2449 */
2450#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
2451
2452/**
2453 * Call a common opcode decoder function taking one extra argument.
2454 *
2455 * We're using macors for this so that adding and removing parameters can be
2456 * done as we please. See FNIEMOP_DEF_1.
2457 */
2458#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
2459/** @} */
2460
2461
2462/** @name Misc Helpers
2463 * @{ */
2464
2465/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
2466 * due to GCC lacking knowledge about the value range of a switch. */
2467#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
2468
2469/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
2470#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
2471
2472/**
2473 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2474 * occation.
2475 */
2476#ifdef LOG_ENABLED
2477# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2478 do { \
2479 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
2480 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2481 } while (0)
2482#else
2483# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2484 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2485#endif
2486
2487/**
2488 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2489 * occation using the supplied logger statement.
2490 *
2491 * @param a_LoggerArgs What to log on failure.
2492 */
2493#ifdef LOG_ENABLED
2494# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2495 do { \
2496 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
2497 /*LogFunc(a_LoggerArgs);*/ \
2498 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2499 } while (0)
2500#else
2501# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2502 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2503#endif
2504
2505/**
2506 * Check if we're currently executing in real or virtual 8086 mode.
2507 *
2508 * @returns @c true if it is, @c false if not.
2509 * @param a_pVCpu The IEM state of the current CPU.
2510 */
2511#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2512
2513/**
2514 * Check if we're currently executing in virtual 8086 mode.
2515 *
2516 * @returns @c true if it is, @c false if not.
2517 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2518 */
2519#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2520
2521/**
2522 * Check if we're currently executing in long mode.
2523 *
2524 * @returns @c true if it is, @c false if not.
2525 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2526 */
2527#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
2528
2529/**
2530 * Check if we're currently executing in a 64-bit code segment.
2531 *
2532 * @returns @c true if it is, @c false if not.
2533 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2534 */
2535#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
2536
2537/**
2538 * Check if we're currently executing in real mode.
2539 *
2540 * @returns @c true if it is, @c false if not.
2541 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2542 */
2543#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
2544
2545/**
2546 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
2547 * @returns PCCPUMFEATURES
2548 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2549 */
2550#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
2551
2552/**
2553 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
2554 * @returns PCCPUMFEATURES
2555 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2556 */
2557#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&g_CpumHostFeatures.s)
2558
2559/**
2560 * Evaluates to true if we're presenting an Intel CPU to the guest.
2561 */
2562#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
2563
2564/**
2565 * Evaluates to true if we're presenting an AMD CPU to the guest.
2566 */
2567#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
2568
2569/**
2570 * Check if the address is canonical.
2571 */
2572#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
2573
2574/** Checks if the ModR/M byte is in register mode or not. */
2575#define IEM_IS_MODRM_REG_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT) )
2576/** Checks if the ModR/M byte is in memory mode or not. */
2577#define IEM_IS_MODRM_MEM_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT) )
2578
2579/**
2580 * Gets the register (reg) part of a ModR/M encoding, with REX.R added in.
2581 *
2582 * For use during decoding.
2583 */
2584#define IEM_GET_MODRM_REG(a_pVCpu, a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | (a_pVCpu)->iem.s.uRexReg )
2585/**
2586 * Gets the r/m part of a ModR/M encoding as a register index, with REX.B added in.
2587 *
2588 * For use during decoding.
2589 */
2590#define IEM_GET_MODRM_RM(a_pVCpu, a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) | (a_pVCpu)->iem.s.uRexB )
2591
2592/**
2593 * Gets the register (reg) part of a ModR/M encoding, without REX.R.
2594 *
2595 * For use during decoding.
2596 */
2597#define IEM_GET_MODRM_REG_8(a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) )
2598/**
2599 * Gets the r/m part of a ModR/M encoding as a register index, without REX.B.
2600 *
2601 * For use during decoding.
2602 */
2603#define IEM_GET_MODRM_RM_8(a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) )
2604
2605/**
2606 * Gets the effective VEX.VVVV value.
2607 *
2608 * The 4th bit is ignored if not 64-bit code.
2609 * @returns effective V-register value.
2610 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2611 */
2612#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
2613 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
2614
2615
2616#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2617
2618/**
2619 * Check if the guest has entered VMX root operation.
2620 */
2621# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
2622
2623/**
2624 * Check if the guest has entered VMX non-root operation.
2625 */
2626# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
2627
2628/**
2629 * Check if the nested-guest has the given Pin-based VM-execution control set.
2630 */
2631# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
2632 (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
2633
2634/**
2635 * Check if the nested-guest has the given Processor-based VM-execution control set.
2636 */
2637# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
2638 (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
2639
2640/**
2641 * Check if the nested-guest has the given Secondary Processor-based VM-execution
2642 * control set.
2643 */
2644# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
2645 (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
2646
2647/** Gets the guest-physical address of the shadows VMCS for the given VCPU. */
2648# define IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs)
2649
2650/** Whether a shadow VMCS is present for the given VCPU. */
2651# define IEM_VMX_HAS_SHADOW_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2652
2653/** Gets the VMXON region pointer. */
2654# define IEM_VMX_GET_VMXON_PTR(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
2655
2656/** Gets the guest-physical address of the current VMCS for the given VCPU. */
2657# define IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs)
2658
2659/** Whether a current VMCS is present for the given VCPU. */
2660# define IEM_VMX_HAS_CURRENT_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2661
2662/** Assigns the guest-physical address of the current VMCS for the given VCPU. */
2663# define IEM_VMX_SET_CURRENT_VMCS(a_pVCpu, a_GCPhysVmcs) \
2664 do \
2665 { \
2666 Assert((a_GCPhysVmcs) != NIL_RTGCPHYS); \
2667 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = (a_GCPhysVmcs); \
2668 } while (0)
2669
2670/** Clears any current VMCS for the given VCPU. */
2671# define IEM_VMX_CLEAR_CURRENT_VMCS(a_pVCpu) \
2672 do \
2673 { \
2674 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = NIL_RTGCPHYS; \
2675 } while (0)
2676
2677/**
2678 * Invokes the VMX VM-exit handler for an instruction intercept.
2679 */
2680# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
2681 do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
2682
2683/**
2684 * Invokes the VMX VM-exit handler for an instruction intercept where the
2685 * instruction provides additional VM-exit information.
2686 */
2687# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
2688 do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
2689
2690/**
2691 * Invokes the VMX VM-exit handler for a task switch.
2692 */
2693# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
2694 do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
2695
2696/**
2697 * Invokes the VMX VM-exit handler for MWAIT.
2698 */
2699# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
2700 do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
2701
2702/**
2703 * Invokes the VMX VM-exit handler for EPT faults.
2704 */
2705# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) \
2706 do { return iemVmxVmexitEpt(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr); } while (0)
2707
2708/**
2709 * Invokes the VMX VM-exit handler.
2710 */
2711# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
2712 do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
2713
2714#else
2715# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
2716# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
2717# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
2718# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
2719# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
2720# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2721# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2722# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2723# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2724# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2725# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
2726
2727#endif
2728
2729#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2730/**
2731 * Check if an SVM control/instruction intercept is set.
2732 */
2733# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
2734 (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
2735
2736/**
2737 * Check if an SVM read CRx intercept is set.
2738 */
2739# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2740 (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2741
2742/**
2743 * Check if an SVM write CRx intercept is set.
2744 */
2745# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2746 (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2747
2748/**
2749 * Check if an SVM read DRx intercept is set.
2750 */
2751# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2752 (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2753
2754/**
2755 * Check if an SVM write DRx intercept is set.
2756 */
2757# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2758 (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2759
2760/**
2761 * Check if an SVM exception intercept is set.
2762 */
2763# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
2764 (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
2765
2766/**
2767 * Invokes the SVM \#VMEXIT handler for the nested-guest.
2768 */
2769# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2770 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
2771
2772/**
2773 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
2774 * corresponding decode assist information.
2775 */
2776# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
2777 do \
2778 { \
2779 uint64_t uExitInfo1; \
2780 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
2781 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
2782 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
2783 else \
2784 uExitInfo1 = 0; \
2785 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
2786 } while (0)
2787
2788/** Check and handles SVM nested-guest instruction intercept and updates
2789 * NRIP if needed.
2790 */
2791# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2792 do \
2793 { \
2794 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
2795 { \
2796 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2797 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
2798 } \
2799 } while (0)
2800
2801/** Checks and handles SVM nested-guest CR0 read intercept. */
2802# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
2803 do \
2804 { \
2805 if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
2806 { /* probably likely */ } \
2807 else \
2808 { \
2809 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2810 IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
2811 } \
2812 } while (0)
2813
2814/**
2815 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
2816 */
2817# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
2818 do { \
2819 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
2820 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
2821 } while (0)
2822
2823#else
2824# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
2825# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2826# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2827# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2828# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2829# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
2830# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
2831# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
2832# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2833# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2834# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
2835
2836#endif
2837
2838/** @} */
2839
2840
2841
2842/**
2843 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
2844 */
2845typedef union IEMSELDESC
2846{
2847 /** The legacy view. */
2848 X86DESC Legacy;
2849 /** The long mode view. */
2850 X86DESC64 Long;
2851} IEMSELDESC;
2852/** Pointer to a selector descriptor table entry. */
2853typedef IEMSELDESC *PIEMSELDESC;
2854
2855/** @name Raising Exceptions.
2856 * @{ */
2857VBOXSTRICTRC iemTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, uint32_t uNextEip, uint32_t fFlags,
2858 uint16_t uErr, uint64_t uCr2, RTSEL SelTSS, PIEMSELDESC pNewDescTSS) RT_NOEXCEPT;
2859
2860VBOXSTRICTRC iemRaiseXcptOrInt(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector, uint32_t fFlags,
2861 uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2862#ifdef IEM_WITH_SETJMP
2863DECL_NO_RETURN(void) iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector,
2864 uint32_t fFlags, uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2865#endif
2866VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT;
2867VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT;
2868VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT;
2869VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT;
2870VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT;
2871VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2872VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT;
2873VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2874VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2875/*VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;*/
2876VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2877VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2878VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2879VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2880VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2881VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2882#ifdef IEM_WITH_SETJMP
2883DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2884#endif
2885VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2886VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT;
2887VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2888#ifdef IEM_WITH_SETJMP
2889DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2890#endif
2891VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2892#ifdef IEM_WITH_SETJMP
2893DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2894#endif
2895VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2896#ifdef IEM_WITH_SETJMP
2897DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2898#endif
2899VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2900#ifdef IEM_WITH_SETJMP
2901DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2902#endif
2903VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu);
2904VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
2905#ifdef IEM_WITH_SETJMP
2906DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2907#endif
2908
2909IEM_CIMPL_DEF_0(iemCImplRaiseDivideError);
2910IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix);
2911IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode);
2912
2913/**
2914 * Macro for calling iemCImplRaiseDivideError().
2915 *
2916 * This enables us to add/remove arguments and force different levels of
2917 * inlining as we wish.
2918 *
2919 * @return Strict VBox status code.
2920 */
2921#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
2922
2923/**
2924 * Macro for calling iemCImplRaiseInvalidLockPrefix().
2925 *
2926 * This enables us to add/remove arguments and force different levels of
2927 * inlining as we wish.
2928 *
2929 * @return Strict VBox status code.
2930 */
2931#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
2932
2933/**
2934 * Macro for calling iemCImplRaiseInvalidOpcode().
2935 *
2936 * This enables us to add/remove arguments and force different levels of
2937 * inlining as we wish.
2938 *
2939 * @return Strict VBox status code.
2940 */
2941#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
2942/** @} */
2943
2944/** @name Register Access.
2945 * @{ */
2946VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT;
2947VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT;
2948VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT;
2949VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT;
2950/** @} */
2951
2952/** @name FPU access and helpers.
2953 * @{ */
2954void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT;
2955void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2956void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT;
2957void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
2958void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
2959void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
2960 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2961void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
2962 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2963void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2964void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2965void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2966void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2967void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2968void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2969void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
2970void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2971void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
2972void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2973void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT;
2974void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
2975void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT;
2976void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
2977void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2978/** @} */
2979
2980/** @name Memory access.
2981 * @{ */
2982
2983/** Report a \#GP instead of \#AC and do not restrict to ring-3 */
2984#define IEM_MEMMAP_F_ALIGN_GP RT_BIT_32(16)
2985/** SSE access that should report a \#GP instead of \#AC, unless MXCSR.MM=1
2986 * when it works like normal \#AC. Always used with IEM_MEMMAP_F_ALIGN_GP. */
2987#define IEM_MEMMAP_F_ALIGN_SSE RT_BIT_32(17)
2988/** If \#AC is applicable, raise it. Always used with IEM_MEMMAP_F_ALIGN_GP.
2989 * Users include FXSAVE & FXRSTOR. */
2990#define IEM_MEMMAP_F_ALIGN_GP_OR_AC RT_BIT_32(18)
2991
2992VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
2993 uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT;
2994VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
2995#ifndef IN_RING3
2996VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
2997#endif
2998void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
2999VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT;
3000VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3001VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
3002
3003#ifdef IEM_WITH_CODE_TLB
3004void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT;
3005#else
3006VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT;
3007#endif
3008#ifdef IEM_WITH_SETJMP
3009uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3010uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3011uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3012uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3013#else
3014VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT;
3015VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3016VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3017VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3018VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3019VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3020VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3021VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3022VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3023VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3024VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3025#endif
3026
3027VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3028VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3029VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3030VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3031VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3032VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3033VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3034VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3035VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3036VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3037VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3038VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3039VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
3040 RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT;
3041#ifdef IEM_WITH_SETJMP
3042uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3043uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3044uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3045uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3046uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3047void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3048void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3049void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3050void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3051void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3052void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3053#endif
3054
3055VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3056VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3057VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3058VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3059VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT;
3060
3061VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3062VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3063VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3064VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3065VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3066VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3067VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3068VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3069VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3070#ifdef IEM_WITH_SETJMP
3071void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3072void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3073void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3074void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3075void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3076void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3077void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3078void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3079#endif
3080
3081VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3082 void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3083VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT;
3084VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT;
3085VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3086VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT;
3087VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3088VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3089VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3090VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3091VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3092 void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3093VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3094VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT;
3095VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT;
3096VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT;
3097VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT;
3098VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3099VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3100VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3101/** @} */
3102
3103/** @name IEMAllCImpl.cpp
3104 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/'
3105 * @{ */
3106IEM_CIMPL_PROTO_0(iemCImpl_popa_16);
3107IEM_CIMPL_PROTO_0(iemCImpl_popa_32);
3108IEM_CIMPL_PROTO_0(iemCImpl_pusha_16);
3109IEM_CIMPL_PROTO_0(iemCImpl_pusha_32);
3110IEM_CIMPL_PROTO_1(iemCImpl_pushf, IEMMODE, enmEffOpSize);
3111IEM_CIMPL_PROTO_1(iemCImpl_popf, IEMMODE, enmEffOpSize);
3112IEM_CIMPL_PROTO_1(iemCImpl_call_16, uint16_t, uNewPC);
3113IEM_CIMPL_PROTO_1(iemCImpl_call_rel_16, int16_t, offDisp);
3114IEM_CIMPL_PROTO_1(iemCImpl_call_32, uint32_t, uNewPC);
3115IEM_CIMPL_PROTO_1(iemCImpl_call_rel_32, int32_t, offDisp);
3116IEM_CIMPL_PROTO_1(iemCImpl_call_64, uint64_t, uNewPC);
3117IEM_CIMPL_PROTO_1(iemCImpl_call_rel_64, int64_t, offDisp);
3118IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3119IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3120IEM_CIMPL_PROTO_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3121IEM_CIMPL_PROTO_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3122IEM_CIMPL_PROTO_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3123IEM_CIMPL_PROTO_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3124IEM_CIMPL_PROTO_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3125IEM_CIMPL_PROTO_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3126IEM_CIMPL_PROTO_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters);
3127IEM_CIMPL_PROTO_1(iemCImpl_leave, IEMMODE, enmEffOpSize);
3128IEM_CIMPL_PROTO_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt);
3129IEM_CIMPL_PROTO_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize);
3130IEM_CIMPL_PROTO_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp);
3131IEM_CIMPL_PROTO_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize);
3132IEM_CIMPL_PROTO_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize);
3133IEM_CIMPL_PROTO_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize);
3134IEM_CIMPL_PROTO_1(iemCImpl_iret, IEMMODE, enmEffOpSize);
3135IEM_CIMPL_PROTO_0(iemCImpl_loadall286);
3136IEM_CIMPL_PROTO_0(iemCImpl_syscall);
3137IEM_CIMPL_PROTO_0(iemCImpl_sysret);
3138IEM_CIMPL_PROTO_0(iemCImpl_sysenter);
3139IEM_CIMPL_PROTO_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize);
3140IEM_CIMPL_PROTO_2(iemCImpl_LoadSReg, uint8_t, iSegReg, uint16_t, uSel);
3141IEM_CIMPL_PROTO_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel);
3142IEM_CIMPL_PROTO_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize);
3143IEM_CIMPL_PROTO_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize);
3144IEM_CIMPL_PROTO_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite);
3145IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar);
3146IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar);
3147IEM_CIMPL_PROTO_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3148IEM_CIMPL_PROTO_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3149IEM_CIMPL_PROTO_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3150IEM_CIMPL_PROTO_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3151IEM_CIMPL_PROTO_1(iemCImpl_lldt, uint16_t, uNewLdt);
3152IEM_CIMPL_PROTO_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3153IEM_CIMPL_PROTO_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3154IEM_CIMPL_PROTO_1(iemCImpl_ltr, uint16_t, uNewTr);
3155IEM_CIMPL_PROTO_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3156IEM_CIMPL_PROTO_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3157IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg);
3158IEM_CIMPL_PROTO_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3159IEM_CIMPL_PROTO_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3160IEM_CIMPL_PROTO_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg);
3161IEM_CIMPL_PROTO_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg);
3162IEM_CIMPL_PROTO_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst);
3163IEM_CIMPL_PROTO_0(iemCImpl_clts);
3164IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg);
3165IEM_CIMPL_PROTO_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg);
3166IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg);
3167IEM_CIMPL_PROTO_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg);
3168IEM_CIMPL_PROTO_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage);
3169IEM_CIMPL_PROTO_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType);
3170IEM_CIMPL_PROTO_0(iemCImpl_invd);
3171IEM_CIMPL_PROTO_0(iemCImpl_wbinvd);
3172IEM_CIMPL_PROTO_0(iemCImpl_rsm);
3173IEM_CIMPL_PROTO_0(iemCImpl_rdtsc);
3174IEM_CIMPL_PROTO_0(iemCImpl_rdtscp);
3175IEM_CIMPL_PROTO_0(iemCImpl_rdpmc);
3176IEM_CIMPL_PROTO_0(iemCImpl_rdmsr);
3177IEM_CIMPL_PROTO_0(iemCImpl_wrmsr);
3178IEM_CIMPL_PROTO_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3179IEM_CIMPL_PROTO_1(iemCImpl_in_eAX_DX, uint8_t, cbReg);
3180IEM_CIMPL_PROTO_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3181IEM_CIMPL_PROTO_1(iemCImpl_out_DX_eAX, uint8_t, cbReg);
3182IEM_CIMPL_PROTO_0(iemCImpl_cli);
3183IEM_CIMPL_PROTO_0(iemCImpl_sti);
3184IEM_CIMPL_PROTO_0(iemCImpl_hlt);
3185IEM_CIMPL_PROTO_1(iemCImpl_monitor, uint8_t, iEffSeg);
3186IEM_CIMPL_PROTO_0(iemCImpl_mwait);
3187IEM_CIMPL_PROTO_0(iemCImpl_swapgs);
3188IEM_CIMPL_PROTO_0(iemCImpl_cpuid);
3189IEM_CIMPL_PROTO_1(iemCImpl_aad, uint8_t, bImm);
3190IEM_CIMPL_PROTO_1(iemCImpl_aam, uint8_t, bImm);
3191IEM_CIMPL_PROTO_0(iemCImpl_daa);
3192IEM_CIMPL_PROTO_0(iemCImpl_das);
3193IEM_CIMPL_PROTO_0(iemCImpl_aaa);
3194IEM_CIMPL_PROTO_0(iemCImpl_aas);
3195IEM_CIMPL_PROTO_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound);
3196IEM_CIMPL_PROTO_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound);
3197IEM_CIMPL_PROTO_0(iemCImpl_xgetbv);
3198IEM_CIMPL_PROTO_0(iemCImpl_xsetbv);
3199IEM_CIMPL_PROTO_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
3200 PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags);
3201IEM_CIMPL_PROTO_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3202IEM_CIMPL_PROTO_1(iemCImpl_finit, bool, fCheckXcpts);
3203IEM_CIMPL_PROTO_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3204IEM_CIMPL_PROTO_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3205IEM_CIMPL_PROTO_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3206IEM_CIMPL_PROTO_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3207IEM_CIMPL_PROTO_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3208IEM_CIMPL_PROTO_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3209IEM_CIMPL_PROTO_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3210IEM_CIMPL_PROTO_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3211IEM_CIMPL_PROTO_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3212IEM_CIMPL_PROTO_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3213IEM_CIMPL_PROTO_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3214IEM_CIMPL_PROTO_1(iemCImpl_fldcw, uint16_t, u16Fcw);
3215IEM_CIMPL_PROTO_1(iemCImpl_fxch_underflow, uint8_t, iStReg);
3216IEM_CIMPL_PROTO_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop);
3217/** @} */
3218
3219/** @name IEMAllCImplStrInstr.cpp.h
3220 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/' -e 's/RT_CONCAT4(//' \
3221 * -e 's/,ADDR_SIZE)/64/g' -e 's/,OP_SIZE,/64/g' -e 's/,OP_rAX,/rax/g' IEMAllCImplStrInstr.cpp.h
3222 * @{ */
3223IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr16, uint8_t, iEffSeg);
3224IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr16, uint8_t, iEffSeg);
3225IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m16);
3226IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m16);
3227IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr16, uint8_t, iEffSeg);
3228IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m16);
3229IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m16, int8_t, iEffSeg);
3230IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr16, bool, fIoChecked);
3231IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr16, bool, fIoChecked);
3232IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3233IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3234
3235IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr16, uint8_t, iEffSeg);
3236IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr16, uint8_t, iEffSeg);
3237IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m16);
3238IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m16);
3239IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr16, uint8_t, iEffSeg);
3240IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m16);
3241IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m16, int8_t, iEffSeg);
3242IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr16, bool, fIoChecked);
3243IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr16, bool, fIoChecked);
3244IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3245IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3246
3247IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr16, uint8_t, iEffSeg);
3248IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr16, uint8_t, iEffSeg);
3249IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m16);
3250IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m16);
3251IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr16, uint8_t, iEffSeg);
3252IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m16);
3253IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m16, int8_t, iEffSeg);
3254IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr16, bool, fIoChecked);
3255IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr16, bool, fIoChecked);
3256IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3257IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3258
3259
3260IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr32, uint8_t, iEffSeg);
3261IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr32, uint8_t, iEffSeg);
3262IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m32);
3263IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m32);
3264IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr32, uint8_t, iEffSeg);
3265IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m32);
3266IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m32, int8_t, iEffSeg);
3267IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr32, bool, fIoChecked);
3268IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr32, bool, fIoChecked);
3269IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3270IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3271
3272IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr32, uint8_t, iEffSeg);
3273IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr32, uint8_t, iEffSeg);
3274IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m32);
3275IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m32);
3276IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr32, uint8_t, iEffSeg);
3277IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m32);
3278IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m32, int8_t, iEffSeg);
3279IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr32, bool, fIoChecked);
3280IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr32, bool, fIoChecked);
3281IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3282IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3283
3284IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr32, uint8_t, iEffSeg);
3285IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr32, uint8_t, iEffSeg);
3286IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m32);
3287IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m32);
3288IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr32, uint8_t, iEffSeg);
3289IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m32);
3290IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m32, int8_t, iEffSeg);
3291IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr32, bool, fIoChecked);
3292IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr32, bool, fIoChecked);
3293IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3294IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3295
3296IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr32, uint8_t, iEffSeg);
3297IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr32, uint8_t, iEffSeg);
3298IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m32);
3299IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m32);
3300IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr32, uint8_t, iEffSeg);
3301IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m32);
3302IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m32, int8_t, iEffSeg);
3303IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr32, bool, fIoChecked);
3304IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr32, bool, fIoChecked);
3305IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3306IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3307
3308
3309IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr64, uint8_t, iEffSeg);
3310IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr64, uint8_t, iEffSeg);
3311IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m64);
3312IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m64);
3313IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr64, uint8_t, iEffSeg);
3314IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m64);
3315IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m64, int8_t, iEffSeg);
3316IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr64, bool, fIoChecked);
3317IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr64, bool, fIoChecked);
3318IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3319IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3320
3321IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr64, uint8_t, iEffSeg);
3322IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr64, uint8_t, iEffSeg);
3323IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m64);
3324IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m64);
3325IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr64, uint8_t, iEffSeg);
3326IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m64);
3327IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m64, int8_t, iEffSeg);
3328IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr64, bool, fIoChecked);
3329IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr64, bool, fIoChecked);
3330IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3331IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3332
3333IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr64, uint8_t, iEffSeg);
3334IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr64, uint8_t, iEffSeg);
3335IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m64);
3336IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m64);
3337IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr64, uint8_t, iEffSeg);
3338IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m64);
3339IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m64, int8_t, iEffSeg);
3340IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr64, bool, fIoChecked);
3341IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr64, bool, fIoChecked);
3342IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3343IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3344
3345IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr64, uint8_t, iEffSeg);
3346IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr64, uint8_t, iEffSeg);
3347IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m64);
3348IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m64);
3349IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr64, uint8_t, iEffSeg);
3350IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m64);
3351IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m64, int8_t, iEffSeg);
3352IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr64, bool, fIoChecked);
3353IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr64, bool, fIoChecked);
3354IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3355IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3356/** @} */
3357
3358#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3359VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT;
3360VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT;
3361VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT;
3362VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT;
3363VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT;
3364VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT;
3365VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT;
3366VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT;
3367VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT;
3368VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port,
3369 bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT;
3370VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess,
3371 bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT;
3372VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3373VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3374VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3375VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3376VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3377VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3378VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT;
3379VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw,
3380 RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT;
3381VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT;
3382VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT;
3383VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT;
3384uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT;
3385void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT;
3386VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
3387 uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT;
3388bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT;
3389IEM_CIMPL_PROTO_0(iemCImpl_vmxoff);
3390IEM_CIMPL_PROTO_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon);
3391IEM_CIMPL_PROTO_0(iemCImpl_vmlaunch);
3392IEM_CIMPL_PROTO_0(iemCImpl_vmresume);
3393IEM_CIMPL_PROTO_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3394IEM_CIMPL_PROTO_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3395IEM_CIMPL_PROTO_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3396IEM_CIMPL_PROTO_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField);
3397IEM_CIMPL_PROTO_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField);
3398IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField);
3399IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField);
3400IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField);
3401IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField);
3402IEM_CIMPL_PROTO_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType);
3403IEM_CIMPL_PROTO_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType);
3404IEM_CIMPL_PROTO_0(iemCImpl_vmx_pause);
3405#endif
3406
3407#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3408VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT;
3409VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3410VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
3411 uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT;
3412VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT;
3413IEM_CIMPL_PROTO_0(iemCImpl_vmrun);
3414IEM_CIMPL_PROTO_0(iemCImpl_vmload);
3415IEM_CIMPL_PROTO_0(iemCImpl_vmsave);
3416IEM_CIMPL_PROTO_0(iemCImpl_clgi);
3417IEM_CIMPL_PROTO_0(iemCImpl_stgi);
3418IEM_CIMPL_PROTO_0(iemCImpl_invlpga);
3419IEM_CIMPL_PROTO_0(iemCImpl_skinit);
3420IEM_CIMPL_PROTO_0(iemCImpl_svm_pause);
3421#endif
3422
3423IEM_CIMPL_PROTO_0(iemCImpl_vmcall); /* vmx */
3424IEM_CIMPL_PROTO_0(iemCImpl_vmmcall); /* svm */
3425IEM_CIMPL_PROTO_1(iemCImpl_Hypercall, uint16_t, uDisOpcode); /* both */
3426
3427
3428extern const PFNIEMOP g_apfnOneByteMap[256];
3429
3430/** @} */
3431
3432RT_C_DECLS_END
3433
3434#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3435
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette