VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 96335

Last change on this file since 96335 was 96335, checked in by vboxsync, 2 years ago

VMM/IEM: Implement maxps/maxpd instructions, bugref:9898
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 191.9 KB
Line 
1/* $Id: IEMInternal.h 96335 2022-08-19 12:07:40Z vboxsync $ */
2/** @file
3 * IEM - Internal header file.
4 */
5
6/*
7 * Copyright (C) 2011-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
19#define VMM_INCLUDED_SRC_include_IEMInternal_h
20#ifndef RT_WITHOUT_PRAGMA_ONCE
21# pragma once
22#endif
23
24#include <VBox/vmm/cpum.h>
25#include <VBox/vmm/iem.h>
26#include <VBox/vmm/pgm.h>
27#include <VBox/vmm/stam.h>
28#include <VBox/param.h>
29
30#include <iprt/setjmp-without-sigmask.h>
31
32
33RT_C_DECLS_BEGIN
34
35
36/** @defgroup grp_iem_int Internals
37 * @ingroup grp_iem
38 * @internal
39 * @{
40 */
41
42/** For expanding symbol in slickedit and other products tagging and
43 * crossreferencing IEM symbols. */
44#ifndef IEM_STATIC
45# define IEM_STATIC static
46#endif
47
48/** @def IEM_WITH_SETJMP
49 * Enables alternative status code handling using setjmps.
50 *
51 * This adds a bit of expense via the setjmp() call since it saves all the
52 * non-volatile registers. However, it eliminates return code checks and allows
53 * for more optimal return value passing (return regs instead of stack buffer).
54 */
55#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
56# define IEM_WITH_SETJMP
57#endif
58
59#define IEM_IMPLEMENTS_TASKSWITCH
60
61/** @def IEM_WITH_3DNOW
62 * Includes the 3DNow decoding. */
63#define IEM_WITH_3DNOW
64
65/** @def IEM_WITH_THREE_0F_38
66 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
67#define IEM_WITH_THREE_0F_38
68
69/** @def IEM_WITH_THREE_0F_3A
70 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
71#define IEM_WITH_THREE_0F_3A
72
73/** @def IEM_WITH_VEX
74 * Includes the VEX decoding. */
75#define IEM_WITH_VEX
76
77/** @def IEM_CFG_TARGET_CPU
78 * The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
79 *
80 * By default we allow this to be configured by the user via the
81 * CPUM/GuestCpuName config string, but this comes at a slight cost during
82 * decoding. So, for applications of this code where there is no need to
83 * be dynamic wrt target CPU, just modify this define.
84 */
85#if !defined(IEM_CFG_TARGET_CPU) || defined(DOXYGEN_RUNNING)
86# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
87#endif
88
89//#define IEM_WITH_CODE_TLB // - work in progress
90//#define IEM_WITH_DATA_TLB // - work in progress
91
92
93/** @def IEM_USE_UNALIGNED_DATA_ACCESS
94 * Use unaligned accesses instead of elaborate byte assembly. */
95#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
96# define IEM_USE_UNALIGNED_DATA_ACCESS
97#endif
98
99//#define IEM_LOG_MEMORY_WRITES
100
101#if !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
102/** Instruction statistics. */
103typedef struct IEMINSTRSTATS
104{
105# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
106# include "IEMInstructionStatisticsTmpl.h"
107# undef IEM_DO_INSTR_STAT
108} IEMINSTRSTATS;
109#else
110struct IEMINSTRSTATS;
111typedef struct IEMINSTRSTATS IEMINSTRSTATS;
112#endif
113/** Pointer to IEM instruction statistics. */
114typedef IEMINSTRSTATS *PIEMINSTRSTATS;
115
116
117/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
118 * @{ */
119#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /**< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. */
120#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /**< Intel EFLAGS result. */
121#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /**< AMD EFLAGS result */
122#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /**< Reserved/dummy entry slot that's the same as 0. */
123#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /**< For masking the index before use. */
124/** Selects the right variant from a_aArray.
125 * pVCpu is implicit in the caller context. */
126#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
127 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
128/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
129 * be used because the host CPU does not support the operation. */
130#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
131 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
132/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
133 * array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
134 * into the two.
135 * @sa IEM_SELECT_NATIVE_OR_FALLBACK */
136#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
137# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
138 (a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
139#else
140# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
141 (a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
142#endif
143/** @} */
144
145/**
146 * Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
147 * indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
148 *
149 * On non-x86 hosts, this will shortcut to the fallback w/o checking the
150 * indicator.
151 *
152 * @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
153 */
154#if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
155# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
156 (g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
157#else
158# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
159#endif
160
161
162/**
163 * Extended operand mode that includes a representation of 8-bit.
164 *
165 * This is used for packing down modes when invoking some C instruction
166 * implementations.
167 */
168typedef enum IEMMODEX
169{
170 IEMMODEX_16BIT = IEMMODE_16BIT,
171 IEMMODEX_32BIT = IEMMODE_32BIT,
172 IEMMODEX_64BIT = IEMMODE_64BIT,
173 IEMMODEX_8BIT
174} IEMMODEX;
175AssertCompileSize(IEMMODEX, 4);
176
177
178/**
179 * Branch types.
180 */
181typedef enum IEMBRANCH
182{
183 IEMBRANCH_JUMP = 1,
184 IEMBRANCH_CALL,
185 IEMBRANCH_TRAP,
186 IEMBRANCH_SOFTWARE_INT,
187 IEMBRANCH_HARDWARE_INT
188} IEMBRANCH;
189AssertCompileSize(IEMBRANCH, 4);
190
191
192/**
193 * INT instruction types.
194 */
195typedef enum IEMINT
196{
197 /** INT n instruction (opcode 0xcd imm). */
198 IEMINT_INTN = 0,
199 /** Single byte INT3 instruction (opcode 0xcc). */
200 IEMINT_INT3 = IEM_XCPT_FLAGS_BP_INSTR,
201 /** Single byte INTO instruction (opcode 0xce). */
202 IEMINT_INTO = IEM_XCPT_FLAGS_OF_INSTR,
203 /** Single byte INT1 (ICEBP) instruction (opcode 0xf1). */
204 IEMINT_INT1 = IEM_XCPT_FLAGS_ICEBP_INSTR
205} IEMINT;
206AssertCompileSize(IEMINT, 4);
207
208
209/**
210 * A FPU result.
211 */
212typedef struct IEMFPURESULT
213{
214 /** The output value. */
215 RTFLOAT80U r80Result;
216 /** The output status. */
217 uint16_t FSW;
218} IEMFPURESULT;
219AssertCompileMemberOffset(IEMFPURESULT, FSW, 10);
220/** Pointer to a FPU result. */
221typedef IEMFPURESULT *PIEMFPURESULT;
222/** Pointer to a const FPU result. */
223typedef IEMFPURESULT const *PCIEMFPURESULT;
224
225
226/**
227 * A FPU result consisting of two output values and FSW.
228 */
229typedef struct IEMFPURESULTTWO
230{
231 /** The first output value. */
232 RTFLOAT80U r80Result1;
233 /** The output status. */
234 uint16_t FSW;
235 /** The second output value. */
236 RTFLOAT80U r80Result2;
237} IEMFPURESULTTWO;
238AssertCompileMemberOffset(IEMFPURESULTTWO, FSW, 10);
239AssertCompileMemberOffset(IEMFPURESULTTWO, r80Result2, 12);
240/** Pointer to a FPU result consisting of two output values and FSW. */
241typedef IEMFPURESULTTWO *PIEMFPURESULTTWO;
242/** Pointer to a const FPU result consisting of two output values and FSW. */
243typedef IEMFPURESULTTWO const *PCIEMFPURESULTTWO;
244
245
246/**
247 * IEM TLB entry.
248 *
249 * Lookup assembly:
250 * @code{.asm}
251 ; Calculate tag.
252 mov rax, [VA]
253 shl rax, 16
254 shr rax, 16 + X86_PAGE_SHIFT
255 or rax, [uTlbRevision]
256
257 ; Do indexing.
258 movzx ecx, al
259 lea rcx, [pTlbEntries + rcx]
260
261 ; Check tag.
262 cmp [rcx + IEMTLBENTRY.uTag], rax
263 jne .TlbMiss
264
265 ; Check access.
266 mov rax, ACCESS_FLAGS | MAPPING_R3_NOT_VALID | 0xffffff00
267 and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
268 cmp rax, [uTlbPhysRev]
269 jne .TlbMiss
270
271 ; Calc address and we're done.
272 mov eax, X86_PAGE_OFFSET_MASK
273 and eax, [VA]
274 or rax, [rcx + IEMTLBENTRY.pMappingR3]
275 %ifdef VBOX_WITH_STATISTICS
276 inc qword [cTlbHits]
277 %endif
278 jmp .Done
279
280 .TlbMiss:
281 mov r8d, ACCESS_FLAGS
282 mov rdx, [VA]
283 mov rcx, [pVCpu]
284 call iemTlbTypeMiss
285 .Done:
286
287 @endcode
288 *
289 */
290typedef struct IEMTLBENTRY
291{
292 /** The TLB entry tag.
293 * Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
294 * is ASSUMING a virtual address width of 48 bits.
295 *
296 * Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
297 *
298 * The TLB lookup code uses the current TLB revision, which won't ever be zero,
299 * enabling an extremely cheap TLB invalidation most of the time. When the TLB
300 * revision wraps around though, the tags needs to be zeroed.
301 *
302 * @note Try use SHRD instruction? After seeing
303 * https://gmplib.org/~tege/x86-timing.pdf, maybe not.
304 *
305 * @todo This will need to be reorganized for 57-bit wide virtual address and
306 * PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
307 * have to move the TLB entry versioning entirely to the
308 * fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
309 * 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
310 * consumed by PCID and ASID (12 + 6 = 18).
311 */
312 uint64_t uTag;
313 /** Access flags and physical TLB revision.
314 *
315 * - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
316 * - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
317 * - Bit 2 - page tables - not user (complemented X86_PTE_US).
318 * - Bit 3 - pgm phys/virt - not directly writable.
319 * - Bit 4 - pgm phys page - not directly readable.
320 * - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
321 * - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
322 * - Bit 7 - tlb entry - pMappingR3 member not valid.
323 * - Bits 63 thru 8 are used for the physical TLB revision number.
324 *
325 * We're using complemented bit meanings here because it makes it easy to check
326 * whether special action is required. For instance a user mode write access
327 * would do a "TEST fFlags, (X86_PTE_RW | X86_PTE_US | X86_PTE_D)" and a
328 * non-zero result would mean special handling needed because either it wasn't
329 * writable, or it wasn't user, or the page wasn't dirty. A user mode read
330 * access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
331 * need to check any PTE flag.
332 */
333 uint64_t fFlagsAndPhysRev;
334 /** The guest physical page address. */
335 uint64_t GCPhys;
336 /** Pointer to the ring-3 mapping. */
337 R3PTRTYPE(uint8_t *) pbMappingR3;
338#if HC_ARCH_BITS == 32
339 uint32_t u32Padding1;
340#endif
341} IEMTLBENTRY;
342AssertCompileSize(IEMTLBENTRY, 32);
343/** Pointer to an IEM TLB entry. */
344typedef IEMTLBENTRY *PIEMTLBENTRY;
345
346/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
347 * @{ */
348#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /**< Page tables: Not executable. */
349#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /**< Page tables: Not writable. */
350#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /**< Page tables: Not user accessible (supervisor only). */
351#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /**< Phys page: Not writable (access handler, ROM, whatever). */
352#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /**< Phys page: Not readable (MMIO / access handler, ROM) */
353#define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /**< Phys tables: Not accessed (need to be marked accessed). */
354#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /**< Page tables: Not dirty (needs to be made dirty on write). */
355#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(7) /**< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. */
356#define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(8) /**< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). */
357#define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffffe00) /**< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR */
358/** @} */
359
360
361/**
362 * An IEM TLB.
363 *
364 * We've got two of these, one for data and one for instructions.
365 */
366typedef struct IEMTLB
367{
368 /** The TLB entries.
369 * We've choosen 256 because that way we can obtain the result directly from a
370 * 8-bit register without an additional AND instruction. */
371 IEMTLBENTRY aEntries[256];
372 /** The TLB revision.
373 * This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
374 * by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
375 * the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
376 * (The revision zero indicates an invalid TLB entry.)
377 *
378 * The initial value is choosen to cause an early wraparound. */
379 uint64_t uTlbRevision;
380 /** The TLB physical address revision - shadow of PGM variable.
381 *
382 * This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
383 * incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
384 * a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
385 * as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
386 *
387 * The initial value is choosen to cause an early wraparound. */
388 uint64_t volatile uTlbPhysRev;
389
390 /* Statistics: */
391
392 /** TLB hits (VBOX_WITH_STATISTICS only). */
393 uint64_t cTlbHits;
394 /** TLB misses. */
395 uint32_t cTlbMisses;
396 /** Slow read path. */
397 uint32_t cTlbSlowReadPath;
398#if 0
399 /** TLB misses because of tag mismatch. */
400 uint32_t cTlbMissesTag;
401 /** TLB misses because of virtual access violation. */
402 uint32_t cTlbMissesVirtAccess;
403 /** TLB misses because of dirty bit. */
404 uint32_t cTlbMissesDirty;
405 /** TLB misses because of MMIO */
406 uint32_t cTlbMissesMmio;
407 /** TLB misses because of write access handlers. */
408 uint32_t cTlbMissesWriteHandler;
409 /** TLB misses because no r3(/r0) mapping. */
410 uint32_t cTlbMissesMapping;
411#endif
412 /** Alignment padding. */
413 uint32_t au32Padding[3+5];
414} IEMTLB;
415AssertCompileSizeAlignment(IEMTLB, 64);
416/** IEMTLB::uTlbRevision increment. */
417#define IEMTLB_REVISION_INCR RT_BIT_64(36)
418/** IEMTLB::uTlbRevision mask. */
419#define IEMTLB_REVISION_MASK (~(RT_BIT_64(36) - 1))
420/** IEMTLB::uTlbPhysRev increment.
421 * @sa IEMTLBE_F_PHYS_REV */
422#define IEMTLB_PHYS_REV_INCR RT_BIT_64(9)
423/**
424 * Calculates the TLB tag for a virtual address.
425 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
426 * @param a_pTlb The TLB.
427 * @param a_GCPtr The virtual address.
428 */
429#define IEMTLB_CALC_TAG(a_pTlb, a_GCPtr) ( IEMTLB_CALC_TAG_NO_REV(a_GCPtr) | (a_pTlb)->uTlbRevision )
430/**
431 * Calculates the TLB tag for a virtual address but without TLB revision.
432 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
433 * @param a_GCPtr The virtual address.
434 */
435#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
436/**
437 * Converts a TLB tag value into a TLB index.
438 * @returns Index into IEMTLB::aEntries.
439 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
440 */
441#define IEMTLB_TAG_TO_INDEX(a_uTag) ( (uint8_t)(a_uTag) )
442/**
443 * Converts a TLB tag value into a TLB index.
444 * @returns Index into IEMTLB::aEntries.
445 * @param a_pTlb The TLB.
446 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
447 */
448#define IEMTLB_TAG_TO_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_INDEX(a_uTag)] )
449
450
451/**
452 * The per-CPU IEM state.
453 */
454typedef struct IEMCPU
455{
456 /** Info status code that needs to be propagated to the IEM caller.
457 * This cannot be passed internally, as it would complicate all success
458 * checks within the interpreter making the code larger and almost impossible
459 * to get right. Instead, we'll store status codes to pass on here. Each
460 * source of these codes will perform appropriate sanity checks. */
461 int32_t rcPassUp; /* 0x00 */
462
463 /** The current CPU execution mode (CS). */
464 IEMMODE enmCpuMode; /* 0x04 */
465 /** The CPL. */
466 uint8_t uCpl; /* 0x05 */
467
468 /** Whether to bypass access handlers or not. */
469 bool fBypassHandlers; /* 0x06 */
470 /** Whether to disregard the lock prefix (implied or not). */
471 bool fDisregardLock; /* 0x07 */
472
473 /** @name Decoder state.
474 * @{ */
475#ifdef IEM_WITH_CODE_TLB
476 /** The offset of the next instruction byte. */
477 uint32_t offInstrNextByte; /* 0x08 */
478 /** The number of bytes available at pbInstrBuf for the current instruction.
479 * This takes the max opcode length into account so that doesn't need to be
480 * checked separately. */
481 uint32_t cbInstrBuf; /* 0x0c */
482 /** Pointer to the page containing RIP, user specified buffer or abOpcode.
483 * This can be NULL if the page isn't mappable for some reason, in which
484 * case we'll do fallback stuff.
485 *
486 * If we're executing an instruction from a user specified buffer,
487 * IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
488 * aligned pointer but pointer to the user data.
489 *
490 * For instructions crossing pages, this will start on the first page and be
491 * advanced to the next page by the time we've decoded the instruction. This
492 * therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
493 */
494 uint8_t const *pbInstrBuf; /* 0x10 */
495# if ARCH_BITS == 32
496 uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
497# endif
498 /** The program counter corresponding to pbInstrBuf.
499 * This is set to a non-canonical address when we need to invalidate it. */
500 uint64_t uInstrBufPc; /* 0x18 */
501 /** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
502 * This takes the CS segment limit into account. */
503 uint16_t cbInstrBufTotal; /* 0x20 */
504 /** Offset into pbInstrBuf of the first byte of the current instruction.
505 * Can be negative to efficiently handle cross page instructions. */
506 int16_t offCurInstrStart; /* 0x22 */
507
508 /** The prefix mask (IEM_OP_PRF_XXX). */
509 uint32_t fPrefixes; /* 0x24 */
510 /** The extra REX ModR/M register field bit (REX.R << 3). */
511 uint8_t uRexReg; /* 0x28 */
512 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
513 * (REX.B << 3). */
514 uint8_t uRexB; /* 0x29 */
515 /** The extra REX SIB index field bit (REX.X << 3). */
516 uint8_t uRexIndex; /* 0x2a */
517
518 /** The effective segment register (X86_SREG_XXX). */
519 uint8_t iEffSeg; /* 0x2b */
520
521 /** The offset of the ModR/M byte relative to the start of the instruction. */
522 uint8_t offModRm; /* 0x2c */
523#else
524 /** The size of what has currently been fetched into abOpcode. */
525 uint8_t cbOpcode; /* 0x08 */
526 /** The current offset into abOpcode. */
527 uint8_t offOpcode; /* 0x09 */
528 /** The offset of the ModR/M byte relative to the start of the instruction. */
529 uint8_t offModRm; /* 0x0a */
530
531 /** The effective segment register (X86_SREG_XXX). */
532 uint8_t iEffSeg; /* 0x0b */
533
534 /** The prefix mask (IEM_OP_PRF_XXX). */
535 uint32_t fPrefixes; /* 0x0c */
536 /** The extra REX ModR/M register field bit (REX.R << 3). */
537 uint8_t uRexReg; /* 0x10 */
538 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
539 * (REX.B << 3). */
540 uint8_t uRexB; /* 0x11 */
541 /** The extra REX SIB index field bit (REX.X << 3). */
542 uint8_t uRexIndex; /* 0x12 */
543
544#endif
545
546 /** The effective operand mode. */
547 IEMMODE enmEffOpSize; /* 0x2d, 0x13 */
548 /** The default addressing mode. */
549 IEMMODE enmDefAddrMode; /* 0x2e, 0x14 */
550 /** The effective addressing mode. */
551 IEMMODE enmEffAddrMode; /* 0x2f, 0x15 */
552 /** The default operand mode. */
553 IEMMODE enmDefOpSize; /* 0x30, 0x16 */
554
555 /** Prefix index (VEX.pp) for two byte and three byte tables. */
556 uint8_t idxPrefix; /* 0x31, 0x17 */
557 /** 3rd VEX/EVEX/XOP register.
558 * Please use IEM_GET_EFFECTIVE_VVVV to access. */
559 uint8_t uVex3rdReg; /* 0x32, 0x18 */
560 /** The VEX/EVEX/XOP length field. */
561 uint8_t uVexLength; /* 0x33, 0x19 */
562 /** Additional EVEX stuff. */
563 uint8_t fEvexStuff; /* 0x34, 0x1a */
564
565 /** Explicit alignment padding. */
566 uint8_t abAlignment2a[1]; /* 0x35, 0x1b */
567 /** The FPU opcode (FOP). */
568 uint16_t uFpuOpcode; /* 0x36, 0x1c */
569#ifndef IEM_WITH_CODE_TLB
570 /** Explicit alignment padding. */
571 uint8_t abAlignment2b[2]; /* 0x1e */
572#endif
573
574 /** The opcode bytes. */
575 uint8_t abOpcode[15]; /* 0x48, 0x20 */
576 /** Explicit alignment padding. */
577#ifdef IEM_WITH_CODE_TLB
578 uint8_t abAlignment2c[0x48 - 0x47]; /* 0x37 */
579#else
580 uint8_t abAlignment2c[0x48 - 0x2f]; /* 0x2f */
581#endif
582 /** @} */
583
584
585 /** The flags of the current exception / interrupt. */
586 uint32_t fCurXcpt; /* 0x48, 0x48 */
587 /** The current exception / interrupt. */
588 uint8_t uCurXcpt;
589 /** Exception / interrupt recursion depth. */
590 int8_t cXcptRecursions;
591
592 /** The number of active guest memory mappings. */
593 uint8_t cActiveMappings;
594 /** The next unused mapping index. */
595 uint8_t iNextMapping;
596 /** Records for tracking guest memory mappings. */
597 struct
598 {
599 /** The address of the mapped bytes. */
600 void *pv;
601 /** The access flags (IEM_ACCESS_XXX).
602 * IEM_ACCESS_INVALID if the entry is unused. */
603 uint32_t fAccess;
604#if HC_ARCH_BITS == 64
605 uint32_t u32Alignment4; /**< Alignment padding. */
606#endif
607 } aMemMappings[3];
608
609 /** Locking records for the mapped memory. */
610 union
611 {
612 PGMPAGEMAPLOCK Lock;
613 uint64_t au64Padding[2];
614 } aMemMappingLocks[3];
615
616 /** Bounce buffer info.
617 * This runs in parallel to aMemMappings. */
618 struct
619 {
620 /** The physical address of the first byte. */
621 RTGCPHYS GCPhysFirst;
622 /** The physical address of the second page. */
623 RTGCPHYS GCPhysSecond;
624 /** The number of bytes in the first page. */
625 uint16_t cbFirst;
626 /** The number of bytes in the second page. */
627 uint16_t cbSecond;
628 /** Whether it's unassigned memory. */
629 bool fUnassigned;
630 /** Explicit alignment padding. */
631 bool afAlignment5[3];
632 } aMemBbMappings[3];
633
634 /** Bounce buffer storage.
635 * This runs in parallel to aMemMappings and aMemBbMappings. */
636 struct
637 {
638 uint8_t ab[512];
639 } aBounceBuffers[3];
640
641
642 /** Pointer set jump buffer - ring-3 context. */
643 R3PTRTYPE(jmp_buf *) pJmpBufR3;
644 /** Pointer set jump buffer - ring-0 context. */
645 R0PTRTYPE(jmp_buf *) pJmpBufR0;
646
647 /** @todo Should move this near @a fCurXcpt later. */
648 /** The CR2 for the current exception / interrupt. */
649 uint64_t uCurXcptCr2;
650 /** The error code for the current exception / interrupt. */
651 uint32_t uCurXcptErr;
652
653 /** @name Statistics
654 * @{ */
655 /** The number of instructions we've executed. */
656 uint32_t cInstructions;
657 /** The number of potential exits. */
658 uint32_t cPotentialExits;
659 /** The number of bytes data or stack written (mostly for IEMExecOneEx).
660 * This may contain uncommitted writes. */
661 uint32_t cbWritten;
662 /** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
663 uint32_t cRetInstrNotImplemented;
664 /** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
665 uint32_t cRetAspectNotImplemented;
666 /** Counts informational statuses returned (other than VINF_SUCCESS). */
667 uint32_t cRetInfStatuses;
668 /** Counts other error statuses returned. */
669 uint32_t cRetErrStatuses;
670 /** Number of times rcPassUp has been used. */
671 uint32_t cRetPassUpStatus;
672 /** Number of times RZ left with instruction commit pending for ring-3. */
673 uint32_t cPendingCommit;
674 /** Number of long jumps. */
675 uint32_t cLongJumps;
676 /** @} */
677
678 /** @name Target CPU information.
679 * @{ */
680#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
681 /** The target CPU. */
682 uint8_t uTargetCpu;
683#else
684 uint8_t bTargetCpuPadding;
685#endif
686 /** For selecting assembly works matching the target CPU EFLAGS behaviour, see
687 * IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
688 * native host support and the 2nd for when there is.
689 *
690 * The two values are typically indexed by a g_CpumHostFeatures bit.
691 *
692 * This is for instance used for the BSF & BSR instructions where AMD and
693 * Intel CPUs produce different EFLAGS. */
694 uint8_t aidxTargetCpuEflFlavour[2];
695
696 /** The CPU vendor. */
697 CPUMCPUVENDOR enmCpuVendor;
698 /** @} */
699
700 /** @name Host CPU information.
701 * @{ */
702 /** The CPU vendor. */
703 CPUMCPUVENDOR enmHostCpuVendor;
704 /** @} */
705
706 /** Counts RDMSR \#GP(0) LogRel(). */
707 uint8_t cLogRelRdMsr;
708 /** Counts WRMSR \#GP(0) LogRel(). */
709 uint8_t cLogRelWrMsr;
710 /** Alignment padding. */
711 uint8_t abAlignment8[50];
712
713 /** Data TLB.
714 * @remarks Must be 64-byte aligned. */
715 IEMTLB DataTlb;
716 /** Instruction TLB.
717 * @remarks Must be 64-byte aligned. */
718 IEMTLB CodeTlb;
719
720 /** Exception statistics. */
721 STAMCOUNTER aStatXcpts[32];
722 /** Interrupt statistics. */
723 uint32_t aStatInts[256];
724
725#if defined(VBOX_WITH_STATISTICS) && !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
726 /** Instruction statistics for ring-0/raw-mode. */
727 IEMINSTRSTATS StatsRZ;
728 /** Instruction statistics for ring-3. */
729 IEMINSTRSTATS StatsR3;
730#endif
731} IEMCPU;
732AssertCompileMemberOffset(IEMCPU, fCurXcpt, 0x48);
733AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
734AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
735/** Pointer to the per-CPU IEM state. */
736typedef IEMCPU *PIEMCPU;
737/** Pointer to the const per-CPU IEM state. */
738typedef IEMCPU const *PCIEMCPU;
739
740
741/** @def IEM_GET_CTX
742 * Gets the guest CPU context for the calling EMT.
743 * @returns PCPUMCTX
744 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
745 */
746#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
747
748/** @def IEM_CTX_ASSERT
749 * Asserts that the @a a_fExtrnMbz is present in the CPU context.
750 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
751 * @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
752 */
753#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
754 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (a_pVCpu)->cpum.GstCtx.fExtrn, \
755 (a_fExtrnMbz)))
756
757/** @def IEM_CTX_IMPORT_RET
758 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
759 *
760 * Will call the keep to import the bits as needed.
761 *
762 * Returns on import failure.
763 *
764 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
765 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
766 */
767#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
768 do { \
769 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
770 { /* likely */ } \
771 else \
772 { \
773 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
774 AssertRCReturn(rcCtxImport, rcCtxImport); \
775 } \
776 } while (0)
777
778/** @def IEM_CTX_IMPORT_NORET
779 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
780 *
781 * Will call the keep to import the bits as needed.
782 *
783 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
784 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
785 */
786#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
787 do { \
788 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
789 { /* likely */ } \
790 else \
791 { \
792 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
793 AssertLogRelRC(rcCtxImport); \
794 } \
795 } while (0)
796
797/** @def IEM_CTX_IMPORT_JMP
798 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
799 *
800 * Will call the keep to import the bits as needed.
801 *
802 * Jumps on import failure.
803 *
804 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
805 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
806 */
807#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
808 do { \
809 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
810 { /* likely */ } \
811 else \
812 { \
813 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
814 AssertRCStmt(rcCtxImport, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), rcCtxImport)); \
815 } \
816 } while (0)
817
818
819
820/** @def IEM_GET_TARGET_CPU
821 * Gets the current IEMTARGETCPU value.
822 * @returns IEMTARGETCPU value.
823 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
824 */
825#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
826# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
827#else
828# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
829#endif
830
831/** @def IEM_GET_INSTR_LEN
832 * Gets the instruction length. */
833#ifdef IEM_WITH_CODE_TLB
834# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
835#else
836# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
837#endif
838
839
840/**
841 * Shared per-VM IEM data.
842 */
843typedef struct IEM
844{
845 /** The VMX APIC-access page handler type. */
846 PGMPHYSHANDLERTYPE hVmxApicAccessPage;
847} IEM;
848
849
850
851/** @name IEM_ACCESS_XXX - Access details.
852 * @{ */
853#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
854#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
855#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
856#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
857#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
858#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
859#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
860#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
861#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
862#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
863/** The writes are partial, so if initialize the bounce buffer with the
864 * orignal RAM content. */
865#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
866/** Used in aMemMappings to indicate that the entry is bounce buffered. */
867#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
868/** Bounce buffer with ring-3 write pending, first page. */
869#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
870/** Bounce buffer with ring-3 write pending, second page. */
871#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
872/** Not locked, accessed via the TLB. */
873#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
874/** Valid bit mask. */
875#define IEM_ACCESS_VALID_MASK UINT32_C(0x00001fff)
876/** Shift count for the TLB flags (upper word). */
877#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
878
879/** Read+write data alias. */
880#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
881/** Write data alias. */
882#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
883/** Read data alias. */
884#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA)
885/** Instruction fetch alias. */
886#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_WHAT_CODE)
887/** Stack write alias. */
888#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
889/** Stack read alias. */
890#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_STACK)
891/** Stack read+write alias. */
892#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
893/** Read system table alias. */
894#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_SYS)
895/** Read+write system table alias. */
896#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_SYS)
897/** @} */
898
899/** @name Prefix constants (IEMCPU::fPrefixes)
900 * @{ */
901#define IEM_OP_PRF_SEG_CS RT_BIT_32(0) /**< CS segment prefix (0x2e). */
902#define IEM_OP_PRF_SEG_SS RT_BIT_32(1) /**< SS segment prefix (0x36). */
903#define IEM_OP_PRF_SEG_DS RT_BIT_32(2) /**< DS segment prefix (0x3e). */
904#define IEM_OP_PRF_SEG_ES RT_BIT_32(3) /**< ES segment prefix (0x26). */
905#define IEM_OP_PRF_SEG_FS RT_BIT_32(4) /**< FS segment prefix (0x64). */
906#define IEM_OP_PRF_SEG_GS RT_BIT_32(5) /**< GS segment prefix (0x65). */
907#define IEM_OP_PRF_SEG_MASK UINT32_C(0x3f)
908
909#define IEM_OP_PRF_SIZE_OP RT_BIT_32(8) /**< Operand size prefix (0x66). */
910#define IEM_OP_PRF_SIZE_REX_W RT_BIT_32(9) /**< REX.W prefix (0x48-0x4f). */
911#define IEM_OP_PRF_SIZE_ADDR RT_BIT_32(10) /**< Address size prefix (0x67). */
912
913#define IEM_OP_PRF_LOCK RT_BIT_32(16) /**< Lock prefix (0xf0). */
914#define IEM_OP_PRF_REPNZ RT_BIT_32(17) /**< Repeat-not-zero prefix (0xf2). */
915#define IEM_OP_PRF_REPZ RT_BIT_32(18) /**< Repeat-if-zero prefix (0xf3). */
916
917#define IEM_OP_PRF_REX RT_BIT_32(24) /**< Any REX prefix (0x40-0x4f). */
918#define IEM_OP_PRF_REX_R RT_BIT_32(25) /**< REX.R prefix (0x44,0x45,0x46,0x47,0x4c,0x4d,0x4e,0x4f). */
919#define IEM_OP_PRF_REX_B RT_BIT_32(26) /**< REX.B prefix (0x41,0x43,0x45,0x47,0x49,0x4b,0x4d,0x4f). */
920#define IEM_OP_PRF_REX_X RT_BIT_32(27) /**< REX.X prefix (0x42,0x43,0x46,0x47,0x4a,0x4b,0x4e,0x4f). */
921/** Mask with all the REX prefix flags.
922 * This is generally for use when needing to undo the REX prefixes when they
923 * are followed legacy prefixes and therefore does not immediately preceed
924 * the first opcode byte.
925 * For testing whether any REX prefix is present, use IEM_OP_PRF_REX instead. */
926#define IEM_OP_PRF_REX_MASK (IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W )
927
928#define IEM_OP_PRF_VEX RT_BIT_32(28) /**< Indiciates VEX prefix. */
929#define IEM_OP_PRF_EVEX RT_BIT_32(29) /**< Indiciates EVEX prefix. */
930#define IEM_OP_PRF_XOP RT_BIT_32(30) /**< Indiciates XOP prefix. */
931/** @} */
932
933/** @name IEMOPFORM_XXX - Opcode forms
934 * @note These are ORed together with IEMOPHINT_XXX.
935 * @{ */
936/** ModR/M: reg, r/m */
937#define IEMOPFORM_RM 0
938/** ModR/M: reg, r/m (register) */
939#define IEMOPFORM_RM_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
940/** ModR/M: reg, r/m (memory) */
941#define IEMOPFORM_RM_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
942/** ModR/M: reg, r/m */
943#define IEMOPFORM_RMI 1
944/** ModR/M: reg, r/m (register) */
945#define IEMOPFORM_RMI_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
946/** ModR/M: reg, r/m (memory) */
947#define IEMOPFORM_RMI_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
948/** ModR/M: r/m, reg */
949#define IEMOPFORM_MR 2
950/** ModR/M: r/m (register), reg */
951#define IEMOPFORM_MR_REG (IEMOPFORM_MR | IEMOPFORM_MOD3)
952/** ModR/M: r/m (memory), reg */
953#define IEMOPFORM_MR_MEM (IEMOPFORM_MR | IEMOPFORM_NOT_MOD3)
954/** ModR/M: r/m only */
955#define IEMOPFORM_M 3
956/** ModR/M: r/m only (register). */
957#define IEMOPFORM_M_REG (IEMOPFORM_M | IEMOPFORM_MOD3)
958/** ModR/M: r/m only (memory). */
959#define IEMOPFORM_M_MEM (IEMOPFORM_M | IEMOPFORM_NOT_MOD3)
960/** ModR/M: reg only */
961#define IEMOPFORM_R 4
962
963/** VEX+ModR/M: reg, r/m */
964#define IEMOPFORM_VEX_RM 8
965/** VEX+ModR/M: reg, r/m (register) */
966#define IEMOPFORM_VEX_RM_REG (IEMOPFORM_VEX_RM | IEMOPFORM_MOD3)
967/** VEX+ModR/M: reg, r/m (memory) */
968#define IEMOPFORM_VEX_RM_MEM (IEMOPFORM_VEX_RM | IEMOPFORM_NOT_MOD3)
969/** VEX+ModR/M: r/m, reg */
970#define IEMOPFORM_VEX_MR 9
971/** VEX+ModR/M: r/m (register), reg */
972#define IEMOPFORM_VEX_MR_REG (IEMOPFORM_VEX_MR | IEMOPFORM_MOD3)
973/** VEX+ModR/M: r/m (memory), reg */
974#define IEMOPFORM_VEX_MR_MEM (IEMOPFORM_VEX_MR | IEMOPFORM_NOT_MOD3)
975/** VEX+ModR/M: r/m only */
976#define IEMOPFORM_VEX_M 10
977/** VEX+ModR/M: r/m only (register). */
978#define IEMOPFORM_VEX_M_REG (IEMOPFORM_VEX_M | IEMOPFORM_MOD3)
979/** VEX+ModR/M: r/m only (memory). */
980#define IEMOPFORM_VEX_M_MEM (IEMOPFORM_VEX_M | IEMOPFORM_NOT_MOD3)
981/** VEX+ModR/M: reg only */
982#define IEMOPFORM_VEX_R 11
983/** VEX+ModR/M: reg, vvvv, r/m */
984#define IEMOPFORM_VEX_RVM 12
985/** VEX+ModR/M: reg, vvvv, r/m (register). */
986#define IEMOPFORM_VEX_RVM_REG (IEMOPFORM_VEX_RVM | IEMOPFORM_MOD3)
987/** VEX+ModR/M: reg, vvvv, r/m (memory). */
988#define IEMOPFORM_VEX_RVM_MEM (IEMOPFORM_VEX_RVM | IEMOPFORM_NOT_MOD3)
989/** VEX+ModR/M: reg, r/m, vvvv */
990#define IEMOPFORM_VEX_RMV 13
991/** VEX+ModR/M: reg, r/m, vvvv (register). */
992#define IEMOPFORM_VEX_RMV_REG (IEMOPFORM_VEX_RMV | IEMOPFORM_MOD3)
993/** VEX+ModR/M: reg, r/m, vvvv (memory). */
994#define IEMOPFORM_VEX_RMV_MEM (IEMOPFORM_VEX_RMV | IEMOPFORM_NOT_MOD3)
995/** VEX+ModR/M: reg, r/m, imm8 */
996#define IEMOPFORM_VEX_RMI 14
997/** VEX+ModR/M: reg, r/m, imm8 (register). */
998#define IEMOPFORM_VEX_RMI_REG (IEMOPFORM_VEX_RMI | IEMOPFORM_MOD3)
999/** VEX+ModR/M: reg, r/m, imm8 (memory). */
1000#define IEMOPFORM_VEX_RMI_MEM (IEMOPFORM_VEX_RMI | IEMOPFORM_NOT_MOD3)
1001/** VEX+ModR/M: r/m, vvvv, reg */
1002#define IEMOPFORM_VEX_MVR 15
1003/** VEX+ModR/M: r/m, vvvv, reg (register) */
1004#define IEMOPFORM_VEX_MVR_REG (IEMOPFORM_VEX_MVR | IEMOPFORM_MOD3)
1005/** VEX+ModR/M: r/m, vvvv, reg (memory) */
1006#define IEMOPFORM_VEX_MVR_MEM (IEMOPFORM_VEX_MVR | IEMOPFORM_NOT_MOD3)
1007/** VEX+ModR/M+/n: vvvv, r/m */
1008#define IEMOPFORM_VEX_VM 16
1009/** VEX+ModR/M+/n: vvvv, r/m (register) */
1010#define IEMOPFORM_VEX_VM_REG (IEMOPFORM_VEX_VM | IEMOPFORM_MOD3)
1011/** VEX+ModR/M+/n: vvvv, r/m (memory) */
1012#define IEMOPFORM_VEX_VM_MEM (IEMOPFORM_VEX_VM | IEMOPFORM_NOT_MOD3)
1013
1014/** Fixed register instruction, no R/M. */
1015#define IEMOPFORM_FIXED 32
1016
1017/** The r/m is a register. */
1018#define IEMOPFORM_MOD3 RT_BIT_32(8)
1019/** The r/m is a memory access. */
1020#define IEMOPFORM_NOT_MOD3 RT_BIT_32(9)
1021/** @} */
1022
1023/** @name IEMOPHINT_XXX - Additional Opcode Hints
1024 * @note These are ORed together with IEMOPFORM_XXX.
1025 * @{ */
1026/** Ignores the operand size prefix (66h). */
1027#define IEMOPHINT_IGNORES_OZ_PFX RT_BIT_32(10)
1028/** Ignores REX.W (aka WIG). */
1029#define IEMOPHINT_IGNORES_REXW RT_BIT_32(11)
1030/** Both the operand size prefixes (66h + REX.W) are ignored. */
1031#define IEMOPHINT_IGNORES_OP_SIZES (IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW)
1032/** Allowed with the lock prefix. */
1033#define IEMOPHINT_LOCK_ALLOWED RT_BIT_32(11)
1034/** The VEX.L value is ignored (aka LIG). */
1035#define IEMOPHINT_VEX_L_IGNORED RT_BIT_32(12)
1036/** The VEX.L value must be zero (i.e. 128-bit width only). */
1037#define IEMOPHINT_VEX_L_ZERO RT_BIT_32(13)
1038/** The VEX.V value must be zero. */
1039#define IEMOPHINT_VEX_V_ZERO RT_BIT_32(14)
1040
1041/** Hint to IEMAllInstructionPython.py that this macro should be skipped. */
1042#define IEMOPHINT_SKIP_PYTHON RT_BIT_32(31)
1043/** @} */
1044
1045/**
1046 * Possible hardware task switch sources.
1047 */
1048typedef enum IEMTASKSWITCH
1049{
1050 /** Task switch caused by an interrupt/exception. */
1051 IEMTASKSWITCH_INT_XCPT = 1,
1052 /** Task switch caused by a far CALL. */
1053 IEMTASKSWITCH_CALL,
1054 /** Task switch caused by a far JMP. */
1055 IEMTASKSWITCH_JUMP,
1056 /** Task switch caused by an IRET. */
1057 IEMTASKSWITCH_IRET
1058} IEMTASKSWITCH;
1059AssertCompileSize(IEMTASKSWITCH, 4);
1060
1061/**
1062 * Possible CrX load (write) sources.
1063 */
1064typedef enum IEMACCESSCRX
1065{
1066 /** CrX access caused by 'mov crX' instruction. */
1067 IEMACCESSCRX_MOV_CRX,
1068 /** CrX (CR0) write caused by 'lmsw' instruction. */
1069 IEMACCESSCRX_LMSW,
1070 /** CrX (CR0) write caused by 'clts' instruction. */
1071 IEMACCESSCRX_CLTS,
1072 /** CrX (CR0) read caused by 'smsw' instruction. */
1073 IEMACCESSCRX_SMSW
1074} IEMACCESSCRX;
1075
1076#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1077/** @name IEM_SLAT_FAIL_XXX - Second-level address translation failure information.
1078 *
1079 * These flags provide further context to SLAT page-walk failures that could not be
1080 * determined by PGM (e.g, PGM is not privy to memory access permissions).
1081 *
1082 * @{
1083 */
1084/** Translating a nested-guest linear address failed accessing a nested-guest
1085 * physical address. */
1086# define IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR RT_BIT_32(0)
1087/** Translating a nested-guest linear address failed accessing a
1088 * paging-structure entry or updating accessed/dirty bits. */
1089# define IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE RT_BIT_32(1)
1090/** @} */
1091
1092DECLCALLBACK(FNPGMPHYSHANDLER) iemVmxApicAccessPageHandler;
1093# ifndef IN_RING3
1094DECLCALLBACK(FNPGMRZPHYSPFHANDLER) iemVmxApicAccessPagePfHandler;
1095# endif
1096#endif
1097
1098/**
1099 * Indicates to the verifier that the given flag set is undefined.
1100 *
1101 * Can be invoked again to add more flags.
1102 *
1103 * This is a NOOP if the verifier isn't compiled in.
1104 *
1105 * @note We're temporarily keeping this until code is converted to new
1106 * disassembler style opcode handling.
1107 */
1108#define IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fEfl) do { } while (0)
1109
1110
1111/** @def IEM_DECL_IMPL_TYPE
1112 * For typedef'ing an instruction implementation function.
1113 *
1114 * @param a_RetType The return type.
1115 * @param a_Name The name of the type.
1116 * @param a_ArgList The argument list enclosed in parentheses.
1117 */
1118
1119/** @def IEM_DECL_IMPL_DEF
1120 * For defining an instruction implementation function.
1121 *
1122 * @param a_RetType The return type.
1123 * @param a_Name The name of the type.
1124 * @param a_ArgList The argument list enclosed in parentheses.
1125 */
1126
1127#if defined(__GNUC__) && defined(RT_ARCH_X86)
1128# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1129 __attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
1130# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1131 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1132# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1133 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1134
1135#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
1136# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1137 a_RetType (__fastcall a_Name) a_ArgList
1138# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1139 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1140# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1141 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1142
1143#elif __cplusplus >= 201700 /* P0012R1 support */
1144# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1145 a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
1146# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1147 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1148# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1149 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1150
1151#else
1152# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1153 a_RetType (VBOXCALL a_Name) a_ArgList
1154# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1155 a_RetType VBOXCALL a_Name a_ArgList
1156# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1157 a_RetType VBOXCALL a_Name a_ArgList
1158
1159#endif
1160
1161/** Defined in IEMAllAImplC.cpp but also used by IEMAllAImplA.asm. */
1162RT_C_DECLS_BEGIN
1163extern uint8_t const g_afParity[256];
1164RT_C_DECLS_END
1165
1166
1167/** @name Arithmetic assignment operations on bytes (binary).
1168 * @{ */
1169typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU8, (uint8_t *pu8Dst, uint8_t u8Src, uint32_t *pEFlags));
1170typedef FNIEMAIMPLBINU8 *PFNIEMAIMPLBINU8;
1171FNIEMAIMPLBINU8 iemAImpl_add_u8, iemAImpl_add_u8_locked;
1172FNIEMAIMPLBINU8 iemAImpl_adc_u8, iemAImpl_adc_u8_locked;
1173FNIEMAIMPLBINU8 iemAImpl_sub_u8, iemAImpl_sub_u8_locked;
1174FNIEMAIMPLBINU8 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked;
1175FNIEMAIMPLBINU8 iemAImpl_or_u8, iemAImpl_or_u8_locked;
1176FNIEMAIMPLBINU8 iemAImpl_xor_u8, iemAImpl_xor_u8_locked;
1177FNIEMAIMPLBINU8 iemAImpl_and_u8, iemAImpl_and_u8_locked;
1178/** @} */
1179
1180/** @name Arithmetic assignment operations on words (binary).
1181 * @{ */
1182typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU16, (uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pEFlags));
1183typedef FNIEMAIMPLBINU16 *PFNIEMAIMPLBINU16;
1184FNIEMAIMPLBINU16 iemAImpl_add_u16, iemAImpl_add_u16_locked;
1185FNIEMAIMPLBINU16 iemAImpl_adc_u16, iemAImpl_adc_u16_locked;
1186FNIEMAIMPLBINU16 iemAImpl_sub_u16, iemAImpl_sub_u16_locked;
1187FNIEMAIMPLBINU16 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked;
1188FNIEMAIMPLBINU16 iemAImpl_or_u16, iemAImpl_or_u16_locked;
1189FNIEMAIMPLBINU16 iemAImpl_xor_u16, iemAImpl_xor_u16_locked;
1190FNIEMAIMPLBINU16 iemAImpl_and_u16, iemAImpl_and_u16_locked;
1191/** @} */
1192
1193/** @name Arithmetic assignment operations on double words (binary).
1194 * @{ */
1195typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU32, (uint32_t *pu32Dst, uint32_t u32Src, uint32_t *pEFlags));
1196typedef FNIEMAIMPLBINU32 *PFNIEMAIMPLBINU32;
1197FNIEMAIMPLBINU32 iemAImpl_add_u32, iemAImpl_add_u32_locked;
1198FNIEMAIMPLBINU32 iemAImpl_adc_u32, iemAImpl_adc_u32_locked;
1199FNIEMAIMPLBINU32 iemAImpl_sub_u32, iemAImpl_sub_u32_locked;
1200FNIEMAIMPLBINU32 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked;
1201FNIEMAIMPLBINU32 iemAImpl_or_u32, iemAImpl_or_u32_locked;
1202FNIEMAIMPLBINU32 iemAImpl_xor_u32, iemAImpl_xor_u32_locked;
1203FNIEMAIMPLBINU32 iemAImpl_and_u32, iemAImpl_and_u32_locked;
1204FNIEMAIMPLBINU32 iemAImpl_blsi_u32, iemAImpl_blsi_u32_fallback;
1205FNIEMAIMPLBINU32 iemAImpl_blsr_u32, iemAImpl_blsr_u32_fallback;
1206FNIEMAIMPLBINU32 iemAImpl_blsmsk_u32, iemAImpl_blsmsk_u32_fallback;
1207/** @} */
1208
1209/** @name Arithmetic assignment operations on quad words (binary).
1210 * @{ */
1211typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU64, (uint64_t *pu64Dst, uint64_t u64Src, uint32_t *pEFlags));
1212typedef FNIEMAIMPLBINU64 *PFNIEMAIMPLBINU64;
1213FNIEMAIMPLBINU64 iemAImpl_add_u64, iemAImpl_add_u64_locked;
1214FNIEMAIMPLBINU64 iemAImpl_adc_u64, iemAImpl_adc_u64_locked;
1215FNIEMAIMPLBINU64 iemAImpl_sub_u64, iemAImpl_sub_u64_locked;
1216FNIEMAIMPLBINU64 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked;
1217FNIEMAIMPLBINU64 iemAImpl_or_u64, iemAImpl_or_u64_locked;
1218FNIEMAIMPLBINU64 iemAImpl_xor_u64, iemAImpl_xor_u64_locked;
1219FNIEMAIMPLBINU64 iemAImpl_and_u64, iemAImpl_and_u64_locked;
1220FNIEMAIMPLBINU64 iemAImpl_blsi_u64, iemAImpl_blsi_u64_fallback;
1221FNIEMAIMPLBINU64 iemAImpl_blsr_u64, iemAImpl_blsr_u64_fallback;
1222FNIEMAIMPLBINU64 iemAImpl_blsmsk_u64, iemAImpl_blsmsk_u64_fallback;
1223/** @} */
1224
1225/** @name Compare operations (thrown in with the binary ops).
1226 * @{ */
1227FNIEMAIMPLBINU8 iemAImpl_cmp_u8;
1228FNIEMAIMPLBINU16 iemAImpl_cmp_u16;
1229FNIEMAIMPLBINU32 iemAImpl_cmp_u32;
1230FNIEMAIMPLBINU64 iemAImpl_cmp_u64;
1231/** @} */
1232
1233/** @name Test operations (thrown in with the binary ops).
1234 * @{ */
1235FNIEMAIMPLBINU8 iemAImpl_test_u8;
1236FNIEMAIMPLBINU16 iemAImpl_test_u16;
1237FNIEMAIMPLBINU32 iemAImpl_test_u32;
1238FNIEMAIMPLBINU64 iemAImpl_test_u64;
1239/** @} */
1240
1241/** @name Bit operations operations (thrown in with the binary ops).
1242 * @{ */
1243FNIEMAIMPLBINU16 iemAImpl_bt_u16;
1244FNIEMAIMPLBINU32 iemAImpl_bt_u32;
1245FNIEMAIMPLBINU64 iemAImpl_bt_u64;
1246FNIEMAIMPLBINU16 iemAImpl_btc_u16, iemAImpl_btc_u16_locked;
1247FNIEMAIMPLBINU32 iemAImpl_btc_u32, iemAImpl_btc_u32_locked;
1248FNIEMAIMPLBINU64 iemAImpl_btc_u64, iemAImpl_btc_u64_locked;
1249FNIEMAIMPLBINU16 iemAImpl_btr_u16, iemAImpl_btr_u16_locked;
1250FNIEMAIMPLBINU32 iemAImpl_btr_u32, iemAImpl_btr_u32_locked;
1251FNIEMAIMPLBINU64 iemAImpl_btr_u64, iemAImpl_btr_u64_locked;
1252FNIEMAIMPLBINU16 iemAImpl_bts_u16, iemAImpl_bts_u16_locked;
1253FNIEMAIMPLBINU32 iemAImpl_bts_u32, iemAImpl_bts_u32_locked;
1254FNIEMAIMPLBINU64 iemAImpl_bts_u64, iemAImpl_bts_u64_locked;
1255/** @} */
1256
1257/** @name Arithmetic three operand operations on double words (binary).
1258 * @{ */
1259typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2, uint32_t *pEFlags));
1260typedef FNIEMAIMPLBINVEXU32 *PFNIEMAIMPLBINVEXU32;
1261FNIEMAIMPLBINVEXU32 iemAImpl_andn_u32, iemAImpl_andn_u32_fallback;
1262FNIEMAIMPLBINVEXU32 iemAImpl_bextr_u32, iemAImpl_bextr_u32_fallback;
1263FNIEMAIMPLBINVEXU32 iemAImpl_bzhi_u32, iemAImpl_bzhi_u32_fallback;
1264/** @} */
1265
1266/** @name Arithmetic three operand operations on quad words (binary).
1267 * @{ */
1268typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2, uint32_t *pEFlags));
1269typedef FNIEMAIMPLBINVEXU64 *PFNIEMAIMPLBINVEXU64;
1270FNIEMAIMPLBINVEXU64 iemAImpl_andn_u64, iemAImpl_andn_u64_fallback;
1271FNIEMAIMPLBINVEXU64 iemAImpl_bextr_u64, iemAImpl_bextr_u64_fallback;
1272FNIEMAIMPLBINVEXU64 iemAImpl_bzhi_u64, iemAImpl_bzhi_u64_fallback;
1273/** @} */
1274
1275/** @name Arithmetic three operand operations on double words w/o EFLAGS (binary).
1276 * @{ */
1277typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32NOEFL, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2));
1278typedef FNIEMAIMPLBINVEXU32NOEFL *PFNIEMAIMPLBINVEXU32NOEFL;
1279FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pdep_u32, iemAImpl_pdep_u32_fallback;
1280FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pext_u32, iemAImpl_pext_u32_fallback;
1281FNIEMAIMPLBINVEXU32NOEFL iemAImpl_sarx_u32, iemAImpl_sarx_u32_fallback;
1282FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shlx_u32, iemAImpl_shlx_u32_fallback;
1283FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shrx_u32, iemAImpl_shrx_u32_fallback;
1284FNIEMAIMPLBINVEXU32NOEFL iemAImpl_rorx_u32;
1285/** @} */
1286
1287/** @name Arithmetic three operand operations on quad words w/o EFLAGS (binary).
1288 * @{ */
1289typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64NOEFL, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2));
1290typedef FNIEMAIMPLBINVEXU64NOEFL *PFNIEMAIMPLBINVEXU64NOEFL;
1291FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pdep_u64, iemAImpl_pdep_u64_fallback;
1292FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pext_u64, iemAImpl_pext_u64_fallback;
1293FNIEMAIMPLBINVEXU64NOEFL iemAImpl_sarx_u64, iemAImpl_sarx_u64_fallback;
1294FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shlx_u64, iemAImpl_shlx_u64_fallback;
1295FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shrx_u64, iemAImpl_shrx_u64_fallback;
1296FNIEMAIMPLBINVEXU64NOEFL iemAImpl_rorx_u64;
1297/** @} */
1298
1299/** @name MULX 32-bit and 64-bit.
1300 * @{ */
1301typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU32, (uint32_t *puDst1, uint32_t *puDst2, uint32_t uSrc1, uint32_t uSrc2));
1302typedef FNIEMAIMPLMULXVEXU32 *PFNIEMAIMPLMULXVEXU32;
1303FNIEMAIMPLMULXVEXU32 iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback;
1304
1305typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU64, (uint64_t *puDst1, uint64_t *puDst2, uint64_t uSrc1, uint64_t uSrc2));
1306typedef FNIEMAIMPLMULXVEXU64 *PFNIEMAIMPLMULXVEXU64;
1307FNIEMAIMPLMULXVEXU64 iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback;
1308/** @} */
1309
1310
1311/** @name Exchange memory with register operations.
1312 * @{ */
1313IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1314IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1315IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1316IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1317IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1318IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1319IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1320IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1321/** @} */
1322
1323/** @name Exchange and add operations.
1324 * @{ */
1325IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1326IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1327IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1328IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1329IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8_locked, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1330IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16_locked,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1331IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32_locked,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1332IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1333/** @} */
1334
1335/** @name Compare and exchange.
1336 * @{ */
1337IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1338IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1339IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1340IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1341IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1342IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1343#if ARCH_BITS == 32
1344IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1345IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1346#else
1347IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1348IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1349#endif
1350IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1351 uint32_t *pEFlags));
1352IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1353 uint32_t *pEFlags));
1354IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1355 uint32_t *pEFlags));
1356IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1357 uint32_t *pEFlags));
1358#ifndef RT_ARCH_ARM64
1359IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1360 PRTUINT128U pu128RbxRcx, uint32_t *pEFlags));
1361#endif
1362/** @} */
1363
1364/** @name Memory ordering
1365 * @{ */
1366typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEMFENCE,(void));
1367typedef FNIEMAIMPLMEMFENCE *PFNIEMAIMPLMEMFENCE;
1368IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void));
1369IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void));
1370IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void));
1371#ifndef RT_ARCH_ARM64
1372IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void));
1373#endif
1374/** @} */
1375
1376/** @name Double precision shifts
1377 * @{ */
1378typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU16,(uint16_t *pu16Dst, uint16_t u16Src, uint8_t cShift, uint32_t *pEFlags));
1379typedef FNIEMAIMPLSHIFTDBLU16 *PFNIEMAIMPLSHIFTDBLU16;
1380typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU32,(uint32_t *pu32Dst, uint32_t u32Src, uint8_t cShift, uint32_t *pEFlags));
1381typedef FNIEMAIMPLSHIFTDBLU32 *PFNIEMAIMPLSHIFTDBLU32;
1382typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU64,(uint64_t *pu64Dst, uint64_t u64Src, uint8_t cShift, uint32_t *pEFlags));
1383typedef FNIEMAIMPLSHIFTDBLU64 *PFNIEMAIMPLSHIFTDBLU64;
1384FNIEMAIMPLSHIFTDBLU16 iemAImpl_shld_u16, iemAImpl_shld_u16_amd, iemAImpl_shld_u16_intel;
1385FNIEMAIMPLSHIFTDBLU32 iemAImpl_shld_u32, iemAImpl_shld_u32_amd, iemAImpl_shld_u32_intel;
1386FNIEMAIMPLSHIFTDBLU64 iemAImpl_shld_u64, iemAImpl_shld_u64_amd, iemAImpl_shld_u64_intel;
1387FNIEMAIMPLSHIFTDBLU16 iemAImpl_shrd_u16, iemAImpl_shrd_u16_amd, iemAImpl_shrd_u16_intel;
1388FNIEMAIMPLSHIFTDBLU32 iemAImpl_shrd_u32, iemAImpl_shrd_u32_amd, iemAImpl_shrd_u32_intel;
1389FNIEMAIMPLSHIFTDBLU64 iemAImpl_shrd_u64, iemAImpl_shrd_u64_amd, iemAImpl_shrd_u64_intel;
1390/** @} */
1391
1392
1393/** @name Bit search operations (thrown in with the binary ops).
1394 * @{ */
1395FNIEMAIMPLBINU16 iemAImpl_bsf_u16, iemAImpl_bsf_u16_amd, iemAImpl_bsf_u16_intel;
1396FNIEMAIMPLBINU32 iemAImpl_bsf_u32, iemAImpl_bsf_u32_amd, iemAImpl_bsf_u32_intel;
1397FNIEMAIMPLBINU64 iemAImpl_bsf_u64, iemAImpl_bsf_u64_amd, iemAImpl_bsf_u64_intel;
1398FNIEMAIMPLBINU16 iemAImpl_bsr_u16, iemAImpl_bsr_u16_amd, iemAImpl_bsr_u16_intel;
1399FNIEMAIMPLBINU32 iemAImpl_bsr_u32, iemAImpl_bsr_u32_amd, iemAImpl_bsr_u32_intel;
1400FNIEMAIMPLBINU64 iemAImpl_bsr_u64, iemAImpl_bsr_u64_amd, iemAImpl_bsr_u64_intel;
1401FNIEMAIMPLBINU16 iemAImpl_lzcnt_u16, iemAImpl_lzcnt_u16_amd, iemAImpl_lzcnt_u16_intel;
1402FNIEMAIMPLBINU32 iemAImpl_lzcnt_u32, iemAImpl_lzcnt_u32_amd, iemAImpl_lzcnt_u32_intel;
1403FNIEMAIMPLBINU64 iemAImpl_lzcnt_u64, iemAImpl_lzcnt_u64_amd, iemAImpl_lzcnt_u64_intel;
1404FNIEMAIMPLBINU16 iemAImpl_tzcnt_u16, iemAImpl_tzcnt_u16_amd, iemAImpl_tzcnt_u16_intel;
1405FNIEMAIMPLBINU32 iemAImpl_tzcnt_u32, iemAImpl_tzcnt_u32_amd, iemAImpl_tzcnt_u32_intel;
1406FNIEMAIMPLBINU64 iemAImpl_tzcnt_u64, iemAImpl_tzcnt_u64_amd, iemAImpl_tzcnt_u64_intel;
1407FNIEMAIMPLBINU16 iemAImpl_popcnt_u16, iemAImpl_popcnt_u16_fallback;
1408FNIEMAIMPLBINU32 iemAImpl_popcnt_u32, iemAImpl_popcnt_u32_fallback;
1409FNIEMAIMPLBINU64 iemAImpl_popcnt_u64, iemAImpl_popcnt_u64_fallback;
1410/** @} */
1411
1412/** @name Signed multiplication operations (thrown in with the binary ops).
1413 * @{ */
1414FNIEMAIMPLBINU16 iemAImpl_imul_two_u16, iemAImpl_imul_two_u16_amd, iemAImpl_imul_two_u16_intel;
1415FNIEMAIMPLBINU32 iemAImpl_imul_two_u32, iemAImpl_imul_two_u32_amd, iemAImpl_imul_two_u32_intel;
1416FNIEMAIMPLBINU64 iemAImpl_imul_two_u64, iemAImpl_imul_two_u64_amd, iemAImpl_imul_two_u64_intel;
1417/** @} */
1418
1419/** @name Arithmetic assignment operations on bytes (unary).
1420 * @{ */
1421typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU8, (uint8_t *pu8Dst, uint32_t *pEFlags));
1422typedef FNIEMAIMPLUNARYU8 *PFNIEMAIMPLUNARYU8;
1423FNIEMAIMPLUNARYU8 iemAImpl_inc_u8, iemAImpl_inc_u8_locked;
1424FNIEMAIMPLUNARYU8 iemAImpl_dec_u8, iemAImpl_dec_u8_locked;
1425FNIEMAIMPLUNARYU8 iemAImpl_not_u8, iemAImpl_not_u8_locked;
1426FNIEMAIMPLUNARYU8 iemAImpl_neg_u8, iemAImpl_neg_u8_locked;
1427/** @} */
1428
1429/** @name Arithmetic assignment operations on words (unary).
1430 * @{ */
1431typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU16, (uint16_t *pu16Dst, uint32_t *pEFlags));
1432typedef FNIEMAIMPLUNARYU16 *PFNIEMAIMPLUNARYU16;
1433FNIEMAIMPLUNARYU16 iemAImpl_inc_u16, iemAImpl_inc_u16_locked;
1434FNIEMAIMPLUNARYU16 iemAImpl_dec_u16, iemAImpl_dec_u16_locked;
1435FNIEMAIMPLUNARYU16 iemAImpl_not_u16, iemAImpl_not_u16_locked;
1436FNIEMAIMPLUNARYU16 iemAImpl_neg_u16, iemAImpl_neg_u16_locked;
1437/** @} */
1438
1439/** @name Arithmetic assignment operations on double words (unary).
1440 * @{ */
1441typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU32, (uint32_t *pu32Dst, uint32_t *pEFlags));
1442typedef FNIEMAIMPLUNARYU32 *PFNIEMAIMPLUNARYU32;
1443FNIEMAIMPLUNARYU32 iemAImpl_inc_u32, iemAImpl_inc_u32_locked;
1444FNIEMAIMPLUNARYU32 iemAImpl_dec_u32, iemAImpl_dec_u32_locked;
1445FNIEMAIMPLUNARYU32 iemAImpl_not_u32, iemAImpl_not_u32_locked;
1446FNIEMAIMPLUNARYU32 iemAImpl_neg_u32, iemAImpl_neg_u32_locked;
1447/** @} */
1448
1449/** @name Arithmetic assignment operations on quad words (unary).
1450 * @{ */
1451typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU64, (uint64_t *pu64Dst, uint32_t *pEFlags));
1452typedef FNIEMAIMPLUNARYU64 *PFNIEMAIMPLUNARYU64;
1453FNIEMAIMPLUNARYU64 iemAImpl_inc_u64, iemAImpl_inc_u64_locked;
1454FNIEMAIMPLUNARYU64 iemAImpl_dec_u64, iemAImpl_dec_u64_locked;
1455FNIEMAIMPLUNARYU64 iemAImpl_not_u64, iemAImpl_not_u64_locked;
1456FNIEMAIMPLUNARYU64 iemAImpl_neg_u64, iemAImpl_neg_u64_locked;
1457/** @} */
1458
1459
1460/** @name Shift operations on bytes (Group 2).
1461 * @{ */
1462typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU8,(uint8_t *pu8Dst, uint8_t cShift, uint32_t *pEFlags));
1463typedef FNIEMAIMPLSHIFTU8 *PFNIEMAIMPLSHIFTU8;
1464FNIEMAIMPLSHIFTU8 iemAImpl_rol_u8, iemAImpl_rol_u8_amd, iemAImpl_rol_u8_intel;
1465FNIEMAIMPLSHIFTU8 iemAImpl_ror_u8, iemAImpl_ror_u8_amd, iemAImpl_ror_u8_intel;
1466FNIEMAIMPLSHIFTU8 iemAImpl_rcl_u8, iemAImpl_rcl_u8_amd, iemAImpl_rcl_u8_intel;
1467FNIEMAIMPLSHIFTU8 iemAImpl_rcr_u8, iemAImpl_rcr_u8_amd, iemAImpl_rcr_u8_intel;
1468FNIEMAIMPLSHIFTU8 iemAImpl_shl_u8, iemAImpl_shl_u8_amd, iemAImpl_shl_u8_intel;
1469FNIEMAIMPLSHIFTU8 iemAImpl_shr_u8, iemAImpl_shr_u8_amd, iemAImpl_shr_u8_intel;
1470FNIEMAIMPLSHIFTU8 iemAImpl_sar_u8, iemAImpl_sar_u8_amd, iemAImpl_sar_u8_intel;
1471/** @} */
1472
1473/** @name Shift operations on words (Group 2).
1474 * @{ */
1475typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU16,(uint16_t *pu16Dst, uint8_t cShift, uint32_t *pEFlags));
1476typedef FNIEMAIMPLSHIFTU16 *PFNIEMAIMPLSHIFTU16;
1477FNIEMAIMPLSHIFTU16 iemAImpl_rol_u16, iemAImpl_rol_u16_amd, iemAImpl_rol_u16_intel;
1478FNIEMAIMPLSHIFTU16 iemAImpl_ror_u16, iemAImpl_ror_u16_amd, iemAImpl_ror_u16_intel;
1479FNIEMAIMPLSHIFTU16 iemAImpl_rcl_u16, iemAImpl_rcl_u16_amd, iemAImpl_rcl_u16_intel;
1480FNIEMAIMPLSHIFTU16 iemAImpl_rcr_u16, iemAImpl_rcr_u16_amd, iemAImpl_rcr_u16_intel;
1481FNIEMAIMPLSHIFTU16 iemAImpl_shl_u16, iemAImpl_shl_u16_amd, iemAImpl_shl_u16_intel;
1482FNIEMAIMPLSHIFTU16 iemAImpl_shr_u16, iemAImpl_shr_u16_amd, iemAImpl_shr_u16_intel;
1483FNIEMAIMPLSHIFTU16 iemAImpl_sar_u16, iemAImpl_sar_u16_amd, iemAImpl_sar_u16_intel;
1484/** @} */
1485
1486/** @name Shift operations on double words (Group 2).
1487 * @{ */
1488typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU32,(uint32_t *pu32Dst, uint8_t cShift, uint32_t *pEFlags));
1489typedef FNIEMAIMPLSHIFTU32 *PFNIEMAIMPLSHIFTU32;
1490FNIEMAIMPLSHIFTU32 iemAImpl_rol_u32, iemAImpl_rol_u32_amd, iemAImpl_rol_u32_intel;
1491FNIEMAIMPLSHIFTU32 iemAImpl_ror_u32, iemAImpl_ror_u32_amd, iemAImpl_ror_u32_intel;
1492FNIEMAIMPLSHIFTU32 iemAImpl_rcl_u32, iemAImpl_rcl_u32_amd, iemAImpl_rcl_u32_intel;
1493FNIEMAIMPLSHIFTU32 iemAImpl_rcr_u32, iemAImpl_rcr_u32_amd, iemAImpl_rcr_u32_intel;
1494FNIEMAIMPLSHIFTU32 iemAImpl_shl_u32, iemAImpl_shl_u32_amd, iemAImpl_shl_u32_intel;
1495FNIEMAIMPLSHIFTU32 iemAImpl_shr_u32, iemAImpl_shr_u32_amd, iemAImpl_shr_u32_intel;
1496FNIEMAIMPLSHIFTU32 iemAImpl_sar_u32, iemAImpl_sar_u32_amd, iemAImpl_sar_u32_intel;
1497/** @} */
1498
1499/** @name Shift operations on words (Group 2).
1500 * @{ */
1501typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU64,(uint64_t *pu64Dst, uint8_t cShift, uint32_t *pEFlags));
1502typedef FNIEMAIMPLSHIFTU64 *PFNIEMAIMPLSHIFTU64;
1503FNIEMAIMPLSHIFTU64 iemAImpl_rol_u64, iemAImpl_rol_u64_amd, iemAImpl_rol_u64_intel;
1504FNIEMAIMPLSHIFTU64 iemAImpl_ror_u64, iemAImpl_ror_u64_amd, iemAImpl_ror_u64_intel;
1505FNIEMAIMPLSHIFTU64 iemAImpl_rcl_u64, iemAImpl_rcl_u64_amd, iemAImpl_rcl_u64_intel;
1506FNIEMAIMPLSHIFTU64 iemAImpl_rcr_u64, iemAImpl_rcr_u64_amd, iemAImpl_rcr_u64_intel;
1507FNIEMAIMPLSHIFTU64 iemAImpl_shl_u64, iemAImpl_shl_u64_amd, iemAImpl_shl_u64_intel;
1508FNIEMAIMPLSHIFTU64 iemAImpl_shr_u64, iemAImpl_shr_u64_amd, iemAImpl_shr_u64_intel;
1509FNIEMAIMPLSHIFTU64 iemAImpl_sar_u64, iemAImpl_sar_u64_amd, iemAImpl_sar_u64_intel;
1510/** @} */
1511
1512/** @name Multiplication and division operations.
1513 * @{ */
1514typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU8,(uint16_t *pu16AX, uint8_t u8FactorDivisor, uint32_t *pEFlags));
1515typedef FNIEMAIMPLMULDIVU8 *PFNIEMAIMPLMULDIVU8;
1516FNIEMAIMPLMULDIVU8 iemAImpl_mul_u8, iemAImpl_mul_u8_amd, iemAImpl_mul_u8_intel;
1517FNIEMAIMPLMULDIVU8 iemAImpl_imul_u8, iemAImpl_imul_u8_amd, iemAImpl_imul_u8_intel;
1518FNIEMAIMPLMULDIVU8 iemAImpl_div_u8, iemAImpl_div_u8_amd, iemAImpl_div_u8_intel;
1519FNIEMAIMPLMULDIVU8 iemAImpl_idiv_u8, iemAImpl_idiv_u8_amd, iemAImpl_idiv_u8_intel;
1520
1521typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU16,(uint16_t *pu16AX, uint16_t *pu16DX, uint16_t u16FactorDivisor, uint32_t *pEFlags));
1522typedef FNIEMAIMPLMULDIVU16 *PFNIEMAIMPLMULDIVU16;
1523FNIEMAIMPLMULDIVU16 iemAImpl_mul_u16, iemAImpl_mul_u16_amd, iemAImpl_mul_u16_intel;
1524FNIEMAIMPLMULDIVU16 iemAImpl_imul_u16, iemAImpl_imul_u16_amd, iemAImpl_imul_u16_intel;
1525FNIEMAIMPLMULDIVU16 iemAImpl_div_u16, iemAImpl_div_u16_amd, iemAImpl_div_u16_intel;
1526FNIEMAIMPLMULDIVU16 iemAImpl_idiv_u16, iemAImpl_idiv_u16_amd, iemAImpl_idiv_u16_intel;
1527
1528typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU32,(uint32_t *pu32EAX, uint32_t *pu32EDX, uint32_t u32FactorDivisor, uint32_t *pEFlags));
1529typedef FNIEMAIMPLMULDIVU32 *PFNIEMAIMPLMULDIVU32;
1530FNIEMAIMPLMULDIVU32 iemAImpl_mul_u32, iemAImpl_mul_u32_amd, iemAImpl_mul_u32_intel;
1531FNIEMAIMPLMULDIVU32 iemAImpl_imul_u32, iemAImpl_imul_u32_amd, iemAImpl_imul_u32_intel;
1532FNIEMAIMPLMULDIVU32 iemAImpl_div_u32, iemAImpl_div_u32_amd, iemAImpl_div_u32_intel;
1533FNIEMAIMPLMULDIVU32 iemAImpl_idiv_u32, iemAImpl_idiv_u32_amd, iemAImpl_idiv_u32_intel;
1534
1535typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64FactorDivisor, uint32_t *pEFlags));
1536typedef FNIEMAIMPLMULDIVU64 *PFNIEMAIMPLMULDIVU64;
1537FNIEMAIMPLMULDIVU64 iemAImpl_mul_u64, iemAImpl_mul_u64_amd, iemAImpl_mul_u64_intel;
1538FNIEMAIMPLMULDIVU64 iemAImpl_imul_u64, iemAImpl_imul_u64_amd, iemAImpl_imul_u64_intel;
1539FNIEMAIMPLMULDIVU64 iemAImpl_div_u64, iemAImpl_div_u64_amd, iemAImpl_div_u64_intel;
1540FNIEMAIMPLMULDIVU64 iemAImpl_idiv_u64, iemAImpl_idiv_u64_amd, iemAImpl_idiv_u64_intel;
1541/** @} */
1542
1543/** @name Byte Swap.
1544 * @{ */
1545IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u16,(uint32_t *pu32Dst)); /* Yes, 32-bit register access. */
1546IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u32,(uint32_t *pu32Dst));
1547IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u64,(uint64_t *pu64Dst));
1548/** @} */
1549
1550/** @name Misc.
1551 * @{ */
1552FNIEMAIMPLBINU16 iemAImpl_arpl;
1553/** @} */
1554
1555
1556/** @name FPU operations taking a 32-bit float argument
1557 * @{ */
1558typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1559 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1560typedef FNIEMAIMPLFPUR32FSW *PFNIEMAIMPLFPUR32FSW;
1561
1562typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1563 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1564typedef FNIEMAIMPLFPUR32 *PFNIEMAIMPLFPUR32;
1565
1566FNIEMAIMPLFPUR32FSW iemAImpl_fcom_r80_by_r32;
1567FNIEMAIMPLFPUR32 iemAImpl_fadd_r80_by_r32;
1568FNIEMAIMPLFPUR32 iemAImpl_fmul_r80_by_r32;
1569FNIEMAIMPLFPUR32 iemAImpl_fsub_r80_by_r32;
1570FNIEMAIMPLFPUR32 iemAImpl_fsubr_r80_by_r32;
1571FNIEMAIMPLFPUR32 iemAImpl_fdiv_r80_by_r32;
1572FNIEMAIMPLFPUR32 iemAImpl_fdivr_r80_by_r32;
1573
1574IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val));
1575IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1576 PRTFLOAT32U pr32Val, PCRTFLOAT80U pr80Val));
1577/** @} */
1578
1579/** @name FPU operations taking a 64-bit float argument
1580 * @{ */
1581typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1582 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1583typedef FNIEMAIMPLFPUR64FSW *PFNIEMAIMPLFPUR64FSW;
1584
1585typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1586 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1587typedef FNIEMAIMPLFPUR64 *PFNIEMAIMPLFPUR64;
1588
1589FNIEMAIMPLFPUR64FSW iemAImpl_fcom_r80_by_r64;
1590FNIEMAIMPLFPUR64 iemAImpl_fadd_r80_by_r64;
1591FNIEMAIMPLFPUR64 iemAImpl_fmul_r80_by_r64;
1592FNIEMAIMPLFPUR64 iemAImpl_fsub_r80_by_r64;
1593FNIEMAIMPLFPUR64 iemAImpl_fsubr_r80_by_r64;
1594FNIEMAIMPLFPUR64 iemAImpl_fdiv_r80_by_r64;
1595FNIEMAIMPLFPUR64 iemAImpl_fdivr_r80_by_r64;
1596
1597IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val));
1598IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1599 PRTFLOAT64U pr32Val, PCRTFLOAT80U pr80Val));
1600/** @} */
1601
1602/** @name FPU operations taking a 80-bit float argument
1603 * @{ */
1604typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1605 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1606typedef FNIEMAIMPLFPUR80 *PFNIEMAIMPLFPUR80;
1607FNIEMAIMPLFPUR80 iemAImpl_fadd_r80_by_r80;
1608FNIEMAIMPLFPUR80 iemAImpl_fmul_r80_by_r80;
1609FNIEMAIMPLFPUR80 iemAImpl_fsub_r80_by_r80;
1610FNIEMAIMPLFPUR80 iemAImpl_fsubr_r80_by_r80;
1611FNIEMAIMPLFPUR80 iemAImpl_fdiv_r80_by_r80;
1612FNIEMAIMPLFPUR80 iemAImpl_fdivr_r80_by_r80;
1613FNIEMAIMPLFPUR80 iemAImpl_fprem_r80_by_r80;
1614FNIEMAIMPLFPUR80 iemAImpl_fprem1_r80_by_r80;
1615FNIEMAIMPLFPUR80 iemAImpl_fscale_r80_by_r80;
1616
1617FNIEMAIMPLFPUR80 iemAImpl_fpatan_r80_by_r80, iemAImpl_fpatan_r80_by_r80_amd, iemAImpl_fpatan_r80_by_r80_intel;
1618FNIEMAIMPLFPUR80 iemAImpl_fyl2x_r80_by_r80, iemAImpl_fyl2x_r80_by_r80_amd, iemAImpl_fyl2x_r80_by_r80_intel;
1619FNIEMAIMPLFPUR80 iemAImpl_fyl2xp1_r80_by_r80, iemAImpl_fyl2xp1_r80_by_r80_amd, iemAImpl_fyl2xp1_r80_by_r80_intel;
1620
1621typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1622 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1623typedef FNIEMAIMPLFPUR80FSW *PFNIEMAIMPLFPUR80FSW;
1624FNIEMAIMPLFPUR80FSW iemAImpl_fcom_r80_by_r80;
1625FNIEMAIMPLFPUR80FSW iemAImpl_fucom_r80_by_r80;
1626
1627typedef IEM_DECL_IMPL_TYPE(uint32_t, FNIEMAIMPLFPUR80EFL,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
1628 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1629typedef FNIEMAIMPLFPUR80EFL *PFNIEMAIMPLFPUR80EFL;
1630FNIEMAIMPLFPUR80EFL iemAImpl_fcomi_r80_by_r80;
1631FNIEMAIMPLFPUR80EFL iemAImpl_fucomi_r80_by_r80;
1632
1633typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARY,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1634typedef FNIEMAIMPLFPUR80UNARY *PFNIEMAIMPLFPUR80UNARY;
1635FNIEMAIMPLFPUR80UNARY iemAImpl_fabs_r80;
1636FNIEMAIMPLFPUR80UNARY iemAImpl_fchs_r80;
1637FNIEMAIMPLFPUR80UNARY iemAImpl_f2xm1_r80, iemAImpl_f2xm1_r80_amd, iemAImpl_f2xm1_r80_intel;
1638FNIEMAIMPLFPUR80UNARY iemAImpl_fsqrt_r80;
1639FNIEMAIMPLFPUR80UNARY iemAImpl_frndint_r80;
1640FNIEMAIMPLFPUR80UNARY iemAImpl_fsin_r80, iemAImpl_fsin_r80_amd, iemAImpl_fsin_r80_intel;
1641FNIEMAIMPLFPUR80UNARY iemAImpl_fcos_r80, iemAImpl_fcos_r80_amd, iemAImpl_fcos_r80_intel;
1642
1643typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYFSW,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val));
1644typedef FNIEMAIMPLFPUR80UNARYFSW *PFNIEMAIMPLFPUR80UNARYFSW;
1645FNIEMAIMPLFPUR80UNARYFSW iemAImpl_ftst_r80;
1646FNIEMAIMPLFPUR80UNARYFSW iemAImpl_fxam_r80;
1647
1648typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80LDCONST,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes));
1649typedef FNIEMAIMPLFPUR80LDCONST *PFNIEMAIMPLFPUR80LDCONST;
1650FNIEMAIMPLFPUR80LDCONST iemAImpl_fld1;
1651FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2t;
1652FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2e;
1653FNIEMAIMPLFPUR80LDCONST iemAImpl_fldpi;
1654FNIEMAIMPLFPUR80LDCONST iemAImpl_fldlg2;
1655FNIEMAIMPLFPUR80LDCONST iemAImpl_fldln2;
1656FNIEMAIMPLFPUR80LDCONST iemAImpl_fldz;
1657
1658typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYTWO,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo,
1659 PCRTFLOAT80U pr80Val));
1660typedef FNIEMAIMPLFPUR80UNARYTWO *PFNIEMAIMPLFPUR80UNARYTWO;
1661FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fptan_r80_r80, iemAImpl_fptan_r80_r80_amd, iemAImpl_fptan_r80_r80_intel;
1662FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fxtract_r80_r80;
1663FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fsincos_r80_r80, iemAImpl_fsincos_r80_r80_amd, iemAImpl_fsincos_r80_r80_intel;
1664
1665IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1666IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1667 PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src));
1668
1669IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val));
1670IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1671 PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src));
1672
1673/** @} */
1674
1675/** @name FPU operations taking a 16-bit signed integer argument
1676 * @{ */
1677typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1678 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1679typedef FNIEMAIMPLFPUI16 *PFNIEMAIMPLFPUI16;
1680typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI16,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1681 int16_t *pi16Dst, PCRTFLOAT80U pr80Src));
1682typedef FNIEMAIMPLFPUSTR80TOI16 *PFNIEMAIMPLFPUSTR80TOI16;
1683
1684FNIEMAIMPLFPUI16 iemAImpl_fiadd_r80_by_i16;
1685FNIEMAIMPLFPUI16 iemAImpl_fimul_r80_by_i16;
1686FNIEMAIMPLFPUI16 iemAImpl_fisub_r80_by_i16;
1687FNIEMAIMPLFPUI16 iemAImpl_fisubr_r80_by_i16;
1688FNIEMAIMPLFPUI16 iemAImpl_fidiv_r80_by_i16;
1689FNIEMAIMPLFPUI16 iemAImpl_fidivr_r80_by_i16;
1690
1691typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1692 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1693typedef FNIEMAIMPLFPUI16FSW *PFNIEMAIMPLFPUI16FSW;
1694FNIEMAIMPLFPUI16FSW iemAImpl_ficom_r80_by_i16;
1695
1696IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val));
1697FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fist_r80_to_i16;
1698FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fistt_r80_to_i16, iemAImpl_fistt_r80_to_i16_amd, iemAImpl_fistt_r80_to_i16_intel;
1699/** @} */
1700
1701/** @name FPU operations taking a 32-bit signed integer argument
1702 * @{ */
1703typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1704 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1705typedef FNIEMAIMPLFPUI32 *PFNIEMAIMPLFPUI32;
1706typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI32,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1707 int32_t *pi32Dst, PCRTFLOAT80U pr80Src));
1708typedef FNIEMAIMPLFPUSTR80TOI32 *PFNIEMAIMPLFPUSTR80TOI32;
1709
1710FNIEMAIMPLFPUI32 iemAImpl_fiadd_r80_by_i32;
1711FNIEMAIMPLFPUI32 iemAImpl_fimul_r80_by_i32;
1712FNIEMAIMPLFPUI32 iemAImpl_fisub_r80_by_i32;
1713FNIEMAIMPLFPUI32 iemAImpl_fisubr_r80_by_i32;
1714FNIEMAIMPLFPUI32 iemAImpl_fidiv_r80_by_i32;
1715FNIEMAIMPLFPUI32 iemAImpl_fidivr_r80_by_i32;
1716
1717typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1718 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1719typedef FNIEMAIMPLFPUI32FSW *PFNIEMAIMPLFPUI32FSW;
1720FNIEMAIMPLFPUI32FSW iemAImpl_ficom_r80_by_i32;
1721
1722IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val));
1723FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fist_r80_to_i32;
1724FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fistt_r80_to_i32;
1725/** @} */
1726
1727/** @name FPU operations taking a 64-bit signed integer argument
1728 * @{ */
1729typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI64,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1730 int64_t *pi64Dst, PCRTFLOAT80U pr80Src));
1731typedef FNIEMAIMPLFPUSTR80TOI64 *PFNIEMAIMPLFPUSTR80TOI64;
1732
1733IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val));
1734FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fist_r80_to_i64;
1735FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fistt_r80_to_i64;
1736/** @} */
1737
1738
1739/** Temporary type representing a 256-bit vector register. */
1740typedef struct { uint64_t au64[4]; } IEMVMM256;
1741/** Temporary type pointing to a 256-bit vector register. */
1742typedef IEMVMM256 *PIEMVMM256;
1743/** Temporary type pointing to a const 256-bit vector register. */
1744typedef IEMVMM256 *PCIEMVMM256;
1745
1746
1747/** @name Media (SSE/MMX/AVX) operations: full1 + full2 -> full1.
1748 * @{ */
1749typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc));
1750typedef FNIEMAIMPLMEDIAF2U64 *PFNIEMAIMPLMEDIAF2U64;
1751typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
1752typedef FNIEMAIMPLMEDIAF2U128 *PFNIEMAIMPLMEDIAF2U128;
1753typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U128,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1754typedef FNIEMAIMPLMEDIAF3U128 *PFNIEMAIMPLMEDIAF3U128;
1755typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U256,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1756typedef FNIEMAIMPLMEDIAF3U256 *PFNIEMAIMPLMEDIAF3U256;
1757typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U64,(uint64_t *puDst, uint64_t const *puSrc));
1758typedef FNIEMAIMPLMEDIAOPTF2U64 *PFNIEMAIMPLMEDIAOPTF2U64;
1759typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U128,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1760typedef FNIEMAIMPLMEDIAOPTF2U128 *PFNIEMAIMPLMEDIAOPTF2U128;
1761typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U128,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1762typedef FNIEMAIMPLMEDIAOPTF3U128 *PFNIEMAIMPLMEDIAOPTF3U128;
1763typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U256,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1764typedef FNIEMAIMPLMEDIAOPTF3U256 *PFNIEMAIMPLMEDIAOPTF3U256;
1765typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U256,(PRTUINT256U puDst, PCRTUINT256U puSrc));
1766typedef FNIEMAIMPLMEDIAOPTF2U256 *PFNIEMAIMPLMEDIAOPTF2U256;
1767FNIEMAIMPLMEDIAF2U64 iemAImpl_pshufb_u64, iemAImpl_pshufb_u64_fallback;
1768FNIEMAIMPLMEDIAF2U64 iemAImpl_pand_u64, iemAImpl_pandn_u64, iemAImpl_por_u64, iemAImpl_pxor_u64;
1769FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqd_u64;
1770FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpgtb_u64, iemAImpl_pcmpgtw_u64, iemAImpl_pcmpgtd_u64;
1771FNIEMAIMPLMEDIAF2U64 iemAImpl_paddb_u64, iemAImpl_paddsb_u64, iemAImpl_paddusb_u64;
1772FNIEMAIMPLMEDIAF2U64 iemAImpl_paddw_u64, iemAImpl_paddsw_u64, iemAImpl_paddusw_u64;
1773FNIEMAIMPLMEDIAF2U64 iemAImpl_paddd_u64;
1774FNIEMAIMPLMEDIAF2U64 iemAImpl_paddq_u64;
1775FNIEMAIMPLMEDIAF2U64 iemAImpl_psubb_u64, iemAImpl_psubsb_u64, iemAImpl_psubusb_u64;
1776FNIEMAIMPLMEDIAF2U64 iemAImpl_psubw_u64, iemAImpl_psubsw_u64, iemAImpl_psubusw_u64;
1777FNIEMAIMPLMEDIAF2U64 iemAImpl_psubd_u64;
1778FNIEMAIMPLMEDIAF2U64 iemAImpl_psubq_u64;
1779FNIEMAIMPLMEDIAF2U64 iemAImpl_pmaddwd_u64;
1780FNIEMAIMPLMEDIAF2U64 iemAImpl_pmullw_u64, iemAImpl_pmulhw_u64;
1781FNIEMAIMPLMEDIAF2U64 iemAImpl_pminub_u64, iemAImpl_pmaxub_u64;
1782FNIEMAIMPLMEDIAF2U64 iemAImpl_pminsw_u64, iemAImpl_pmaxsw_u64;
1783FNIEMAIMPLMEDIAF2U64 iemAImpl_pabsb_u64, iemAImpl_pabsb_u64_fallback;
1784FNIEMAIMPLMEDIAF2U64 iemAImpl_pabsw_u64, iemAImpl_pabsw_u64_fallback;
1785FNIEMAIMPLMEDIAF2U64 iemAImpl_pabsd_u64, iemAImpl_pabsd_u64_fallback;
1786FNIEMAIMPLMEDIAF2U64 iemAImpl_psignb_u64, iemAImpl_psignb_u64_fallback;
1787FNIEMAIMPLMEDIAF2U64 iemAImpl_psignw_u64, iemAImpl_psignw_u64_fallback;
1788FNIEMAIMPLMEDIAF2U64 iemAImpl_psignd_u64, iemAImpl_psignd_u64_fallback;
1789FNIEMAIMPLMEDIAF2U64 iemAImpl_phaddw_u64, iemAImpl_phaddw_u64_fallback;
1790FNIEMAIMPLMEDIAF2U64 iemAImpl_phaddd_u64, iemAImpl_phaddd_u64_fallback;
1791FNIEMAIMPLMEDIAF2U64 iemAImpl_phsubw_u64, iemAImpl_phsubw_u64_fallback;
1792FNIEMAIMPLMEDIAF2U64 iemAImpl_phsubd_u64, iemAImpl_phsubd_u64_fallback;
1793FNIEMAIMPLMEDIAF2U64 iemAImpl_phaddsw_u64, iemAImpl_phaddsw_u64_fallback;
1794FNIEMAIMPLMEDIAF2U64 iemAImpl_phsubsw_u64, iemAImpl_phsubsw_u64_fallback;
1795FNIEMAIMPLMEDIAF2U64 iemAImpl_pmaddubsw_u64, iemAImpl_pmaddubsw_u64_fallback;
1796FNIEMAIMPLMEDIAF2U64 iemAImpl_pmulhrsw_u64, iemAImpl_pmulhrsw_u64_fallback;
1797FNIEMAIMPLMEDIAF2U64 iemAImpl_pmuludq_u64;
1798FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_psllw_u64, iemAImpl_psrlw_u64, iemAImpl_psraw_u64;
1799FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_pslld_u64, iemAImpl_psrld_u64, iemAImpl_psrad_u64;
1800FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_psllq_u64, iemAImpl_psrlq_u64;
1801FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packsswb_u64, iemAImpl_packuswb_u64;
1802FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packssdw_u64;
1803FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_pmulhuw_u64;
1804FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_pavgb_u64, iemAImpl_pavgw_u64;
1805FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_psadbw_u64;
1806
1807FNIEMAIMPLMEDIAF2U128 iemAImpl_pshufb_u128, iemAImpl_pshufb_u128_fallback;
1808FNIEMAIMPLMEDIAF2U128 iemAImpl_pand_u128, iemAImpl_pandn_u128, iemAImpl_por_u128, iemAImpl_pxor_u128;
1809FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqb_u128, iemAImpl_pcmpeqw_u128, iemAImpl_pcmpeqd_u128;
1810FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqq_u128, iemAImpl_pcmpeqq_u128_fallback;
1811FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtb_u128, iemAImpl_pcmpgtw_u128, iemAImpl_pcmpgtd_u128;
1812FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtq_u128, iemAImpl_pcmpgtq_u128_fallback;
1813FNIEMAIMPLMEDIAF2U128 iemAImpl_paddb_u128, iemAImpl_paddsb_u128, iemAImpl_paddusb_u128;
1814FNIEMAIMPLMEDIAF2U128 iemAImpl_paddw_u128, iemAImpl_paddsw_u128, iemAImpl_paddusw_u128;
1815FNIEMAIMPLMEDIAF2U128 iemAImpl_paddd_u128;
1816FNIEMAIMPLMEDIAF2U128 iemAImpl_paddq_u128;
1817FNIEMAIMPLMEDIAF2U128 iemAImpl_psubb_u128, iemAImpl_psubsb_u128, iemAImpl_psubusb_u128;
1818FNIEMAIMPLMEDIAF2U128 iemAImpl_psubw_u128, iemAImpl_psubsw_u128, iemAImpl_psubusw_u128;
1819FNIEMAIMPLMEDIAF2U128 iemAImpl_psubd_u128;
1820FNIEMAIMPLMEDIAF2U128 iemAImpl_psubq_u128;
1821FNIEMAIMPLMEDIAF2U128 iemAImpl_pmullw_u128, iemAImpl_pmullw_u128_fallback;
1822FNIEMAIMPLMEDIAF2U128 iemAImpl_pmulhw_u128;
1823FNIEMAIMPLMEDIAF2U128 iemAImpl_pmulld_u128, iemAImpl_pmulld_u128_fallback;
1824FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaddwd_u128;
1825FNIEMAIMPLMEDIAF2U128 iemAImpl_pminub_u128;
1826FNIEMAIMPLMEDIAF2U128 iemAImpl_pminud_u128, iemAImpl_pminud_u128_fallback;
1827FNIEMAIMPLMEDIAF2U128 iemAImpl_pminuw_u128, iemAImpl_pminuw_u128_fallback;
1828FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsb_u128, iemAImpl_pminsb_u128_fallback;
1829FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsd_u128, iemAImpl_pminsd_u128_fallback;
1830FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsw_u128, iemAImpl_pminsw_u128_fallback;
1831FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxub_u128;
1832FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxud_u128, iemAImpl_pmaxud_u128_fallback;
1833FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxuw_u128, iemAImpl_pmaxuw_u128_fallback;
1834FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsb_u128, iemAImpl_pmaxsb_u128_fallback;
1835FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsw_u128;
1836FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsd_u128, iemAImpl_pmaxsd_u128_fallback;
1837FNIEMAIMPLMEDIAF2U128 iemAImpl_pabsb_u128, iemAImpl_pabsb_u128_fallback;
1838FNIEMAIMPLMEDIAF2U128 iemAImpl_pabsw_u128, iemAImpl_pabsw_u128_fallback;
1839FNIEMAIMPLMEDIAF2U128 iemAImpl_pabsd_u128, iemAImpl_pabsd_u128_fallback;
1840FNIEMAIMPLMEDIAF2U128 iemAImpl_psignb_u128, iemAImpl_psignb_u128_fallback;
1841FNIEMAIMPLMEDIAF2U128 iemAImpl_psignw_u128, iemAImpl_psignw_u128_fallback;
1842FNIEMAIMPLMEDIAF2U128 iemAImpl_psignd_u128, iemAImpl_psignd_u128_fallback;
1843FNIEMAIMPLMEDIAF2U128 iemAImpl_phaddw_u128, iemAImpl_phaddw_u128_fallback;
1844FNIEMAIMPLMEDIAF2U128 iemAImpl_phaddd_u128, iemAImpl_phaddd_u128_fallback;
1845FNIEMAIMPLMEDIAF2U128 iemAImpl_phsubw_u128, iemAImpl_phsubw_u128_fallback;
1846FNIEMAIMPLMEDIAF2U128 iemAImpl_phsubd_u128, iemAImpl_phsubd_u128_fallback;
1847FNIEMAIMPLMEDIAF2U128 iemAImpl_phaddsw_u128, iemAImpl_phaddsw_u128_fallback;
1848FNIEMAIMPLMEDIAF2U128 iemAImpl_phsubsw_u128, iemAImpl_phsubsw_u128_fallback;
1849FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaddubsw_u128, iemAImpl_pmaddubsw_u128_fallback;
1850FNIEMAIMPLMEDIAF2U128 iemAImpl_pmulhrsw_u128, iemAImpl_pmulhrsw_u128_fallback;
1851FNIEMAIMPLMEDIAF2U128 iemAImpl_pmuludq_u128;
1852FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packsswb_u128, iemAImpl_packuswb_u128;
1853FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packssdw_u128, iemAImpl_packusdw_u128;
1854FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_psllw_u128, iemAImpl_psrlw_u128, iemAImpl_psraw_u128;
1855FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pslld_u128, iemAImpl_psrld_u128, iemAImpl_psrad_u128;
1856FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_psllq_u128, iemAImpl_psrlq_u128;
1857FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pmulhuw_u128;
1858FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pavgb_u128, iemAImpl_pavgw_u128;
1859FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_psadbw_u128;
1860FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pmuldq_u128, iemAImpl_pmuldq_u128_fallback;
1861FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_unpcklps_u128, iemAImpl_unpcklpd_u128;
1862FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_unpckhps_u128, iemAImpl_unpckhpd_u128;
1863
1864FNIEMAIMPLMEDIAF3U128 iemAImpl_vpshufb_u128, iemAImpl_vpshufb_u128_fallback;
1865FNIEMAIMPLMEDIAF3U128 iemAImpl_vpand_u128, iemAImpl_vpand_u128_fallback;
1866FNIEMAIMPLMEDIAF3U128 iemAImpl_vpandn_u128, iemAImpl_vpandn_u128_fallback;
1867FNIEMAIMPLMEDIAF3U128 iemAImpl_vpor_u128, iemAImpl_vpor_u128_fallback;
1868FNIEMAIMPLMEDIAF3U128 iemAImpl_vpxor_u128, iemAImpl_vpxor_u128_fallback;
1869FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqb_u128, iemAImpl_vpcmpeqb_u128_fallback;
1870FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqw_u128, iemAImpl_vpcmpeqw_u128_fallback;
1871FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqd_u128, iemAImpl_vpcmpeqd_u128_fallback;
1872FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqq_u128, iemAImpl_vpcmpeqq_u128_fallback;
1873FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtb_u128, iemAImpl_vpcmpgtb_u128_fallback;
1874FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtw_u128, iemAImpl_vpcmpgtw_u128_fallback;
1875FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtd_u128, iemAImpl_vpcmpgtd_u128_fallback;
1876FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtq_u128, iemAImpl_vpcmpgtq_u128_fallback;
1877FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddb_u128, iemAImpl_vpaddb_u128_fallback;
1878FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddw_u128, iemAImpl_vpaddw_u128_fallback;
1879FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddd_u128, iemAImpl_vpaddd_u128_fallback;
1880FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddq_u128, iemAImpl_vpaddq_u128_fallback;
1881FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubb_u128, iemAImpl_vpsubb_u128_fallback;
1882FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubw_u128, iemAImpl_vpsubw_u128_fallback;
1883FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubd_u128, iemAImpl_vpsubd_u128_fallback;
1884FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubq_u128, iemAImpl_vpsubq_u128_fallback;
1885FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminub_u128, iemAImpl_vpminub_u128_fallback;
1886FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminuw_u128, iemAImpl_vpminuw_u128_fallback;
1887FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminud_u128, iemAImpl_vpminud_u128_fallback;
1888FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsb_u128, iemAImpl_vpminsb_u128_fallback;
1889FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsw_u128, iemAImpl_vpminsw_u128_fallback;
1890FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsd_u128, iemAImpl_vpminsd_u128_fallback;
1891FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxub_u128, iemAImpl_vpmaxub_u128_fallback;
1892FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxuw_u128, iemAImpl_vpmaxuw_u128_fallback;
1893FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxud_u128, iemAImpl_vpmaxud_u128_fallback;
1894FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsb_u128, iemAImpl_vpmaxsb_u128_fallback;
1895FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsw_u128, iemAImpl_vpmaxsw_u128_fallback;
1896FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsd_u128, iemAImpl_vpmaxsd_u128_fallback;
1897FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpacksswb_u128, iemAImpl_vpacksswb_u128_fallback;
1898FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackssdw_u128, iemAImpl_vpackssdw_u128_fallback;
1899FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackuswb_u128, iemAImpl_vpackuswb_u128_fallback;
1900FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackusdw_u128, iemAImpl_vpackusdw_u128_fallback;
1901FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmullw_u128, iemAImpl_vpmullw_u128_fallback;
1902FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulld_u128, iemAImpl_vpmulld_u128_fallback;
1903FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulhw_u128, iemAImpl_vpmulhw_u128_fallback;
1904FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulhuw_u128, iemAImpl_vpmulhuw_u128_fallback;
1905FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpavgb_u128, iemAImpl_vpavgb_u128_fallback;
1906FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpavgw_u128, iemAImpl_vpavgw_u128_fallback;
1907FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpsignb_u128, iemAImpl_vpsignb_u128_fallback;
1908FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpsignw_u128, iemAImpl_vpsignw_u128_fallback;
1909FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpsignd_u128, iemAImpl_vpsignd_u128_fallback;
1910FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphaddw_u128, iemAImpl_vphaddw_u128_fallback;
1911FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphaddd_u128, iemAImpl_vphaddd_u128_fallback;
1912FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphsubw_u128, iemAImpl_vphsubw_u128_fallback;
1913FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphsubd_u128, iemAImpl_vphsubd_u128_fallback;
1914FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphaddsw_u128, iemAImpl_vphaddsw_u128_fallback;
1915FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vphsubsw_u128, iemAImpl_vphsubsw_u128_fallback;
1916FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmaddubsw_u128, iemAImpl_vpmaddubsw_u128_fallback;
1917FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulhrsw_u128, iemAImpl_vpmulhrsw_u128_fallback;
1918FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpsadbw_u128, iemAImpl_vpsadbw_u128_fallback;
1919FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmuldq_u128, iemAImpl_vpmuldq_u128_fallback;
1920FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmuludq_u128, iemAImpl_vpmuludq_u128_fallback;
1921
1922FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_vpabsb_u128, iemAImpl_vpabsb_u128_fallback;
1923FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_vpabsw_u128, iemAImpl_vpabsd_u128_fallback;
1924FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_vpabsd_u128, iemAImpl_vpabsw_u128_fallback;
1925
1926FNIEMAIMPLMEDIAF3U256 iemAImpl_vpshufb_u256, iemAImpl_vpshufb_u256_fallback;
1927FNIEMAIMPLMEDIAF3U256 iemAImpl_vpand_u256, iemAImpl_vpand_u256_fallback;
1928FNIEMAIMPLMEDIAF3U256 iemAImpl_vpandn_u256, iemAImpl_vpandn_u256_fallback;
1929FNIEMAIMPLMEDIAF3U256 iemAImpl_vpor_u256, iemAImpl_vpor_u256_fallback;
1930FNIEMAIMPLMEDIAF3U256 iemAImpl_vpxor_u256, iemAImpl_vpxor_u256_fallback;
1931FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqb_u256, iemAImpl_vpcmpeqb_u256_fallback;
1932FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqw_u256, iemAImpl_vpcmpeqw_u256_fallback;
1933FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqd_u256, iemAImpl_vpcmpeqd_u256_fallback;
1934FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqq_u256, iemAImpl_vpcmpeqq_u256_fallback;
1935FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtb_u256, iemAImpl_vpcmpgtb_u256_fallback;
1936FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtw_u256, iemAImpl_vpcmpgtw_u256_fallback;
1937FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtd_u256, iemAImpl_vpcmpgtd_u256_fallback;
1938FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtq_u256, iemAImpl_vpcmpgtq_u256_fallback;
1939FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddb_u256, iemAImpl_vpaddb_u256_fallback;
1940FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddw_u256, iemAImpl_vpaddw_u256_fallback;
1941FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddd_u256, iemAImpl_vpaddd_u256_fallback;
1942FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddq_u256, iemAImpl_vpaddq_u256_fallback;
1943FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubb_u256, iemAImpl_vpsubb_u256_fallback;
1944FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubw_u256, iemAImpl_vpsubw_u256_fallback;
1945FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubd_u256, iemAImpl_vpsubd_u256_fallback;
1946FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubq_u256, iemAImpl_vpsubq_u256_fallback;
1947FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminub_u256, iemAImpl_vpminub_u256_fallback;
1948FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminuw_u256, iemAImpl_vpminuw_u256_fallback;
1949FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminud_u256, iemAImpl_vpminud_u256_fallback;
1950FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsb_u256, iemAImpl_vpminsb_u256_fallback;
1951FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsw_u256, iemAImpl_vpminsw_u256_fallback;
1952FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsd_u256, iemAImpl_vpminsd_u256_fallback;
1953FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxub_u256, iemAImpl_vpmaxub_u256_fallback;
1954FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxuw_u256, iemAImpl_vpmaxuw_u256_fallback;
1955FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxud_u256, iemAImpl_vpmaxud_u256_fallback;
1956FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsb_u256, iemAImpl_vpmaxsb_u256_fallback;
1957FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsw_u256, iemAImpl_vpmaxsw_u256_fallback;
1958FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsd_u256, iemAImpl_vpmaxsd_u256_fallback;
1959FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpacksswb_u256, iemAImpl_vpacksswb_u256_fallback;
1960FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackssdw_u256, iemAImpl_vpackssdw_u256_fallback;
1961FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackuswb_u256, iemAImpl_vpackuswb_u256_fallback;
1962FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackusdw_u256, iemAImpl_vpackusdw_u256_fallback;
1963FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmullw_u256, iemAImpl_vpmullw_u256_fallback;
1964FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulld_u256, iemAImpl_vpmulld_u256_fallback;
1965FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulhw_u256, iemAImpl_vpmulhw_u256_fallback;
1966FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulhuw_u256, iemAImpl_vpmulhuw_u256_fallback;
1967FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpavgb_u256, iemAImpl_vpavgb_u256_fallback;
1968FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpavgw_u256, iemAImpl_vpavgw_u256_fallback;
1969FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpsignb_u256, iemAImpl_vpsignb_u256_fallback;
1970FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpsignw_u256, iemAImpl_vpsignw_u256_fallback;
1971FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpsignd_u256, iemAImpl_vpsignd_u256_fallback;
1972FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphaddw_u256, iemAImpl_vphaddw_u256_fallback;
1973FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphaddd_u256, iemAImpl_vphaddd_u256_fallback;
1974FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphsubw_u256, iemAImpl_vphsubw_u256_fallback;
1975FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphsubd_u256, iemAImpl_vphsubd_u256_fallback;
1976FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphaddsw_u256, iemAImpl_vphaddsw_u256_fallback;
1977FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vphsubsw_u256, iemAImpl_vphsubsw_u256_fallback;
1978FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmaddubsw_u256, iemAImpl_vpmaddubsw_u256_fallback;
1979FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulhrsw_u256, iemAImpl_vpmulhrsw_u256_fallback;
1980FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpsadbw_u256, iemAImpl_vpsadbw_u256_fallback;
1981FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmuldq_u256, iemAImpl_vpmuldq_u256_fallback;
1982FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmuludq_u256, iemAImpl_vpmuludq_u256_fallback;
1983
1984FNIEMAIMPLMEDIAOPTF2U256 iemAImpl_vpabsb_u256, iemAImpl_vpabsb_u256_fallback;
1985FNIEMAIMPLMEDIAOPTF2U256 iemAImpl_vpabsw_u256, iemAImpl_vpabsw_u256_fallback;
1986FNIEMAIMPLMEDIAOPTF2U256 iemAImpl_vpabsd_u256, iemAImpl_vpabsd_u256_fallback;
1987/** @} */
1988
1989/** @name Media (SSE/MMX/AVX) operations: lowhalf1 + lowhalf1 -> full1.
1990 * @{ */
1991FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpcklbw_u64, iemAImpl_punpcklwd_u64, iemAImpl_punpckldq_u64;
1992FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpcklbw_u128, iemAImpl_punpcklwd_u128, iemAImpl_punpckldq_u128, iemAImpl_punpcklqdq_u128;
1993FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpcklbw_u128, iemAImpl_vpunpcklbw_u128_fallback,
1994 iemAImpl_vpunpcklwd_u128, iemAImpl_vpunpcklwd_u128_fallback,
1995 iemAImpl_vpunpckldq_u128, iemAImpl_vpunpckldq_u128_fallback,
1996 iemAImpl_vpunpcklqdq_u128, iemAImpl_vpunpcklqdq_u128_fallback,
1997 iemAImpl_vunpcklps_u128, iemAImpl_vunpcklps_u128_fallback,
1998 iemAImpl_vunpcklpd_u128, iemAImpl_vunpcklpd_u128_fallback,
1999 iemAImpl_vunpckhps_u128, iemAImpl_vunpckhps_u128_fallback,
2000 iemAImpl_vunpckhpd_u128, iemAImpl_vunpckhpd_u128_fallback;
2001
2002FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpcklbw_u256, iemAImpl_vpunpcklbw_u256_fallback,
2003 iemAImpl_vpunpcklwd_u256, iemAImpl_vpunpcklwd_u256_fallback,
2004 iemAImpl_vpunpckldq_u256, iemAImpl_vpunpckldq_u256_fallback,
2005 iemAImpl_vpunpcklqdq_u256, iemAImpl_vpunpcklqdq_u256_fallback,
2006 iemAImpl_vunpcklps_u256, iemAImpl_vunpcklps_u256_fallback,
2007 iemAImpl_vunpcklpd_u256, iemAImpl_vunpcklpd_u256_fallback,
2008 iemAImpl_vunpckhps_u256, iemAImpl_vunpckhps_u256_fallback,
2009 iemAImpl_vunpckhpd_u256, iemAImpl_vunpckhpd_u256_fallback;
2010/** @} */
2011
2012/** @name Media (SSE/MMX/AVX) operations: hihalf1 + hihalf2 -> full1.
2013 * @{ */
2014FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpckhbw_u64, iemAImpl_punpckhwd_u64, iemAImpl_punpckhdq_u64;
2015FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpckhbw_u128, iemAImpl_punpckhwd_u128, iemAImpl_punpckhdq_u128, iemAImpl_punpckhqdq_u128;
2016FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpckhbw_u128, iemAImpl_vpunpckhbw_u128_fallback,
2017 iemAImpl_vpunpckhwd_u128, iemAImpl_vpunpckhwd_u128_fallback,
2018 iemAImpl_vpunpckhdq_u128, iemAImpl_vpunpckhdq_u128_fallback,
2019 iemAImpl_vpunpckhqdq_u128, iemAImpl_vpunpckhqdq_u128_fallback;
2020FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpckhbw_u256, iemAImpl_vpunpckhbw_u256_fallback,
2021 iemAImpl_vpunpckhwd_u256, iemAImpl_vpunpckhwd_u256_fallback,
2022 iemAImpl_vpunpckhdq_u256, iemAImpl_vpunpckhdq_u256_fallback,
2023 iemAImpl_vpunpckhqdq_u256, iemAImpl_vpunpckhqdq_u256_fallback;
2024/** @} */
2025
2026/** @name Media (SSE/MMX/AVX) operation: Packed Shuffle Stuff (evil)
2027 * @{ */
2028typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil));
2029typedef FNIEMAIMPLMEDIAPSHUFU128 *PFNIEMAIMPLMEDIAPSHUFU128;
2030typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU256,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil));
2031typedef FNIEMAIMPLMEDIAPSHUFU256 *PFNIEMAIMPLMEDIAPSHUFU256;
2032IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw_u64,(uint64_t *puDst, uint64_t const *puSrc, uint8_t bEvil));
2033FNIEMAIMPLMEDIAPSHUFU128 iemAImpl_pshufhw_u128, iemAImpl_pshuflw_u128, iemAImpl_pshufd_u128;
2034#ifndef IEM_WITHOUT_ASSEMBLY
2035FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256, iemAImpl_vpshuflw_u256, iemAImpl_vpshufd_u256;
2036#endif
2037FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256_fallback, iemAImpl_vpshuflw_u256_fallback, iemAImpl_vpshufd_u256_fallback;
2038/** @} */
2039
2040/** @name Media (SSE/MMX/AVX) operation: Shift Immediate Stuff (evil)
2041 * @{ */
2042typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU64,(uint64_t *puDst, uint8_t bShift));
2043typedef FNIEMAIMPLMEDIAPSHIFTU64 *PFNIEMAIMPLMEDIAPSHIFTU64;
2044typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU128,(PRTUINT128U puDst, uint8_t bShift));
2045typedef FNIEMAIMPLMEDIAPSHIFTU128 *PFNIEMAIMPLMEDIAPSHIFTU128;
2046typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU256,(PRTUINT256U puDst, uint8_t bShift));
2047typedef FNIEMAIMPLMEDIAPSHIFTU256 *PFNIEMAIMPLMEDIAPSHIFTU256;
2048FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psllw_imm_u64, iemAImpl_pslld_imm_u64, iemAImpl_psllq_imm_u64;
2049FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psrlw_imm_u64, iemAImpl_psrld_imm_u64, iemAImpl_psrlq_imm_u64;
2050FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psraw_imm_u64, iemAImpl_psrad_imm_u64;
2051FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psllw_imm_u128, iemAImpl_pslld_imm_u128, iemAImpl_psllq_imm_u128;
2052FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psrlw_imm_u128, iemAImpl_psrld_imm_u128, iemAImpl_psrlq_imm_u128;
2053FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psraw_imm_u128, iemAImpl_psrad_imm_u128;
2054FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_pslldq_imm_u128, iemAImpl_psrldq_imm_u128;
2055/** @} */
2056
2057/** @name Media (SSE/MMX/AVX) operation: Move Byte Mask
2058 * @{ */
2059IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(uint64_t *pu64Dst, uint64_t const *puSrc));
2060IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(uint64_t *pu64Dst, PCRTUINT128U puSrc));
2061#ifndef IEM_WITHOUT_ASSEMBLY
2062IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
2063#endif
2064IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256_fallback,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
2065/** @} */
2066
2067/** @name Media (SSE/MMX/AVX) operation: Sort this later
2068 * @{ */
2069IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
2070IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
2071IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PRTUINT128U puDst, uint64_t uSrc));
2072
2073IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
2074IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
2075IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
2076IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
2077IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
2078IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
2079
2080IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxbw_u128,(PRTUINT128U puDst, uint64_t uSrc));
2081IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u128,(PRTUINT128U puDst, uint64_t uSrc));
2082IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2083IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2084IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2085
2086IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxbd_u128,(PRTUINT128U puDst, uint32_t uSrc));
2087IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u128,(PRTUINT128U puDst, uint32_t uSrc));
2088IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc));
2089IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2090IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2091
2092IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxbq_u128,(PRTUINT128U puDst, uint16_t uSrc));
2093IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u128,(PRTUINT128U puDst, uint16_t uSrc));
2094IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc));
2095IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2096IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2097
2098IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxwd_u128,(PRTUINT128U puDst, uint64_t uSrc));
2099IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u128,(PRTUINT128U puDst, uint64_t uSrc));
2100IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2101IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2102IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2103
2104IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxwq_u128,(PRTUINT128U puDst, uint32_t uSrc));
2105IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u128,(PRTUINT128U puDst, uint32_t uSrc));
2106IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc));
2107IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2108IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2109
2110IEM_DECL_IMPL_DEF(void, iemAImpl_pmovsxdq_u128,(PRTUINT128U puDst, uint64_t uSrc));
2111IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u128,(PRTUINT128U puDst, uint64_t uSrc));
2112IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2113IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2114IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2115
2116IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxbw_u128,(PRTUINT128U puDst, uint64_t uSrc));
2117IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u128,(PRTUINT128U puDst, uint64_t uSrc));
2118IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2119IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2120IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2121
2122IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxbd_u128,(PRTUINT128U puDst, uint32_t uSrc));
2123IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u128,(PRTUINT128U puDst, uint32_t uSrc));
2124IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc));
2125IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2126IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2127
2128IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxbq_u128,(PRTUINT128U puDst, uint16_t uSrc));
2129IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u128,(PRTUINT128U puDst, uint16_t uSrc));
2130IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc));
2131IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2132IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2133
2134IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxwd_u128,(PRTUINT128U puDst, uint64_t uSrc));
2135IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u128,(PRTUINT128U puDst, uint64_t uSrc));
2136IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2137IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2138IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2139
2140IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxwq_u128,(PRTUINT128U puDst, uint32_t uSrc));
2141IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u128,(PRTUINT128U puDst, uint32_t uSrc));
2142IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc));
2143IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2144IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2145
2146IEM_DECL_IMPL_DEF(void, iemAImpl_pmovzxdq_u128,(PRTUINT128U puDst, uint64_t uSrc));
2147IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u128,(PRTUINT128U puDst, uint64_t uSrc));
2148IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc));
2149IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u256,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2150IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc));
2151
2152/** @} */
2153
2154/** @name Media Odds and Ends
2155 * @{ */
2156typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U8,(uint32_t *puDst, uint8_t uSrc));
2157typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U16,(uint32_t *puDst, uint16_t uSrc));
2158typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U32,(uint32_t *puDst, uint32_t uSrc));
2159typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U64,(uint32_t *puDst, uint64_t uSrc));
2160FNIEMAIMPLCR32U8 iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback;
2161FNIEMAIMPLCR32U16 iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback;
2162FNIEMAIMPLCR32U32 iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback;
2163FNIEMAIMPLCR32U64 iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback;
2164
2165typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLF2EFL128,(PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint32_t *pEFlags));
2166typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLF2EFL256,(PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint32_t *pEFlags));
2167FNIEMAIMPLF2EFL128 iemAImpl_ptest_u128;
2168FNIEMAIMPLF2EFL256 iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback;
2169/** @} */
2170
2171
2172/** @name Function tables.
2173 * @{
2174 */
2175
2176/**
2177 * Function table for a binary operator providing implementation based on
2178 * operand size.
2179 */
2180typedef struct IEMOPBINSIZES
2181{
2182 PFNIEMAIMPLBINU8 pfnNormalU8, pfnLockedU8;
2183 PFNIEMAIMPLBINU16 pfnNormalU16, pfnLockedU16;
2184 PFNIEMAIMPLBINU32 pfnNormalU32, pfnLockedU32;
2185 PFNIEMAIMPLBINU64 pfnNormalU64, pfnLockedU64;
2186} IEMOPBINSIZES;
2187/** Pointer to a binary operator function table. */
2188typedef IEMOPBINSIZES const *PCIEMOPBINSIZES;
2189
2190
2191/**
2192 * Function table for a unary operator providing implementation based on
2193 * operand size.
2194 */
2195typedef struct IEMOPUNARYSIZES
2196{
2197 PFNIEMAIMPLUNARYU8 pfnNormalU8, pfnLockedU8;
2198 PFNIEMAIMPLUNARYU16 pfnNormalU16, pfnLockedU16;
2199 PFNIEMAIMPLUNARYU32 pfnNormalU32, pfnLockedU32;
2200 PFNIEMAIMPLUNARYU64 pfnNormalU64, pfnLockedU64;
2201} IEMOPUNARYSIZES;
2202/** Pointer to a unary operator function table. */
2203typedef IEMOPUNARYSIZES const *PCIEMOPUNARYSIZES;
2204
2205
2206/**
2207 * Function table for a shift operator providing implementation based on
2208 * operand size.
2209 */
2210typedef struct IEMOPSHIFTSIZES
2211{
2212 PFNIEMAIMPLSHIFTU8 pfnNormalU8;
2213 PFNIEMAIMPLSHIFTU16 pfnNormalU16;
2214 PFNIEMAIMPLSHIFTU32 pfnNormalU32;
2215 PFNIEMAIMPLSHIFTU64 pfnNormalU64;
2216} IEMOPSHIFTSIZES;
2217/** Pointer to a shift operator function table. */
2218typedef IEMOPSHIFTSIZES const *PCIEMOPSHIFTSIZES;
2219
2220
2221/**
2222 * Function table for a multiplication or division operation.
2223 */
2224typedef struct IEMOPMULDIVSIZES
2225{
2226 PFNIEMAIMPLMULDIVU8 pfnU8;
2227 PFNIEMAIMPLMULDIVU16 pfnU16;
2228 PFNIEMAIMPLMULDIVU32 pfnU32;
2229 PFNIEMAIMPLMULDIVU64 pfnU64;
2230} IEMOPMULDIVSIZES;
2231/** Pointer to a multiplication or division operation function table. */
2232typedef IEMOPMULDIVSIZES const *PCIEMOPMULDIVSIZES;
2233
2234
2235/**
2236 * Function table for a double precision shift operator providing implementation
2237 * based on operand size.
2238 */
2239typedef struct IEMOPSHIFTDBLSIZES
2240{
2241 PFNIEMAIMPLSHIFTDBLU16 pfnNormalU16;
2242 PFNIEMAIMPLSHIFTDBLU32 pfnNormalU32;
2243 PFNIEMAIMPLSHIFTDBLU64 pfnNormalU64;
2244} IEMOPSHIFTDBLSIZES;
2245/** Pointer to a double precision shift function table. */
2246typedef IEMOPSHIFTDBLSIZES const *PCIEMOPSHIFTDBLSIZES;
2247
2248
2249/**
2250 * Function table for media instruction taking two full sized media source
2251 * registers and one full sized destination register (AVX).
2252 */
2253typedef struct IEMOPMEDIAF3
2254{
2255 PFNIEMAIMPLMEDIAF3U128 pfnU128;
2256 PFNIEMAIMPLMEDIAF3U256 pfnU256;
2257} IEMOPMEDIAF3;
2258/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2259typedef IEMOPMEDIAF3 const *PCIEMOPMEDIAF3;
2260
2261/** @def IEMOPMEDIAF3_INIT_VARS_EX
2262 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2263 * given functions as initializers. For use in AVX functions where a pair of
2264 * functions are only used once and the function table need not be public. */
2265#ifndef TST_IEM_CHECK_MC
2266# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2267# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2268 static IEMOPMEDIAF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2269 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2270# else
2271# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2272 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2273# endif
2274#else
2275# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2276#endif
2277/** @def IEMOPMEDIAF3_INIT_VARS
2278 * Generate AVX function tables for the @a a_InstrNm instruction.
2279 * @sa IEMOPMEDIAF3_INIT_VARS_EX */
2280#define IEMOPMEDIAF3_INIT_VARS(a_InstrNm) \
2281 IEMOPMEDIAF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2282 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2283
2284/**
2285 * Function table for media instruction taking two full sized media source
2286 * registers and one full sized destination register, but no additional state
2287 * (AVX).
2288 */
2289typedef struct IEMOPMEDIAOPTF3
2290{
2291 PFNIEMAIMPLMEDIAOPTF3U128 pfnU128;
2292 PFNIEMAIMPLMEDIAOPTF3U256 pfnU256;
2293} IEMOPMEDIAOPTF3;
2294/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2295typedef IEMOPMEDIAOPTF3 const *PCIEMOPMEDIAOPTF3;
2296
2297/** @def IEMOPMEDIAOPTF3_INIT_VARS_EX
2298 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2299 * given functions as initializers. For use in AVX functions where a pair of
2300 * functions are only used once and the function table need not be public. */
2301#ifndef TST_IEM_CHECK_MC
2302# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2303# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2304 static IEMOPMEDIAOPTF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2305 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2306# else
2307# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2308 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2309# endif
2310#else
2311# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2312#endif
2313/** @def IEMOPMEDIAOPTF3_INIT_VARS
2314 * Generate AVX function tables for the @a a_InstrNm instruction.
2315 * @sa IEMOPMEDIAOPTF3_INIT_VARS_EX */
2316#define IEMOPMEDIAOPTF3_INIT_VARS(a_InstrNm) \
2317 IEMOPMEDIAOPTF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2318 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2319
2320/**
2321 * Function table for media instruction taking one full sized media source
2322 * registers and one full sized destination register, but no additional state
2323 * (AVX).
2324 */
2325typedef struct IEMOPMEDIAOPTF2
2326{
2327 PFNIEMAIMPLMEDIAOPTF2U128 pfnU128;
2328 PFNIEMAIMPLMEDIAOPTF2U256 pfnU256;
2329} IEMOPMEDIAOPTF2;
2330/** Pointer to a media operation function table for 2 full sized ops (AVX). */
2331typedef IEMOPMEDIAOPTF2 const *PCIEMOPMEDIAOPTF2;
2332
2333/** @def IEMOPMEDIAOPTF2_INIT_VARS_EX
2334 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2335 * given functions as initializers. For use in AVX functions where a pair of
2336 * functions are only used once and the function table need not be public. */
2337#ifndef TST_IEM_CHECK_MC
2338# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2339# define IEMOPMEDIAOPTF2_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2340 static IEMOPMEDIAOPTF2 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2341 static IEMOPMEDIAOPTF2 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2342# else
2343# define IEMOPMEDIAOPTF2_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2344 static IEMOPMEDIAOPTF2 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2345# endif
2346#else
2347# define IEMOPMEDIAOPTF2_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2348#endif
2349/** @def IEMOPMEDIAOPTF2_INIT_VARS
2350 * Generate AVX function tables for the @a a_InstrNm instruction.
2351 * @sa IEMOPMEDIAOPTF2_INIT_VARS_EX */
2352#define IEMOPMEDIAOPTF2_INIT_VARS(a_InstrNm) \
2353 IEMOPMEDIAOPTF2_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2354 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2355/** @} */
2356
2357
2358/** @name SSE/AVX single/double precision floating point operations.
2359 * @{ */
2360/**
2361 * A SSE result.
2362 */
2363typedef struct IEMSSERESULT
2364{
2365 /** The output value. */
2366 X86XMMREG uResult;
2367 /** The output status. */
2368 uint32_t MXCSR;
2369} IEMSSERESULT;
2370AssertCompileMemberOffset(IEMSSERESULT, MXCSR, 128 / 8);
2371/** Pointer to a SSE result. */
2372typedef IEMSSERESULT *PIEMSSERESULT;
2373/** Pointer to a const SSE result. */
2374typedef IEMSSERESULT const *PCIEMSSERESULT;
2375
2376
2377/**
2378 * A AVX128 result.
2379 */
2380typedef struct IEMAVX128RESULT
2381{
2382 /** The output value. */
2383 X86XMMREG uResult;
2384 /** The output status. */
2385 uint32_t MXCSR;
2386} IEMAVX128RESULT;
2387AssertCompileMemberOffset(IEMAVX128RESULT, MXCSR, 128 / 8);
2388/** Pointer to a AVX128 result. */
2389typedef IEMAVX128RESULT *PIEMAVX128RESULT;
2390/** Pointer to a const AVX128 result. */
2391typedef IEMAVX128RESULT const *PCIEMAVX128RESULT;
2392
2393
2394/**
2395 * A AVX256 result.
2396 */
2397typedef struct IEMAVX256RESULT
2398{
2399 /** The output value. */
2400 X86YMMREG uResult;
2401 /** The output status. */
2402 uint32_t MXCSR;
2403} IEMAVX256RESULT;
2404AssertCompileMemberOffset(IEMAVX256RESULT, MXCSR, 256 / 8);
2405/** Pointer to a AVX256 result. */
2406typedef IEMAVX256RESULT *PIEMAVX256RESULT;
2407/** Pointer to a const AVX256 result. */
2408typedef IEMAVX256RESULT const *PCIEMAVX256RESULT;
2409
2410
2411typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPSSEF2U128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2));
2412typedef FNIEMAIMPLFPSSEF2U128 *PFNIEMAIMPLFPSSEF2U128;
2413typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPAVXF3U128,(PX86XSAVEAREA pExtState, PIEMAVX128RESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2));
2414typedef FNIEMAIMPLFPAVXF3U128 *PFNIEMAIMPLFPAVXF3U128;
2415typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPAVXF3U256,(PX86XSAVEAREA pExtState, PIEMAVX256RESULT pResult, PCX86YMMREG puSrc1, PCX86YMMREG puSrc2));
2416typedef FNIEMAIMPLFPAVXF3U256 *PFNIEMAIMPLFPAVXF3U256;
2417
2418FNIEMAIMPLFPSSEF2U128 iemAImpl_addps_u128;
2419FNIEMAIMPLFPSSEF2U128 iemAImpl_addpd_u128;
2420FNIEMAIMPLFPSSEF2U128 iemAImpl_mulps_u128;
2421FNIEMAIMPLFPSSEF2U128 iemAImpl_mulpd_u128;
2422FNIEMAIMPLFPSSEF2U128 iemAImpl_subps_u128;
2423FNIEMAIMPLFPSSEF2U128 iemAImpl_subpd_u128;
2424FNIEMAIMPLFPSSEF2U128 iemAImpl_minps_u128;
2425FNIEMAIMPLFPSSEF2U128 iemAImpl_minpd_u128;
2426FNIEMAIMPLFPSSEF2U128 iemAImpl_divps_u128;
2427FNIEMAIMPLFPSSEF2U128 iemAImpl_divpd_u128;
2428FNIEMAIMPLFPSSEF2U128 iemAImpl_maxps_u128;
2429FNIEMAIMPLFPSSEF2U128 iemAImpl_maxpd_u128;
2430
2431FNIEMAIMPLFPAVXF3U128 iemAImpl_vaddps_u128, iemAImpl_vaddps_u128_fallback;
2432FNIEMAIMPLFPAVXF3U128 iemAImpl_vaddpd_u128, iemAImpl_vaddpd_u128_fallback;
2433FNIEMAIMPLFPAVXF3U128 iemAImpl_vmulps_u128, iemAImpl_vmulps_u128_fallback;
2434FNIEMAIMPLFPAVXF3U128 iemAImpl_vmulpd_u128, iemAImpl_vmulpd_u128_fallback;
2435FNIEMAIMPLFPAVXF3U128 iemAImpl_vsubps_u128, iemAImpl_vsubps_u128_fallback;
2436FNIEMAIMPLFPAVXF3U128 iemAImpl_vsubpd_u128, iemAImpl_vsubpd_u128_fallback;
2437FNIEMAIMPLFPAVXF3U128 iemAImpl_vminps_u128, iemAImpl_vminps_u128_fallback;
2438FNIEMAIMPLFPAVXF3U128 iemAImpl_vminpd_u128, iemAImpl_vminpd_u128_fallback;
2439FNIEMAIMPLFPAVXF3U128 iemAImpl_vdivps_u128, iemAImpl_vdivps_u128_fallback;
2440FNIEMAIMPLFPAVXF3U128 iemAImpl_vdivpd_u128, iemAImpl_vdivpd_u128_fallback;
2441FNIEMAIMPLFPAVXF3U128 iemAImpl_vmaxps_u128, iemAImpl_vmaxps_u128_fallback;
2442FNIEMAIMPLFPAVXF3U128 iemAImpl_vmaxpd_u128, iemAImpl_vmaxpd_u128_fallback;
2443
2444FNIEMAIMPLFPAVXF3U256 iemAImpl_vaddps_u256, iemAImpl_vaddps_u256_fallback;
2445FNIEMAIMPLFPAVXF3U256 iemAImpl_vaddpd_u256, iemAImpl_vaddpd_u256_fallback;
2446FNIEMAIMPLFPAVXF3U256 iemAImpl_vmulps_u256, iemAImpl_vmulps_u256_fallback;
2447FNIEMAIMPLFPAVXF3U256 iemAImpl_vmulpd_u256, iemAImpl_vmulpd_u256_fallback;
2448FNIEMAIMPLFPAVXF3U256 iemAImpl_vsubps_u256, iemAImpl_vsubps_u256_fallback;
2449FNIEMAIMPLFPAVXF3U256 iemAImpl_vsubpd_u256, iemAImpl_vsubpd_u256_fallback;
2450FNIEMAIMPLFPAVXF3U256 iemAImpl_vminps_u256, iemAImpl_vminps_u256_fallback;
2451FNIEMAIMPLFPAVXF3U256 iemAImpl_vminpd_u256, iemAImpl_vminpd_u256_fallback;
2452FNIEMAIMPLFPAVXF3U256 iemAImpl_vdivps_u256, iemAImpl_vdivps_u256_fallback;
2453FNIEMAIMPLFPAVXF3U256 iemAImpl_vdivpd_u256, iemAImpl_vdivpd_u256_fallback;
2454FNIEMAIMPLFPAVXF3U256 iemAImpl_vmaxps_u256, iemAImpl_vmaxps_u256_fallback;
2455FNIEMAIMPLFPAVXF3U256 iemAImpl_vmaxpd_u256, iemAImpl_vmaxpd_u256_fallback;
2456/** @} */
2457
2458/** @name C instruction implementations for anything slightly complicated.
2459 * @{ */
2460
2461/**
2462 * For typedef'ing or declaring a C instruction implementation function taking
2463 * no extra arguments.
2464 *
2465 * @param a_Name The name of the type.
2466 */
2467# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2468 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2469/**
2470 * For defining a C instruction implementation function taking no extra
2471 * arguments.
2472 *
2473 * @param a_Name The name of the function
2474 */
2475# define IEM_CIMPL_DEF_0(a_Name) \
2476 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2477/**
2478 * Prototype version of IEM_CIMPL_DEF_0.
2479 */
2480# define IEM_CIMPL_PROTO_0(a_Name) \
2481 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2482/**
2483 * For calling a C instruction implementation function taking no extra
2484 * arguments.
2485 *
2486 * This special call macro adds default arguments to the call and allow us to
2487 * change these later.
2488 *
2489 * @param a_fn The name of the function.
2490 */
2491# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2492
2493/**
2494 * For typedef'ing or declaring a C instruction implementation function taking
2495 * one extra argument.
2496 *
2497 * @param a_Name The name of the type.
2498 * @param a_Type0 The argument type.
2499 * @param a_Arg0 The argument name.
2500 */
2501# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2502 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2503/**
2504 * For defining a C instruction implementation function taking one extra
2505 * argument.
2506 *
2507 * @param a_Name The name of the function
2508 * @param a_Type0 The argument type.
2509 * @param a_Arg0 The argument name.
2510 */
2511# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2512 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2513/**
2514 * Prototype version of IEM_CIMPL_DEF_1.
2515 */
2516# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2517 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2518/**
2519 * For calling a C instruction implementation function taking one extra
2520 * argument.
2521 *
2522 * This special call macro adds default arguments to the call and allow us to
2523 * change these later.
2524 *
2525 * @param a_fn The name of the function.
2526 * @param a0 The name of the 1st argument.
2527 */
2528# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2529
2530/**
2531 * For typedef'ing or declaring a C instruction implementation function taking
2532 * two extra arguments.
2533 *
2534 * @param a_Name The name of the type.
2535 * @param a_Type0 The type of the 1st argument
2536 * @param a_Arg0 The name of the 1st argument.
2537 * @param a_Type1 The type of the 2nd argument.
2538 * @param a_Arg1 The name of the 2nd argument.
2539 */
2540# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2541 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2542/**
2543 * For defining a C instruction implementation function taking two extra
2544 * arguments.
2545 *
2546 * @param a_Name The name of the function.
2547 * @param a_Type0 The type of the 1st argument
2548 * @param a_Arg0 The name of the 1st argument.
2549 * @param a_Type1 The type of the 2nd argument.
2550 * @param a_Arg1 The name of the 2nd argument.
2551 */
2552# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2553 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2554/**
2555 * Prototype version of IEM_CIMPL_DEF_2.
2556 */
2557# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2558 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2559/**
2560 * For calling a C instruction implementation function taking two extra
2561 * arguments.
2562 *
2563 * This special call macro adds default arguments to the call and allow us to
2564 * change these later.
2565 *
2566 * @param a_fn The name of the function.
2567 * @param a0 The name of the 1st argument.
2568 * @param a1 The name of the 2nd argument.
2569 */
2570# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
2571
2572/**
2573 * For typedef'ing or declaring a C instruction implementation function taking
2574 * three extra arguments.
2575 *
2576 * @param a_Name The name of the type.
2577 * @param a_Type0 The type of the 1st argument
2578 * @param a_Arg0 The name of the 1st argument.
2579 * @param a_Type1 The type of the 2nd argument.
2580 * @param a_Arg1 The name of the 2nd argument.
2581 * @param a_Type2 The type of the 3rd argument.
2582 * @param a_Arg2 The name of the 3rd argument.
2583 */
2584# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2585 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2586/**
2587 * For defining a C instruction implementation function taking three extra
2588 * arguments.
2589 *
2590 * @param a_Name The name of the function.
2591 * @param a_Type0 The type of the 1st argument
2592 * @param a_Arg0 The name of the 1st argument.
2593 * @param a_Type1 The type of the 2nd argument.
2594 * @param a_Arg1 The name of the 2nd argument.
2595 * @param a_Type2 The type of the 3rd argument.
2596 * @param a_Arg2 The name of the 3rd argument.
2597 */
2598# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2599 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2600/**
2601 * Prototype version of IEM_CIMPL_DEF_3.
2602 */
2603# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2604 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2605/**
2606 * For calling a C instruction implementation function taking three extra
2607 * arguments.
2608 *
2609 * This special call macro adds default arguments to the call and allow us to
2610 * change these later.
2611 *
2612 * @param a_fn The name of the function.
2613 * @param a0 The name of the 1st argument.
2614 * @param a1 The name of the 2nd argument.
2615 * @param a2 The name of the 3rd argument.
2616 */
2617# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
2618
2619
2620/**
2621 * For typedef'ing or declaring a C instruction implementation function taking
2622 * four extra arguments.
2623 *
2624 * @param a_Name The name of the type.
2625 * @param a_Type0 The type of the 1st argument
2626 * @param a_Arg0 The name of the 1st argument.
2627 * @param a_Type1 The type of the 2nd argument.
2628 * @param a_Arg1 The name of the 2nd argument.
2629 * @param a_Type2 The type of the 3rd argument.
2630 * @param a_Arg2 The name of the 3rd argument.
2631 * @param a_Type3 The type of the 4th argument.
2632 * @param a_Arg3 The name of the 4th argument.
2633 */
2634# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2635 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
2636/**
2637 * For defining a C instruction implementation function taking four extra
2638 * arguments.
2639 *
2640 * @param a_Name The name of the function.
2641 * @param a_Type0 The type of the 1st argument
2642 * @param a_Arg0 The name of the 1st argument.
2643 * @param a_Type1 The type of the 2nd argument.
2644 * @param a_Arg1 The name of the 2nd argument.
2645 * @param a_Type2 The type of the 3rd argument.
2646 * @param a_Arg2 The name of the 3rd argument.
2647 * @param a_Type3 The type of the 4th argument.
2648 * @param a_Arg3 The name of the 4th argument.
2649 */
2650# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2651 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2652 a_Type2 a_Arg2, a_Type3 a_Arg3))
2653/**
2654 * Prototype version of IEM_CIMPL_DEF_4.
2655 */
2656# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2657 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2658 a_Type2 a_Arg2, a_Type3 a_Arg3))
2659/**
2660 * For calling a C instruction implementation function taking four extra
2661 * arguments.
2662 *
2663 * This special call macro adds default arguments to the call and allow us to
2664 * change these later.
2665 *
2666 * @param a_fn The name of the function.
2667 * @param a0 The name of the 1st argument.
2668 * @param a1 The name of the 2nd argument.
2669 * @param a2 The name of the 3rd argument.
2670 * @param a3 The name of the 4th argument.
2671 */
2672# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
2673
2674
2675/**
2676 * For typedef'ing or declaring a C instruction implementation function taking
2677 * five extra arguments.
2678 *
2679 * @param a_Name The name of the type.
2680 * @param a_Type0 The type of the 1st argument
2681 * @param a_Arg0 The name of the 1st argument.
2682 * @param a_Type1 The type of the 2nd argument.
2683 * @param a_Arg1 The name of the 2nd argument.
2684 * @param a_Type2 The type of the 3rd argument.
2685 * @param a_Arg2 The name of the 3rd argument.
2686 * @param a_Type3 The type of the 4th argument.
2687 * @param a_Arg3 The name of the 4th argument.
2688 * @param a_Type4 The type of the 5th argument.
2689 * @param a_Arg4 The name of the 5th argument.
2690 */
2691# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2692 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
2693 a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
2694 a_Type3 a_Arg3, a_Type4 a_Arg4))
2695/**
2696 * For defining a C instruction implementation function taking five extra
2697 * arguments.
2698 *
2699 * @param a_Name The name of the function.
2700 * @param a_Type0 The type of the 1st argument
2701 * @param a_Arg0 The name of the 1st argument.
2702 * @param a_Type1 The type of the 2nd argument.
2703 * @param a_Arg1 The name of the 2nd argument.
2704 * @param a_Type2 The type of the 3rd argument.
2705 * @param a_Arg2 The name of the 3rd argument.
2706 * @param a_Type3 The type of the 4th argument.
2707 * @param a_Arg3 The name of the 4th argument.
2708 * @param a_Type4 The type of the 5th argument.
2709 * @param a_Arg4 The name of the 5th argument.
2710 */
2711# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2712 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2713 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2714/**
2715 * Prototype version of IEM_CIMPL_DEF_5.
2716 */
2717# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2718 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2719 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2720/**
2721 * For calling a C instruction implementation function taking five extra
2722 * arguments.
2723 *
2724 * This special call macro adds default arguments to the call and allow us to
2725 * change these later.
2726 *
2727 * @param a_fn The name of the function.
2728 * @param a0 The name of the 1st argument.
2729 * @param a1 The name of the 2nd argument.
2730 * @param a2 The name of the 3rd argument.
2731 * @param a3 The name of the 4th argument.
2732 * @param a4 The name of the 5th argument.
2733 */
2734# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2735
2736/** @} */
2737
2738
2739/** @name Opcode Decoder Function Types.
2740 * @{ */
2741
2742/** @typedef PFNIEMOP
2743 * Pointer to an opcode decoder function.
2744 */
2745
2746/** @def FNIEMOP_DEF
2747 * Define an opcode decoder function.
2748 *
2749 * We're using macors for this so that adding and removing parameters as well as
2750 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
2751 *
2752 * @param a_Name The function name.
2753 */
2754
2755/** @typedef PFNIEMOPRM
2756 * Pointer to an opcode decoder function with RM byte.
2757 */
2758
2759/** @def FNIEMOPRM_DEF
2760 * Define an opcode decoder function with RM byte.
2761 *
2762 * We're using macors for this so that adding and removing parameters as well as
2763 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
2764 *
2765 * @param a_Name The function name.
2766 */
2767
2768#if defined(__GNUC__) && defined(RT_ARCH_X86)
2769typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
2770typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2771# define FNIEMOP_DEF(a_Name) \
2772 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
2773# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2774 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2775# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2776 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2777
2778#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2779typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
2780typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2781# define FNIEMOP_DEF(a_Name) \
2782 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2783# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2784 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2785# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2786 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2787
2788#elif defined(__GNUC__)
2789typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2790typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2791# define FNIEMOP_DEF(a_Name) \
2792 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
2793# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2794 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2795# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2796 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2797
2798#else
2799typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2800typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2801# define FNIEMOP_DEF(a_Name) \
2802 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2803# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2804 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2805# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2806 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2807
2808#endif
2809#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
2810
2811/**
2812 * Call an opcode decoder function.
2813 *
2814 * We're using macors for this so that adding and removing parameters can be
2815 * done as we please. See FNIEMOP_DEF.
2816 */
2817#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
2818
2819/**
2820 * Call a common opcode decoder function taking one extra argument.
2821 *
2822 * We're using macors for this so that adding and removing parameters can be
2823 * done as we please. See FNIEMOP_DEF_1.
2824 */
2825#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
2826
2827/**
2828 * Call a common opcode decoder function taking one extra argument.
2829 *
2830 * We're using macors for this so that adding and removing parameters can be
2831 * done as we please. See FNIEMOP_DEF_1.
2832 */
2833#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
2834/** @} */
2835
2836
2837/** @name Misc Helpers
2838 * @{ */
2839
2840/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
2841 * due to GCC lacking knowledge about the value range of a switch. */
2842#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
2843
2844/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
2845#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
2846
2847/**
2848 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2849 * occation.
2850 */
2851#ifdef LOG_ENABLED
2852# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2853 do { \
2854 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
2855 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2856 } while (0)
2857#else
2858# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2859 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2860#endif
2861
2862/**
2863 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2864 * occation using the supplied logger statement.
2865 *
2866 * @param a_LoggerArgs What to log on failure.
2867 */
2868#ifdef LOG_ENABLED
2869# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2870 do { \
2871 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
2872 /*LogFunc(a_LoggerArgs);*/ \
2873 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2874 } while (0)
2875#else
2876# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2877 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2878#endif
2879
2880/**
2881 * Check if we're currently executing in real or virtual 8086 mode.
2882 *
2883 * @returns @c true if it is, @c false if not.
2884 * @param a_pVCpu The IEM state of the current CPU.
2885 */
2886#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2887
2888/**
2889 * Check if we're currently executing in virtual 8086 mode.
2890 *
2891 * @returns @c true if it is, @c false if not.
2892 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2893 */
2894#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2895
2896/**
2897 * Check if we're currently executing in long mode.
2898 *
2899 * @returns @c true if it is, @c false if not.
2900 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2901 */
2902#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
2903
2904/**
2905 * Check if we're currently executing in a 64-bit code segment.
2906 *
2907 * @returns @c true if it is, @c false if not.
2908 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2909 */
2910#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
2911
2912/**
2913 * Check if we're currently executing in real mode.
2914 *
2915 * @returns @c true if it is, @c false if not.
2916 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2917 */
2918#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
2919
2920/**
2921 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
2922 * @returns PCCPUMFEATURES
2923 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2924 */
2925#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
2926
2927/**
2928 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
2929 * @returns PCCPUMFEATURES
2930 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2931 */
2932#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&g_CpumHostFeatures.s)
2933
2934/**
2935 * Evaluates to true if we're presenting an Intel CPU to the guest.
2936 */
2937#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
2938
2939/**
2940 * Evaluates to true if we're presenting an AMD CPU to the guest.
2941 */
2942#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
2943
2944/**
2945 * Check if the address is canonical.
2946 */
2947#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
2948
2949/** Checks if the ModR/M byte is in register mode or not. */
2950#define IEM_IS_MODRM_REG_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT) )
2951/** Checks if the ModR/M byte is in memory mode or not. */
2952#define IEM_IS_MODRM_MEM_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT) )
2953
2954/**
2955 * Gets the register (reg) part of a ModR/M encoding, with REX.R added in.
2956 *
2957 * For use during decoding.
2958 */
2959#define IEM_GET_MODRM_REG(a_pVCpu, a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | (a_pVCpu)->iem.s.uRexReg )
2960/**
2961 * Gets the r/m part of a ModR/M encoding as a register index, with REX.B added in.
2962 *
2963 * For use during decoding.
2964 */
2965#define IEM_GET_MODRM_RM(a_pVCpu, a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) | (a_pVCpu)->iem.s.uRexB )
2966
2967/**
2968 * Gets the register (reg) part of a ModR/M encoding, without REX.R.
2969 *
2970 * For use during decoding.
2971 */
2972#define IEM_GET_MODRM_REG_8(a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) )
2973/**
2974 * Gets the r/m part of a ModR/M encoding as a register index, without REX.B.
2975 *
2976 * For use during decoding.
2977 */
2978#define IEM_GET_MODRM_RM_8(a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) )
2979
2980/**
2981 * Gets the effective VEX.VVVV value.
2982 *
2983 * The 4th bit is ignored if not 64-bit code.
2984 * @returns effective V-register value.
2985 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2986 */
2987#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
2988 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
2989
2990
2991#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2992
2993/**
2994 * Check if the guest has entered VMX root operation.
2995 */
2996# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
2997
2998/**
2999 * Check if the guest has entered VMX non-root operation.
3000 */
3001# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
3002
3003/**
3004 * Check if the nested-guest has the given Pin-based VM-execution control set.
3005 */
3006# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
3007 (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
3008
3009/**
3010 * Check if the nested-guest has the given Processor-based VM-execution control set.
3011 */
3012# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
3013 (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
3014
3015/**
3016 * Check if the nested-guest has the given Secondary Processor-based VM-execution
3017 * control set.
3018 */
3019# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
3020 (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
3021
3022/** Gets the guest-physical address of the shadows VMCS for the given VCPU. */
3023# define IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs)
3024
3025/** Whether a shadow VMCS is present for the given VCPU. */
3026# define IEM_VMX_HAS_SHADOW_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) != NIL_RTGCPHYS)
3027
3028/** Gets the VMXON region pointer. */
3029# define IEM_VMX_GET_VMXON_PTR(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
3030
3031/** Gets the guest-physical address of the current VMCS for the given VCPU. */
3032# define IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs)
3033
3034/** Whether a current VMCS is present for the given VCPU. */
3035# define IEM_VMX_HAS_CURRENT_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) != NIL_RTGCPHYS)
3036
3037/** Assigns the guest-physical address of the current VMCS for the given VCPU. */
3038# define IEM_VMX_SET_CURRENT_VMCS(a_pVCpu, a_GCPhysVmcs) \
3039 do \
3040 { \
3041 Assert((a_GCPhysVmcs) != NIL_RTGCPHYS); \
3042 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = (a_GCPhysVmcs); \
3043 } while (0)
3044
3045/** Clears any current VMCS for the given VCPU. */
3046# define IEM_VMX_CLEAR_CURRENT_VMCS(a_pVCpu) \
3047 do \
3048 { \
3049 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = NIL_RTGCPHYS; \
3050 } while (0)
3051
3052/**
3053 * Invokes the VMX VM-exit handler for an instruction intercept.
3054 */
3055# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
3056 do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
3057
3058/**
3059 * Invokes the VMX VM-exit handler for an instruction intercept where the
3060 * instruction provides additional VM-exit information.
3061 */
3062# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
3063 do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
3064
3065/**
3066 * Invokes the VMX VM-exit handler for a task switch.
3067 */
3068# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
3069 do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
3070
3071/**
3072 * Invokes the VMX VM-exit handler for MWAIT.
3073 */
3074# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
3075 do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
3076
3077/**
3078 * Invokes the VMX VM-exit handler for EPT faults.
3079 */
3080# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) \
3081 do { return iemVmxVmexitEpt(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr); } while (0)
3082
3083/**
3084 * Invokes the VMX VM-exit handler.
3085 */
3086# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
3087 do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
3088
3089#else
3090# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
3091# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
3092# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
3093# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
3094# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
3095# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
3096# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
3097# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
3098# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
3099# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
3100# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
3101
3102#endif
3103
3104#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3105/**
3106 * Check if an SVM control/instruction intercept is set.
3107 */
3108# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
3109 (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
3110
3111/**
3112 * Check if an SVM read CRx intercept is set.
3113 */
3114# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
3115 (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
3116
3117/**
3118 * Check if an SVM write CRx intercept is set.
3119 */
3120# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
3121 (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
3122
3123/**
3124 * Check if an SVM read DRx intercept is set.
3125 */
3126# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
3127 (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
3128
3129/**
3130 * Check if an SVM write DRx intercept is set.
3131 */
3132# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
3133 (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
3134
3135/**
3136 * Check if an SVM exception intercept is set.
3137 */
3138# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
3139 (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
3140
3141/**
3142 * Invokes the SVM \#VMEXIT handler for the nested-guest.
3143 */
3144# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
3145 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
3146
3147/**
3148 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
3149 * corresponding decode assist information.
3150 */
3151# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
3152 do \
3153 { \
3154 uint64_t uExitInfo1; \
3155 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
3156 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
3157 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
3158 else \
3159 uExitInfo1 = 0; \
3160 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
3161 } while (0)
3162
3163/** Check and handles SVM nested-guest instruction intercept and updates
3164 * NRIP if needed.
3165 */
3166# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
3167 do \
3168 { \
3169 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
3170 { \
3171 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
3172 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
3173 } \
3174 } while (0)
3175
3176/** Checks and handles SVM nested-guest CR0 read intercept. */
3177# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
3178 do \
3179 { \
3180 if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
3181 { /* probably likely */ } \
3182 else \
3183 { \
3184 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
3185 IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
3186 } \
3187 } while (0)
3188
3189/**
3190 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
3191 */
3192# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
3193 do { \
3194 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
3195 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
3196 } while (0)
3197
3198#else
3199# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
3200# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
3201# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
3202# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
3203# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
3204# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
3205# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
3206# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
3207# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
3208# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
3209# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
3210
3211#endif
3212
3213/** @} */
3214
3215
3216
3217/**
3218 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
3219 */
3220typedef union IEMSELDESC
3221{
3222 /** The legacy view. */
3223 X86DESC Legacy;
3224 /** The long mode view. */
3225 X86DESC64 Long;
3226} IEMSELDESC;
3227/** Pointer to a selector descriptor table entry. */
3228typedef IEMSELDESC *PIEMSELDESC;
3229
3230/** @name Raising Exceptions.
3231 * @{ */
3232VBOXSTRICTRC iemTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, uint32_t uNextEip, uint32_t fFlags,
3233 uint16_t uErr, uint64_t uCr2, RTSEL SelTSS, PIEMSELDESC pNewDescTSS) RT_NOEXCEPT;
3234
3235VBOXSTRICTRC iemRaiseXcptOrInt(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector, uint32_t fFlags,
3236 uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3237#ifdef IEM_WITH_SETJMP
3238DECL_NO_RETURN(void) iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector,
3239 uint32_t fFlags, uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3240#endif
3241VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT;
3242VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT;
3243VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT;
3244VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT;
3245VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT;
3246VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
3247VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT;
3248VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
3249VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3250/*VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;*/
3251VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
3252VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3253VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3254VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
3255VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
3256VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
3257#ifdef IEM_WITH_SETJMP
3258DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3259#endif
3260VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
3261VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT;
3262VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
3263#ifdef IEM_WITH_SETJMP
3264DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
3265#endif
3266VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
3267#ifdef IEM_WITH_SETJMP
3268DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
3269#endif
3270VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
3271#ifdef IEM_WITH_SETJMP
3272DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
3273#endif
3274VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
3275#ifdef IEM_WITH_SETJMP
3276DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
3277#endif
3278VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu);
3279VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
3280#ifdef IEM_WITH_SETJMP
3281DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3282#endif
3283VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT;
3284
3285IEM_CIMPL_DEF_0(iemCImplRaiseDivideError);
3286IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix);
3287IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode);
3288
3289/**
3290 * Macro for calling iemCImplRaiseDivideError().
3291 *
3292 * This enables us to add/remove arguments and force different levels of
3293 * inlining as we wish.
3294 *
3295 * @return Strict VBox status code.
3296 */
3297#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
3298
3299/**
3300 * Macro for calling iemCImplRaiseInvalidLockPrefix().
3301 *
3302 * This enables us to add/remove arguments and force different levels of
3303 * inlining as we wish.
3304 *
3305 * @return Strict VBox status code.
3306 */
3307#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
3308
3309/**
3310 * Macro for calling iemCImplRaiseInvalidOpcode().
3311 *
3312 * This enables us to add/remove arguments and force different levels of
3313 * inlining as we wish.
3314 *
3315 * @return Strict VBox status code.
3316 */
3317#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
3318/** @} */
3319
3320/** @name Register Access.
3321 * @{ */
3322VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT;
3323VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT;
3324VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT;
3325VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT;
3326/** @} */
3327
3328/** @name FPU access and helpers.
3329 * @{ */
3330void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT;
3331void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3332void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT;
3333void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
3334void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
3335void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
3336 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3337void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
3338 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3339void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3340void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3341void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3342void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3343void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3344void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3345void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
3346void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3347void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
3348void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3349void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT;
3350void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3351void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT;
3352void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3353void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3354/** @} */
3355
3356/** @name SSE+AVX SIMD access and helpers.
3357 * @{ */
3358void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT;
3359/** @} */
3360
3361/** @name Memory access.
3362 * @{ */
3363
3364/** Report a \#GP instead of \#AC and do not restrict to ring-3 */
3365#define IEM_MEMMAP_F_ALIGN_GP RT_BIT_32(16)
3366/** SSE access that should report a \#GP instead of \#AC, unless MXCSR.MM=1
3367 * when it works like normal \#AC. Always used with IEM_MEMMAP_F_ALIGN_GP. */
3368#define IEM_MEMMAP_F_ALIGN_SSE RT_BIT_32(17)
3369/** If \#AC is applicable, raise it. Always used with IEM_MEMMAP_F_ALIGN_GP.
3370 * Users include FXSAVE & FXRSTOR. */
3371#define IEM_MEMMAP_F_ALIGN_GP_OR_AC RT_BIT_32(18)
3372
3373VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
3374 uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT;
3375VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
3376#ifndef IN_RING3
3377VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
3378#endif
3379void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3380VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT;
3381VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3382VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
3383
3384#ifdef IEM_WITH_CODE_TLB
3385void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT;
3386#else
3387VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT;
3388#endif
3389#ifdef IEM_WITH_SETJMP
3390uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3391uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3392uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3393uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3394#else
3395VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT;
3396VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3397VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3398VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3399VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3400VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3401VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3402VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3403VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3404VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3405VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3406#endif
3407
3408VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3409VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3410VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3411VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3412VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3413VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3414VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3415VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3416VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3417VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3418VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3419VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3420VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
3421 RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT;
3422#ifdef IEM_WITH_SETJMP
3423uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3424uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3425uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3426uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3427uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3428void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3429void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3430void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3431void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3432void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3433void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3434#endif
3435
3436VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3437VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3438VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3439VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3440VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT;
3441
3442VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3443VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3444VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3445VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3446VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3447VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3448VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3449VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3450VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3451#ifdef IEM_WITH_SETJMP
3452void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3453void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3454void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3455void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3456void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3457void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3458void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3459void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3460#endif
3461
3462VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3463 void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3464VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT;
3465VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT;
3466VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3467VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT;
3468VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3469VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3470VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3471VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3472VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3473 void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3474VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3475VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT;
3476VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT;
3477VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT;
3478VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT;
3479VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3480VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3481VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3482/** @} */
3483
3484/** @name IEMAllCImpl.cpp
3485 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/'
3486 * @{ */
3487IEM_CIMPL_PROTO_0(iemCImpl_popa_16);
3488IEM_CIMPL_PROTO_0(iemCImpl_popa_32);
3489IEM_CIMPL_PROTO_0(iemCImpl_pusha_16);
3490IEM_CIMPL_PROTO_0(iemCImpl_pusha_32);
3491IEM_CIMPL_PROTO_1(iemCImpl_pushf, IEMMODE, enmEffOpSize);
3492IEM_CIMPL_PROTO_1(iemCImpl_popf, IEMMODE, enmEffOpSize);
3493IEM_CIMPL_PROTO_1(iemCImpl_call_16, uint16_t, uNewPC);
3494IEM_CIMPL_PROTO_1(iemCImpl_call_rel_16, int16_t, offDisp);
3495IEM_CIMPL_PROTO_1(iemCImpl_call_32, uint32_t, uNewPC);
3496IEM_CIMPL_PROTO_1(iemCImpl_call_rel_32, int32_t, offDisp);
3497IEM_CIMPL_PROTO_1(iemCImpl_call_64, uint64_t, uNewPC);
3498IEM_CIMPL_PROTO_1(iemCImpl_call_rel_64, int64_t, offDisp);
3499IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3500IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3501IEM_CIMPL_PROTO_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3502IEM_CIMPL_PROTO_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3503IEM_CIMPL_PROTO_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3504IEM_CIMPL_PROTO_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3505IEM_CIMPL_PROTO_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3506IEM_CIMPL_PROTO_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3507IEM_CIMPL_PROTO_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters);
3508IEM_CIMPL_PROTO_1(iemCImpl_leave, IEMMODE, enmEffOpSize);
3509IEM_CIMPL_PROTO_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt);
3510IEM_CIMPL_PROTO_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize);
3511IEM_CIMPL_PROTO_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp);
3512IEM_CIMPL_PROTO_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize);
3513IEM_CIMPL_PROTO_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize);
3514IEM_CIMPL_PROTO_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize);
3515IEM_CIMPL_PROTO_1(iemCImpl_iret, IEMMODE, enmEffOpSize);
3516IEM_CIMPL_PROTO_0(iemCImpl_loadall286);
3517IEM_CIMPL_PROTO_0(iemCImpl_syscall);
3518IEM_CIMPL_PROTO_0(iemCImpl_sysret);
3519IEM_CIMPL_PROTO_0(iemCImpl_sysenter);
3520IEM_CIMPL_PROTO_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize);
3521IEM_CIMPL_PROTO_2(iemCImpl_LoadSReg, uint8_t, iSegReg, uint16_t, uSel);
3522IEM_CIMPL_PROTO_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel);
3523IEM_CIMPL_PROTO_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize);
3524IEM_CIMPL_PROTO_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize);
3525IEM_CIMPL_PROTO_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite);
3526IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar);
3527IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar);
3528IEM_CIMPL_PROTO_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3529IEM_CIMPL_PROTO_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3530IEM_CIMPL_PROTO_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3531IEM_CIMPL_PROTO_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3532IEM_CIMPL_PROTO_1(iemCImpl_lldt, uint16_t, uNewLdt);
3533IEM_CIMPL_PROTO_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3534IEM_CIMPL_PROTO_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3535IEM_CIMPL_PROTO_1(iemCImpl_ltr, uint16_t, uNewTr);
3536IEM_CIMPL_PROTO_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3537IEM_CIMPL_PROTO_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3538IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg);
3539IEM_CIMPL_PROTO_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3540IEM_CIMPL_PROTO_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3541IEM_CIMPL_PROTO_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg);
3542IEM_CIMPL_PROTO_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg);
3543IEM_CIMPL_PROTO_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst);
3544IEM_CIMPL_PROTO_0(iemCImpl_clts);
3545IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg);
3546IEM_CIMPL_PROTO_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg);
3547IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg);
3548IEM_CIMPL_PROTO_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg);
3549IEM_CIMPL_PROTO_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage);
3550IEM_CIMPL_PROTO_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType);
3551IEM_CIMPL_PROTO_0(iemCImpl_invd);
3552IEM_CIMPL_PROTO_0(iemCImpl_wbinvd);
3553IEM_CIMPL_PROTO_0(iemCImpl_rsm);
3554IEM_CIMPL_PROTO_0(iemCImpl_rdtsc);
3555IEM_CIMPL_PROTO_0(iemCImpl_rdtscp);
3556IEM_CIMPL_PROTO_0(iemCImpl_rdpmc);
3557IEM_CIMPL_PROTO_0(iemCImpl_rdmsr);
3558IEM_CIMPL_PROTO_0(iemCImpl_wrmsr);
3559IEM_CIMPL_PROTO_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3560IEM_CIMPL_PROTO_1(iemCImpl_in_eAX_DX, uint8_t, cbReg);
3561IEM_CIMPL_PROTO_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3562IEM_CIMPL_PROTO_1(iemCImpl_out_DX_eAX, uint8_t, cbReg);
3563IEM_CIMPL_PROTO_0(iemCImpl_cli);
3564IEM_CIMPL_PROTO_0(iemCImpl_sti);
3565IEM_CIMPL_PROTO_0(iemCImpl_hlt);
3566IEM_CIMPL_PROTO_1(iemCImpl_monitor, uint8_t, iEffSeg);
3567IEM_CIMPL_PROTO_0(iemCImpl_mwait);
3568IEM_CIMPL_PROTO_0(iemCImpl_swapgs);
3569IEM_CIMPL_PROTO_0(iemCImpl_cpuid);
3570IEM_CIMPL_PROTO_1(iemCImpl_aad, uint8_t, bImm);
3571IEM_CIMPL_PROTO_1(iemCImpl_aam, uint8_t, bImm);
3572IEM_CIMPL_PROTO_0(iemCImpl_daa);
3573IEM_CIMPL_PROTO_0(iemCImpl_das);
3574IEM_CIMPL_PROTO_0(iemCImpl_aaa);
3575IEM_CIMPL_PROTO_0(iemCImpl_aas);
3576IEM_CIMPL_PROTO_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound);
3577IEM_CIMPL_PROTO_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound);
3578IEM_CIMPL_PROTO_0(iemCImpl_xgetbv);
3579IEM_CIMPL_PROTO_0(iemCImpl_xsetbv);
3580IEM_CIMPL_PROTO_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
3581 PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags);
3582IEM_CIMPL_PROTO_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3583IEM_CIMPL_PROTO_1(iemCImpl_finit, bool, fCheckXcpts);
3584IEM_CIMPL_PROTO_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3585IEM_CIMPL_PROTO_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3586IEM_CIMPL_PROTO_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3587IEM_CIMPL_PROTO_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3588IEM_CIMPL_PROTO_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3589IEM_CIMPL_PROTO_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3590IEM_CIMPL_PROTO_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3591IEM_CIMPL_PROTO_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3592IEM_CIMPL_PROTO_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3593IEM_CIMPL_PROTO_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3594IEM_CIMPL_PROTO_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3595IEM_CIMPL_PROTO_1(iemCImpl_fldcw, uint16_t, u16Fcw);
3596IEM_CIMPL_PROTO_1(iemCImpl_fxch_underflow, uint8_t, iStReg);
3597IEM_CIMPL_PROTO_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop);
3598/** @} */
3599
3600/** @name IEMAllCImplStrInstr.cpp.h
3601 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/' -e 's/RT_CONCAT4(//' \
3602 * -e 's/,ADDR_SIZE)/64/g' -e 's/,OP_SIZE,/64/g' -e 's/,OP_rAX,/rax/g' IEMAllCImplStrInstr.cpp.h
3603 * @{ */
3604IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr16, uint8_t, iEffSeg);
3605IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr16, uint8_t, iEffSeg);
3606IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m16);
3607IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m16);
3608IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr16, uint8_t, iEffSeg);
3609IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m16);
3610IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m16, int8_t, iEffSeg);
3611IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr16, bool, fIoChecked);
3612IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr16, bool, fIoChecked);
3613IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3614IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3615
3616IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr16, uint8_t, iEffSeg);
3617IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr16, uint8_t, iEffSeg);
3618IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m16);
3619IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m16);
3620IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr16, uint8_t, iEffSeg);
3621IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m16);
3622IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m16, int8_t, iEffSeg);
3623IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr16, bool, fIoChecked);
3624IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr16, bool, fIoChecked);
3625IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3626IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3627
3628IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr16, uint8_t, iEffSeg);
3629IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr16, uint8_t, iEffSeg);
3630IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m16);
3631IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m16);
3632IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr16, uint8_t, iEffSeg);
3633IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m16);
3634IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m16, int8_t, iEffSeg);
3635IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr16, bool, fIoChecked);
3636IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr16, bool, fIoChecked);
3637IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3638IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3639
3640
3641IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr32, uint8_t, iEffSeg);
3642IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr32, uint8_t, iEffSeg);
3643IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m32);
3644IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m32);
3645IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr32, uint8_t, iEffSeg);
3646IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m32);
3647IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m32, int8_t, iEffSeg);
3648IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr32, bool, fIoChecked);
3649IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr32, bool, fIoChecked);
3650IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3651IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3652
3653IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr32, uint8_t, iEffSeg);
3654IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr32, uint8_t, iEffSeg);
3655IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m32);
3656IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m32);
3657IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr32, uint8_t, iEffSeg);
3658IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m32);
3659IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m32, int8_t, iEffSeg);
3660IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr32, bool, fIoChecked);
3661IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr32, bool, fIoChecked);
3662IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3663IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3664
3665IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr32, uint8_t, iEffSeg);
3666IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr32, uint8_t, iEffSeg);
3667IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m32);
3668IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m32);
3669IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr32, uint8_t, iEffSeg);
3670IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m32);
3671IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m32, int8_t, iEffSeg);
3672IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr32, bool, fIoChecked);
3673IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr32, bool, fIoChecked);
3674IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3675IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3676
3677IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr32, uint8_t, iEffSeg);
3678IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr32, uint8_t, iEffSeg);
3679IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m32);
3680IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m32);
3681IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr32, uint8_t, iEffSeg);
3682IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m32);
3683IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m32, int8_t, iEffSeg);
3684IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr32, bool, fIoChecked);
3685IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr32, bool, fIoChecked);
3686IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3687IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3688
3689
3690IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr64, uint8_t, iEffSeg);
3691IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr64, uint8_t, iEffSeg);
3692IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m64);
3693IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m64);
3694IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr64, uint8_t, iEffSeg);
3695IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m64);
3696IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m64, int8_t, iEffSeg);
3697IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr64, bool, fIoChecked);
3698IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr64, bool, fIoChecked);
3699IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3700IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3701
3702IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr64, uint8_t, iEffSeg);
3703IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr64, uint8_t, iEffSeg);
3704IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m64);
3705IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m64);
3706IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr64, uint8_t, iEffSeg);
3707IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m64);
3708IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m64, int8_t, iEffSeg);
3709IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr64, bool, fIoChecked);
3710IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr64, bool, fIoChecked);
3711IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3712IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3713
3714IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr64, uint8_t, iEffSeg);
3715IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr64, uint8_t, iEffSeg);
3716IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m64);
3717IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m64);
3718IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr64, uint8_t, iEffSeg);
3719IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m64);
3720IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m64, int8_t, iEffSeg);
3721IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr64, bool, fIoChecked);
3722IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr64, bool, fIoChecked);
3723IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3724IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3725
3726IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr64, uint8_t, iEffSeg);
3727IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr64, uint8_t, iEffSeg);
3728IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m64);
3729IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m64);
3730IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr64, uint8_t, iEffSeg);
3731IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m64);
3732IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m64, int8_t, iEffSeg);
3733IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr64, bool, fIoChecked);
3734IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr64, bool, fIoChecked);
3735IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3736IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3737/** @} */
3738
3739#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3740VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT;
3741VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT;
3742VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT;
3743VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT;
3744VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT;
3745VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT;
3746VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT;
3747VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT;
3748VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT;
3749VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port,
3750 bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT;
3751VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess,
3752 bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT;
3753VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3754VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3755VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3756VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3757VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3758VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3759VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT;
3760VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw,
3761 RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT;
3762VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT;
3763VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT;
3764VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT;
3765uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT;
3766void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT;
3767VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
3768 uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT;
3769bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT;
3770IEM_CIMPL_PROTO_0(iemCImpl_vmxoff);
3771IEM_CIMPL_PROTO_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon);
3772IEM_CIMPL_PROTO_0(iemCImpl_vmlaunch);
3773IEM_CIMPL_PROTO_0(iemCImpl_vmresume);
3774IEM_CIMPL_PROTO_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3775IEM_CIMPL_PROTO_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3776IEM_CIMPL_PROTO_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3777IEM_CIMPL_PROTO_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField);
3778IEM_CIMPL_PROTO_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField);
3779IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField);
3780IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField);
3781IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField);
3782IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField);
3783IEM_CIMPL_PROTO_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType);
3784IEM_CIMPL_PROTO_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType);
3785IEM_CIMPL_PROTO_0(iemCImpl_vmx_pause);
3786#endif
3787
3788#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3789VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT;
3790VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3791VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
3792 uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT;
3793VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT;
3794IEM_CIMPL_PROTO_0(iemCImpl_vmrun);
3795IEM_CIMPL_PROTO_0(iemCImpl_vmload);
3796IEM_CIMPL_PROTO_0(iemCImpl_vmsave);
3797IEM_CIMPL_PROTO_0(iemCImpl_clgi);
3798IEM_CIMPL_PROTO_0(iemCImpl_stgi);
3799IEM_CIMPL_PROTO_0(iemCImpl_invlpga);
3800IEM_CIMPL_PROTO_0(iemCImpl_skinit);
3801IEM_CIMPL_PROTO_0(iemCImpl_svm_pause);
3802#endif
3803
3804IEM_CIMPL_PROTO_0(iemCImpl_vmcall); /* vmx */
3805IEM_CIMPL_PROTO_0(iemCImpl_vmmcall); /* svm */
3806IEM_CIMPL_PROTO_1(iemCImpl_Hypercall, uint16_t, uDisOpcode); /* both */
3807
3808
3809extern const PFNIEMOP g_apfnOneByteMap[256];
3810
3811/** @} */
3812
3813RT_C_DECLS_END
3814
3815#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3816
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette