VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 96025

Last change on this file since 96025 was 96025, checked in by vboxsync, 3 years ago

VMM/IEM: Implement [v]pmulhuw instructions, bugref:9898
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 173.8 KB
Line 
1/* $Id: IEMInternal.h 96025 2022-08-04 09:39:20Z vboxsync $ */
2/** @file
3 * IEM - Internal header file.
4 */
5
6/*
7 * Copyright (C) 2011-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
19#define VMM_INCLUDED_SRC_include_IEMInternal_h
20#ifndef RT_WITHOUT_PRAGMA_ONCE
21# pragma once
22#endif
23
24#include <VBox/vmm/cpum.h>
25#include <VBox/vmm/iem.h>
26#include <VBox/vmm/pgm.h>
27#include <VBox/vmm/stam.h>
28#include <VBox/param.h>
29
30#include <iprt/setjmp-without-sigmask.h>
31
32
33RT_C_DECLS_BEGIN
34
35
36/** @defgroup grp_iem_int Internals
37 * @ingroup grp_iem
38 * @internal
39 * @{
40 */
41
42/** For expanding symbol in slickedit and other products tagging and
43 * crossreferencing IEM symbols. */
44#ifndef IEM_STATIC
45# define IEM_STATIC static
46#endif
47
48/** @def IEM_WITH_SETJMP
49 * Enables alternative status code handling using setjmps.
50 *
51 * This adds a bit of expense via the setjmp() call since it saves all the
52 * non-volatile registers. However, it eliminates return code checks and allows
53 * for more optimal return value passing (return regs instead of stack buffer).
54 */
55#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
56# define IEM_WITH_SETJMP
57#endif
58
59#define IEM_IMPLEMENTS_TASKSWITCH
60
61/** @def IEM_WITH_3DNOW
62 * Includes the 3DNow decoding. */
63#define IEM_WITH_3DNOW
64
65/** @def IEM_WITH_THREE_0F_38
66 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
67#define IEM_WITH_THREE_0F_38
68
69/** @def IEM_WITH_THREE_0F_3A
70 * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
71#define IEM_WITH_THREE_0F_3A
72
73/** @def IEM_WITH_VEX
74 * Includes the VEX decoding. */
75#define IEM_WITH_VEX
76
77/** @def IEM_CFG_TARGET_CPU
78 * The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
79 *
80 * By default we allow this to be configured by the user via the
81 * CPUM/GuestCpuName config string, but this comes at a slight cost during
82 * decoding. So, for applications of this code where there is no need to
83 * be dynamic wrt target CPU, just modify this define.
84 */
85#if !defined(IEM_CFG_TARGET_CPU) || defined(DOXYGEN_RUNNING)
86# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
87#endif
88
89//#define IEM_WITH_CODE_TLB // - work in progress
90//#define IEM_WITH_DATA_TLB // - work in progress
91
92
93/** @def IEM_USE_UNALIGNED_DATA_ACCESS
94 * Use unaligned accesses instead of elaborate byte assembly. */
95#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
96# define IEM_USE_UNALIGNED_DATA_ACCESS
97#endif
98
99//#define IEM_LOG_MEMORY_WRITES
100
101#if !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
102/** Instruction statistics. */
103typedef struct IEMINSTRSTATS
104{
105# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
106# include "IEMInstructionStatisticsTmpl.h"
107# undef IEM_DO_INSTR_STAT
108} IEMINSTRSTATS;
109#else
110struct IEMINSTRSTATS;
111typedef struct IEMINSTRSTATS IEMINSTRSTATS;
112#endif
113/** Pointer to IEM instruction statistics. */
114typedef IEMINSTRSTATS *PIEMINSTRSTATS;
115
116
117/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::aidxTargetCpuEflFlavour
118 * @{ */
119#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /**< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. */
120#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /**< Intel EFLAGS result. */
121#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /**< AMD EFLAGS result */
122#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /**< Reserved/dummy entry slot that's the same as 0. */
123#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /**< For masking the index before use. */
124/** Selects the right variant from a_aArray.
125 * pVCpu is implicit in the caller context. */
126#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
127 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[1] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
128/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for when no native worker can
129 * be used because the host CPU does not support the operation. */
130#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_NON_NATIVE(a_aArray) \
131 (a_aArray[pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
132/** Variation of IEMTARGETCPU_EFL_BEHAVIOR_SELECT for a two dimentional
133 * array paralleling IEMCPU::aidxTargetCpuEflFlavour and a single bit index
134 * into the two.
135 * @sa IEM_SELECT_NATIVE_OR_FALLBACK */
136#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
137# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
138 (a_aaArray[a_fNative][pVCpu->iem.s.aidxTargetCpuEflFlavour[a_fNative] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
139#else
140# define IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(a_aaArray, a_fNative) \
141 (a_aaArray[0][pVCpu->iem.s.aidxTargetCpuEflFlavour[0] & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
142#endif
143/** @} */
144
145/**
146 * Picks @a a_pfnNative or @a a_pfnFallback according to the host CPU feature
147 * indicator given by @a a_fCpumFeatureMember (CPUMFEATURES member).
148 *
149 * On non-x86 hosts, this will shortcut to the fallback w/o checking the
150 * indicator.
151 *
152 * @sa IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX
153 */
154#if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
155# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) \
156 (g_CpumHostFeatures.s.a_fCpumFeatureMember ? a_pfnNative : a_pfnFallback)
157#else
158# define IEM_SELECT_HOST_OR_FALLBACK(a_fCpumFeatureMember, a_pfnNative, a_pfnFallback) (a_pfnFallback)
159#endif
160
161
162/**
163 * Extended operand mode that includes a representation of 8-bit.
164 *
165 * This is used for packing down modes when invoking some C instruction
166 * implementations.
167 */
168typedef enum IEMMODEX
169{
170 IEMMODEX_16BIT = IEMMODE_16BIT,
171 IEMMODEX_32BIT = IEMMODE_32BIT,
172 IEMMODEX_64BIT = IEMMODE_64BIT,
173 IEMMODEX_8BIT
174} IEMMODEX;
175AssertCompileSize(IEMMODEX, 4);
176
177
178/**
179 * Branch types.
180 */
181typedef enum IEMBRANCH
182{
183 IEMBRANCH_JUMP = 1,
184 IEMBRANCH_CALL,
185 IEMBRANCH_TRAP,
186 IEMBRANCH_SOFTWARE_INT,
187 IEMBRANCH_HARDWARE_INT
188} IEMBRANCH;
189AssertCompileSize(IEMBRANCH, 4);
190
191
192/**
193 * INT instruction types.
194 */
195typedef enum IEMINT
196{
197 /** INT n instruction (opcode 0xcd imm). */
198 IEMINT_INTN = 0,
199 /** Single byte INT3 instruction (opcode 0xcc). */
200 IEMINT_INT3 = IEM_XCPT_FLAGS_BP_INSTR,
201 /** Single byte INTO instruction (opcode 0xce). */
202 IEMINT_INTO = IEM_XCPT_FLAGS_OF_INSTR,
203 /** Single byte INT1 (ICEBP) instruction (opcode 0xf1). */
204 IEMINT_INT1 = IEM_XCPT_FLAGS_ICEBP_INSTR
205} IEMINT;
206AssertCompileSize(IEMINT, 4);
207
208
209/**
210 * A FPU result.
211 */
212typedef struct IEMFPURESULT
213{
214 /** The output value. */
215 RTFLOAT80U r80Result;
216 /** The output status. */
217 uint16_t FSW;
218} IEMFPURESULT;
219AssertCompileMemberOffset(IEMFPURESULT, FSW, 10);
220/** Pointer to a FPU result. */
221typedef IEMFPURESULT *PIEMFPURESULT;
222/** Pointer to a const FPU result. */
223typedef IEMFPURESULT const *PCIEMFPURESULT;
224
225
226/**
227 * A FPU result consisting of two output values and FSW.
228 */
229typedef struct IEMFPURESULTTWO
230{
231 /** The first output value. */
232 RTFLOAT80U r80Result1;
233 /** The output status. */
234 uint16_t FSW;
235 /** The second output value. */
236 RTFLOAT80U r80Result2;
237} IEMFPURESULTTWO;
238AssertCompileMemberOffset(IEMFPURESULTTWO, FSW, 10);
239AssertCompileMemberOffset(IEMFPURESULTTWO, r80Result2, 12);
240/** Pointer to a FPU result consisting of two output values and FSW. */
241typedef IEMFPURESULTTWO *PIEMFPURESULTTWO;
242/** Pointer to a const FPU result consisting of two output values and FSW. */
243typedef IEMFPURESULTTWO const *PCIEMFPURESULTTWO;
244
245
246/**
247 * IEM TLB entry.
248 *
249 * Lookup assembly:
250 * @code{.asm}
251 ; Calculate tag.
252 mov rax, [VA]
253 shl rax, 16
254 shr rax, 16 + X86_PAGE_SHIFT
255 or rax, [uTlbRevision]
256
257 ; Do indexing.
258 movzx ecx, al
259 lea rcx, [pTlbEntries + rcx]
260
261 ; Check tag.
262 cmp [rcx + IEMTLBENTRY.uTag], rax
263 jne .TlbMiss
264
265 ; Check access.
266 mov rax, ACCESS_FLAGS | MAPPING_R3_NOT_VALID | 0xffffff00
267 and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
268 cmp rax, [uTlbPhysRev]
269 jne .TlbMiss
270
271 ; Calc address and we're done.
272 mov eax, X86_PAGE_OFFSET_MASK
273 and eax, [VA]
274 or rax, [rcx + IEMTLBENTRY.pMappingR3]
275 %ifdef VBOX_WITH_STATISTICS
276 inc qword [cTlbHits]
277 %endif
278 jmp .Done
279
280 .TlbMiss:
281 mov r8d, ACCESS_FLAGS
282 mov rdx, [VA]
283 mov rcx, [pVCpu]
284 call iemTlbTypeMiss
285 .Done:
286
287 @endcode
288 *
289 */
290typedef struct IEMTLBENTRY
291{
292 /** The TLB entry tag.
293 * Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
294 * is ASSUMING a virtual address width of 48 bits.
295 *
296 * Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
297 *
298 * The TLB lookup code uses the current TLB revision, which won't ever be zero,
299 * enabling an extremely cheap TLB invalidation most of the time. When the TLB
300 * revision wraps around though, the tags needs to be zeroed.
301 *
302 * @note Try use SHRD instruction? After seeing
303 * https://gmplib.org/~tege/x86-timing.pdf, maybe not.
304 *
305 * @todo This will need to be reorganized for 57-bit wide virtual address and
306 * PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
307 * have to move the TLB entry versioning entirely to the
308 * fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
309 * 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
310 * consumed by PCID and ASID (12 + 6 = 18).
311 */
312 uint64_t uTag;
313 /** Access flags and physical TLB revision.
314 *
315 * - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
316 * - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
317 * - Bit 2 - page tables - not user (complemented X86_PTE_US).
318 * - Bit 3 - pgm phys/virt - not directly writable.
319 * - Bit 4 - pgm phys page - not directly readable.
320 * - Bit 5 - page tables - not accessed (complemented X86_PTE_A).
321 * - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
322 * - Bit 7 - tlb entry - pMappingR3 member not valid.
323 * - Bits 63 thru 8 are used for the physical TLB revision number.
324 *
325 * We're using complemented bit meanings here because it makes it easy to check
326 * whether special action is required. For instance a user mode write access
327 * would do a "TEST fFlags, (X86_PTE_RW | X86_PTE_US | X86_PTE_D)" and a
328 * non-zero result would mean special handling needed because either it wasn't
329 * writable, or it wasn't user, or the page wasn't dirty. A user mode read
330 * access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
331 * need to check any PTE flag.
332 */
333 uint64_t fFlagsAndPhysRev;
334 /** The guest physical page address. */
335 uint64_t GCPhys;
336 /** Pointer to the ring-3 mapping. */
337 R3PTRTYPE(uint8_t *) pbMappingR3;
338#if HC_ARCH_BITS == 32
339 uint32_t u32Padding1;
340#endif
341} IEMTLBENTRY;
342AssertCompileSize(IEMTLBENTRY, 32);
343/** Pointer to an IEM TLB entry. */
344typedef IEMTLBENTRY *PIEMTLBENTRY;
345
346/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
347 * @{ */
348#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /**< Page tables: Not executable. */
349#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /**< Page tables: Not writable. */
350#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /**< Page tables: Not user accessible (supervisor only). */
351#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /**< Phys page: Not writable (access handler, ROM, whatever). */
352#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /**< Phys page: Not readable (MMIO / access handler, ROM) */
353#define IEMTLBE_F_PT_NO_ACCESSED RT_BIT_64(5) /**< Phys tables: Not accessed (need to be marked accessed). */
354#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /**< Page tables: Not dirty (needs to be made dirty on write). */
355#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(7) /**< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. */
356#define IEMTLBE_F_PG_UNASSIGNED RT_BIT_64(8) /**< Phys page: Unassigned memory (not RAM, ROM, MMIO2 or MMIO). */
357#define IEMTLBE_F_PHYS_REV UINT64_C(0xfffffffffffffe00) /**< Physical revision mask. @sa IEMTLB_PHYS_REV_INCR */
358/** @} */
359
360
361/**
362 * An IEM TLB.
363 *
364 * We've got two of these, one for data and one for instructions.
365 */
366typedef struct IEMTLB
367{
368 /** The TLB entries.
369 * We've choosen 256 because that way we can obtain the result directly from a
370 * 8-bit register without an additional AND instruction. */
371 IEMTLBENTRY aEntries[256];
372 /** The TLB revision.
373 * This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
374 * by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
375 * the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
376 * (The revision zero indicates an invalid TLB entry.)
377 *
378 * The initial value is choosen to cause an early wraparound. */
379 uint64_t uTlbRevision;
380 /** The TLB physical address revision - shadow of PGM variable.
381 *
382 * This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
383 * incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
384 * a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
385 * as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
386 *
387 * The initial value is choosen to cause an early wraparound. */
388 uint64_t volatile uTlbPhysRev;
389
390 /* Statistics: */
391
392 /** TLB hits (VBOX_WITH_STATISTICS only). */
393 uint64_t cTlbHits;
394 /** TLB misses. */
395 uint32_t cTlbMisses;
396 /** Slow read path. */
397 uint32_t cTlbSlowReadPath;
398#if 0
399 /** TLB misses because of tag mismatch. */
400 uint32_t cTlbMissesTag;
401 /** TLB misses because of virtual access violation. */
402 uint32_t cTlbMissesVirtAccess;
403 /** TLB misses because of dirty bit. */
404 uint32_t cTlbMissesDirty;
405 /** TLB misses because of MMIO */
406 uint32_t cTlbMissesMmio;
407 /** TLB misses because of write access handlers. */
408 uint32_t cTlbMissesWriteHandler;
409 /** TLB misses because no r3(/r0) mapping. */
410 uint32_t cTlbMissesMapping;
411#endif
412 /** Alignment padding. */
413 uint32_t au32Padding[3+5];
414} IEMTLB;
415AssertCompileSizeAlignment(IEMTLB, 64);
416/** IEMTLB::uTlbRevision increment. */
417#define IEMTLB_REVISION_INCR RT_BIT_64(36)
418/** IEMTLB::uTlbRevision mask. */
419#define IEMTLB_REVISION_MASK (~(RT_BIT_64(36) - 1))
420/** IEMTLB::uTlbPhysRev increment.
421 * @sa IEMTLBE_F_PHYS_REV */
422#define IEMTLB_PHYS_REV_INCR RT_BIT_64(9)
423/**
424 * Calculates the TLB tag for a virtual address.
425 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
426 * @param a_pTlb The TLB.
427 * @param a_GCPtr The virtual address.
428 */
429#define IEMTLB_CALC_TAG(a_pTlb, a_GCPtr) ( IEMTLB_CALC_TAG_NO_REV(a_GCPtr) | (a_pTlb)->uTlbRevision )
430/**
431 * Calculates the TLB tag for a virtual address but without TLB revision.
432 * @returns Tag value for indexing and comparing with IEMTLB::uTag.
433 * @param a_GCPtr The virtual address.
434 */
435#define IEMTLB_CALC_TAG_NO_REV(a_GCPtr) ( (((a_GCPtr) << 16) >> (GUEST_PAGE_SHIFT + 16)) )
436/**
437 * Converts a TLB tag value into a TLB index.
438 * @returns Index into IEMTLB::aEntries.
439 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
440 */
441#define IEMTLB_TAG_TO_INDEX(a_uTag) ( (uint8_t)(a_uTag) )
442/**
443 * Converts a TLB tag value into a TLB index.
444 * @returns Index into IEMTLB::aEntries.
445 * @param a_pTlb The TLB.
446 * @param a_uTag Value returned by IEMTLB_CALC_TAG.
447 */
448#define IEMTLB_TAG_TO_ENTRY(a_pTlb, a_uTag) ( &(a_pTlb)->aEntries[IEMTLB_TAG_TO_INDEX(a_uTag)] )
449
450
451/**
452 * The per-CPU IEM state.
453 */
454typedef struct IEMCPU
455{
456 /** Info status code that needs to be propagated to the IEM caller.
457 * This cannot be passed internally, as it would complicate all success
458 * checks within the interpreter making the code larger and almost impossible
459 * to get right. Instead, we'll store status codes to pass on here. Each
460 * source of these codes will perform appropriate sanity checks. */
461 int32_t rcPassUp; /* 0x00 */
462
463 /** The current CPU execution mode (CS). */
464 IEMMODE enmCpuMode; /* 0x04 */
465 /** The CPL. */
466 uint8_t uCpl; /* 0x05 */
467
468 /** Whether to bypass access handlers or not. */
469 bool fBypassHandlers; /* 0x06 */
470 /** Whether to disregard the lock prefix (implied or not). */
471 bool fDisregardLock; /* 0x07 */
472
473 /** @name Decoder state.
474 * @{ */
475#ifdef IEM_WITH_CODE_TLB
476 /** The offset of the next instruction byte. */
477 uint32_t offInstrNextByte; /* 0x08 */
478 /** The number of bytes available at pbInstrBuf for the current instruction.
479 * This takes the max opcode length into account so that doesn't need to be
480 * checked separately. */
481 uint32_t cbInstrBuf; /* 0x0c */
482 /** Pointer to the page containing RIP, user specified buffer or abOpcode.
483 * This can be NULL if the page isn't mappable for some reason, in which
484 * case we'll do fallback stuff.
485 *
486 * If we're executing an instruction from a user specified buffer,
487 * IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
488 * aligned pointer but pointer to the user data.
489 *
490 * For instructions crossing pages, this will start on the first page and be
491 * advanced to the next page by the time we've decoded the instruction. This
492 * therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
493 */
494 uint8_t const *pbInstrBuf; /* 0x10 */
495# if ARCH_BITS == 32
496 uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
497# endif
498 /** The program counter corresponding to pbInstrBuf.
499 * This is set to a non-canonical address when we need to invalidate it. */
500 uint64_t uInstrBufPc; /* 0x18 */
501 /** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
502 * This takes the CS segment limit into account. */
503 uint16_t cbInstrBufTotal; /* 0x20 */
504 /** Offset into pbInstrBuf of the first byte of the current instruction.
505 * Can be negative to efficiently handle cross page instructions. */
506 int16_t offCurInstrStart; /* 0x22 */
507
508 /** The prefix mask (IEM_OP_PRF_XXX). */
509 uint32_t fPrefixes; /* 0x24 */
510 /** The extra REX ModR/M register field bit (REX.R << 3). */
511 uint8_t uRexReg; /* 0x28 */
512 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
513 * (REX.B << 3). */
514 uint8_t uRexB; /* 0x29 */
515 /** The extra REX SIB index field bit (REX.X << 3). */
516 uint8_t uRexIndex; /* 0x2a */
517
518 /** The effective segment register (X86_SREG_XXX). */
519 uint8_t iEffSeg; /* 0x2b */
520
521 /** The offset of the ModR/M byte relative to the start of the instruction. */
522 uint8_t offModRm; /* 0x2c */
523#else
524 /** The size of what has currently been fetched into abOpcode. */
525 uint8_t cbOpcode; /* 0x08 */
526 /** The current offset into abOpcode. */
527 uint8_t offOpcode; /* 0x09 */
528 /** The offset of the ModR/M byte relative to the start of the instruction. */
529 uint8_t offModRm; /* 0x0a */
530
531 /** The effective segment register (X86_SREG_XXX). */
532 uint8_t iEffSeg; /* 0x0b */
533
534 /** The prefix mask (IEM_OP_PRF_XXX). */
535 uint32_t fPrefixes; /* 0x0c */
536 /** The extra REX ModR/M register field bit (REX.R << 3). */
537 uint8_t uRexReg; /* 0x10 */
538 /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
539 * (REX.B << 3). */
540 uint8_t uRexB; /* 0x11 */
541 /** The extra REX SIB index field bit (REX.X << 3). */
542 uint8_t uRexIndex; /* 0x12 */
543
544#endif
545
546 /** The effective operand mode. */
547 IEMMODE enmEffOpSize; /* 0x2d, 0x13 */
548 /** The default addressing mode. */
549 IEMMODE enmDefAddrMode; /* 0x2e, 0x14 */
550 /** The effective addressing mode. */
551 IEMMODE enmEffAddrMode; /* 0x2f, 0x15 */
552 /** The default operand mode. */
553 IEMMODE enmDefOpSize; /* 0x30, 0x16 */
554
555 /** Prefix index (VEX.pp) for two byte and three byte tables. */
556 uint8_t idxPrefix; /* 0x31, 0x17 */
557 /** 3rd VEX/EVEX/XOP register.
558 * Please use IEM_GET_EFFECTIVE_VVVV to access. */
559 uint8_t uVex3rdReg; /* 0x32, 0x18 */
560 /** The VEX/EVEX/XOP length field. */
561 uint8_t uVexLength; /* 0x33, 0x19 */
562 /** Additional EVEX stuff. */
563 uint8_t fEvexStuff; /* 0x34, 0x1a */
564
565 /** Explicit alignment padding. */
566 uint8_t abAlignment2a[1]; /* 0x35, 0x1b */
567 /** The FPU opcode (FOP). */
568 uint16_t uFpuOpcode; /* 0x36, 0x1c */
569#ifndef IEM_WITH_CODE_TLB
570 /** Explicit alignment padding. */
571 uint8_t abAlignment2b[2]; /* 0x1e */
572#endif
573
574 /** The opcode bytes. */
575 uint8_t abOpcode[15]; /* 0x48, 0x20 */
576 /** Explicit alignment padding. */
577#ifdef IEM_WITH_CODE_TLB
578 uint8_t abAlignment2c[0x48 - 0x47]; /* 0x37 */
579#else
580 uint8_t abAlignment2c[0x48 - 0x2f]; /* 0x2f */
581#endif
582 /** @} */
583
584
585 /** The flags of the current exception / interrupt. */
586 uint32_t fCurXcpt; /* 0x48, 0x48 */
587 /** The current exception / interrupt. */
588 uint8_t uCurXcpt;
589 /** Exception / interrupt recursion depth. */
590 int8_t cXcptRecursions;
591
592 /** The number of active guest memory mappings. */
593 uint8_t cActiveMappings;
594 /** The next unused mapping index. */
595 uint8_t iNextMapping;
596 /** Records for tracking guest memory mappings. */
597 struct
598 {
599 /** The address of the mapped bytes. */
600 void *pv;
601 /** The access flags (IEM_ACCESS_XXX).
602 * IEM_ACCESS_INVALID if the entry is unused. */
603 uint32_t fAccess;
604#if HC_ARCH_BITS == 64
605 uint32_t u32Alignment4; /**< Alignment padding. */
606#endif
607 } aMemMappings[3];
608
609 /** Locking records for the mapped memory. */
610 union
611 {
612 PGMPAGEMAPLOCK Lock;
613 uint64_t au64Padding[2];
614 } aMemMappingLocks[3];
615
616 /** Bounce buffer info.
617 * This runs in parallel to aMemMappings. */
618 struct
619 {
620 /** The physical address of the first byte. */
621 RTGCPHYS GCPhysFirst;
622 /** The physical address of the second page. */
623 RTGCPHYS GCPhysSecond;
624 /** The number of bytes in the first page. */
625 uint16_t cbFirst;
626 /** The number of bytes in the second page. */
627 uint16_t cbSecond;
628 /** Whether it's unassigned memory. */
629 bool fUnassigned;
630 /** Explicit alignment padding. */
631 bool afAlignment5[3];
632 } aMemBbMappings[3];
633
634 /** Bounce buffer storage.
635 * This runs in parallel to aMemMappings and aMemBbMappings. */
636 struct
637 {
638 uint8_t ab[512];
639 } aBounceBuffers[3];
640
641
642 /** Pointer set jump buffer - ring-3 context. */
643 R3PTRTYPE(jmp_buf *) pJmpBufR3;
644 /** Pointer set jump buffer - ring-0 context. */
645 R0PTRTYPE(jmp_buf *) pJmpBufR0;
646
647 /** @todo Should move this near @a fCurXcpt later. */
648 /** The CR2 for the current exception / interrupt. */
649 uint64_t uCurXcptCr2;
650 /** The error code for the current exception / interrupt. */
651 uint32_t uCurXcptErr;
652
653 /** @name Statistics
654 * @{ */
655 /** The number of instructions we've executed. */
656 uint32_t cInstructions;
657 /** The number of potential exits. */
658 uint32_t cPotentialExits;
659 /** The number of bytes data or stack written (mostly for IEMExecOneEx).
660 * This may contain uncommitted writes. */
661 uint32_t cbWritten;
662 /** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
663 uint32_t cRetInstrNotImplemented;
664 /** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
665 uint32_t cRetAspectNotImplemented;
666 /** Counts informational statuses returned (other than VINF_SUCCESS). */
667 uint32_t cRetInfStatuses;
668 /** Counts other error statuses returned. */
669 uint32_t cRetErrStatuses;
670 /** Number of times rcPassUp has been used. */
671 uint32_t cRetPassUpStatus;
672 /** Number of times RZ left with instruction commit pending for ring-3. */
673 uint32_t cPendingCommit;
674 /** Number of long jumps. */
675 uint32_t cLongJumps;
676 /** @} */
677
678 /** @name Target CPU information.
679 * @{ */
680#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
681 /** The target CPU. */
682 uint8_t uTargetCpu;
683#else
684 uint8_t bTargetCpuPadding;
685#endif
686 /** For selecting assembly works matching the target CPU EFLAGS behaviour, see
687 * IEMTARGETCPU_EFL_BEHAVIOR_XXX for values, with the 1st entry for when no
688 * native host support and the 2nd for when there is.
689 *
690 * The two values are typically indexed by a g_CpumHostFeatures bit.
691 *
692 * This is for instance used for the BSF & BSR instructions where AMD and
693 * Intel CPUs produce different EFLAGS. */
694 uint8_t aidxTargetCpuEflFlavour[2];
695
696 /** The CPU vendor. */
697 CPUMCPUVENDOR enmCpuVendor;
698 /** @} */
699
700 /** @name Host CPU information.
701 * @{ */
702 /** The CPU vendor. */
703 CPUMCPUVENDOR enmHostCpuVendor;
704 /** @} */
705
706 /** Counts RDMSR \#GP(0) LogRel(). */
707 uint8_t cLogRelRdMsr;
708 /** Counts WRMSR \#GP(0) LogRel(). */
709 uint8_t cLogRelWrMsr;
710 /** Alignment padding. */
711 uint8_t abAlignment8[50];
712
713 /** Data TLB.
714 * @remarks Must be 64-byte aligned. */
715 IEMTLB DataTlb;
716 /** Instruction TLB.
717 * @remarks Must be 64-byte aligned. */
718 IEMTLB CodeTlb;
719
720 /** Exception statistics. */
721 STAMCOUNTER aStatXcpts[32];
722 /** Interrupt statistics. */
723 uint32_t aStatInts[256];
724
725#if defined(VBOX_WITH_STATISTICS) && !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
726 /** Instruction statistics for ring-0/raw-mode. */
727 IEMINSTRSTATS StatsRZ;
728 /** Instruction statistics for ring-3. */
729 IEMINSTRSTATS StatsR3;
730#endif
731} IEMCPU;
732AssertCompileMemberOffset(IEMCPU, fCurXcpt, 0x48);
733AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
734AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
735/** Pointer to the per-CPU IEM state. */
736typedef IEMCPU *PIEMCPU;
737/** Pointer to the const per-CPU IEM state. */
738typedef IEMCPU const *PCIEMCPU;
739
740
741/** @def IEM_GET_CTX
742 * Gets the guest CPU context for the calling EMT.
743 * @returns PCPUMCTX
744 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
745 */
746#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
747
748/** @def IEM_CTX_ASSERT
749 * Asserts that the @a a_fExtrnMbz is present in the CPU context.
750 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
751 * @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
752 */
753#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
754 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (a_pVCpu)->cpum.GstCtx.fExtrn, \
755 (a_fExtrnMbz)))
756
757/** @def IEM_CTX_IMPORT_RET
758 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
759 *
760 * Will call the keep to import the bits as needed.
761 *
762 * Returns on import failure.
763 *
764 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
765 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
766 */
767#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
768 do { \
769 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
770 { /* likely */ } \
771 else \
772 { \
773 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
774 AssertRCReturn(rcCtxImport, rcCtxImport); \
775 } \
776 } while (0)
777
778/** @def IEM_CTX_IMPORT_NORET
779 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
780 *
781 * Will call the keep to import the bits as needed.
782 *
783 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
784 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
785 */
786#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
787 do { \
788 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
789 { /* likely */ } \
790 else \
791 { \
792 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
793 AssertLogRelRC(rcCtxImport); \
794 } \
795 } while (0)
796
797/** @def IEM_CTX_IMPORT_JMP
798 * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
799 *
800 * Will call the keep to import the bits as needed.
801 *
802 * Jumps on import failure.
803 *
804 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
805 * @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
806 */
807#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
808 do { \
809 if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
810 { /* likely */ } \
811 else \
812 { \
813 int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
814 AssertRCStmt(rcCtxImport, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), rcCtxImport)); \
815 } \
816 } while (0)
817
818
819
820/** @def IEM_GET_TARGET_CPU
821 * Gets the current IEMTARGETCPU value.
822 * @returns IEMTARGETCPU value.
823 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
824 */
825#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
826# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
827#else
828# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
829#endif
830
831/** @def IEM_GET_INSTR_LEN
832 * Gets the instruction length. */
833#ifdef IEM_WITH_CODE_TLB
834# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
835#else
836# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
837#endif
838
839
840/**
841 * Shared per-VM IEM data.
842 */
843typedef struct IEM
844{
845 /** The VMX APIC-access page handler type. */
846 PGMPHYSHANDLERTYPE hVmxApicAccessPage;
847} IEM;
848
849
850
851/** @name IEM_ACCESS_XXX - Access details.
852 * @{ */
853#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
854#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
855#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
856#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
857#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
858#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
859#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
860#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
861#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
862#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
863/** The writes are partial, so if initialize the bounce buffer with the
864 * orignal RAM content. */
865#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
866/** Used in aMemMappings to indicate that the entry is bounce buffered. */
867#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
868/** Bounce buffer with ring-3 write pending, first page. */
869#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
870/** Bounce buffer with ring-3 write pending, second page. */
871#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
872/** Not locked, accessed via the TLB. */
873#define IEM_ACCESS_NOT_LOCKED UINT32_C(0x00001000)
874/** Valid bit mask. */
875#define IEM_ACCESS_VALID_MASK UINT32_C(0x00001fff)
876/** Shift count for the TLB flags (upper word). */
877#define IEM_ACCESS_SHIFT_TLB_FLAGS 16
878
879/** Read+write data alias. */
880#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
881/** Write data alias. */
882#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
883/** Read data alias. */
884#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA)
885/** Instruction fetch alias. */
886#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_WHAT_CODE)
887/** Stack write alias. */
888#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
889/** Stack read alias. */
890#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_STACK)
891/** Stack read+write alias. */
892#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
893/** Read system table alias. */
894#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_SYS)
895/** Read+write system table alias. */
896#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_SYS)
897/** @} */
898
899/** @name Prefix constants (IEMCPU::fPrefixes)
900 * @{ */
901#define IEM_OP_PRF_SEG_CS RT_BIT_32(0) /**< CS segment prefix (0x2e). */
902#define IEM_OP_PRF_SEG_SS RT_BIT_32(1) /**< SS segment prefix (0x36). */
903#define IEM_OP_PRF_SEG_DS RT_BIT_32(2) /**< DS segment prefix (0x3e). */
904#define IEM_OP_PRF_SEG_ES RT_BIT_32(3) /**< ES segment prefix (0x26). */
905#define IEM_OP_PRF_SEG_FS RT_BIT_32(4) /**< FS segment prefix (0x64). */
906#define IEM_OP_PRF_SEG_GS RT_BIT_32(5) /**< GS segment prefix (0x65). */
907#define IEM_OP_PRF_SEG_MASK UINT32_C(0x3f)
908
909#define IEM_OP_PRF_SIZE_OP RT_BIT_32(8) /**< Operand size prefix (0x66). */
910#define IEM_OP_PRF_SIZE_REX_W RT_BIT_32(9) /**< REX.W prefix (0x48-0x4f). */
911#define IEM_OP_PRF_SIZE_ADDR RT_BIT_32(10) /**< Address size prefix (0x67). */
912
913#define IEM_OP_PRF_LOCK RT_BIT_32(16) /**< Lock prefix (0xf0). */
914#define IEM_OP_PRF_REPNZ RT_BIT_32(17) /**< Repeat-not-zero prefix (0xf2). */
915#define IEM_OP_PRF_REPZ RT_BIT_32(18) /**< Repeat-if-zero prefix (0xf3). */
916
917#define IEM_OP_PRF_REX RT_BIT_32(24) /**< Any REX prefix (0x40-0x4f). */
918#define IEM_OP_PRF_REX_R RT_BIT_32(25) /**< REX.R prefix (0x44,0x45,0x46,0x47,0x4c,0x4d,0x4e,0x4f). */
919#define IEM_OP_PRF_REX_B RT_BIT_32(26) /**< REX.B prefix (0x41,0x43,0x45,0x47,0x49,0x4b,0x4d,0x4f). */
920#define IEM_OP_PRF_REX_X RT_BIT_32(27) /**< REX.X prefix (0x42,0x43,0x46,0x47,0x4a,0x4b,0x4e,0x4f). */
921/** Mask with all the REX prefix flags.
922 * This is generally for use when needing to undo the REX prefixes when they
923 * are followed legacy prefixes and therefore does not immediately preceed
924 * the first opcode byte.
925 * For testing whether any REX prefix is present, use IEM_OP_PRF_REX instead. */
926#define IEM_OP_PRF_REX_MASK (IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W )
927
928#define IEM_OP_PRF_VEX RT_BIT_32(28) /**< Indiciates VEX prefix. */
929#define IEM_OP_PRF_EVEX RT_BIT_32(29) /**< Indiciates EVEX prefix. */
930#define IEM_OP_PRF_XOP RT_BIT_32(30) /**< Indiciates XOP prefix. */
931/** @} */
932
933/** @name IEMOPFORM_XXX - Opcode forms
934 * @note These are ORed together with IEMOPHINT_XXX.
935 * @{ */
936/** ModR/M: reg, r/m */
937#define IEMOPFORM_RM 0
938/** ModR/M: reg, r/m (register) */
939#define IEMOPFORM_RM_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
940/** ModR/M: reg, r/m (memory) */
941#define IEMOPFORM_RM_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
942/** ModR/M: reg, r/m */
943#define IEMOPFORM_RMI 1
944/** ModR/M: reg, r/m (register) */
945#define IEMOPFORM_RMI_REG (IEMOPFORM_RM | IEMOPFORM_MOD3)
946/** ModR/M: reg, r/m (memory) */
947#define IEMOPFORM_RMI_MEM (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
948/** ModR/M: r/m, reg */
949#define IEMOPFORM_MR 2
950/** ModR/M: r/m (register), reg */
951#define IEMOPFORM_MR_REG (IEMOPFORM_MR | IEMOPFORM_MOD3)
952/** ModR/M: r/m (memory), reg */
953#define IEMOPFORM_MR_MEM (IEMOPFORM_MR | IEMOPFORM_NOT_MOD3)
954/** ModR/M: r/m only */
955#define IEMOPFORM_M 3
956/** ModR/M: r/m only (register). */
957#define IEMOPFORM_M_REG (IEMOPFORM_M | IEMOPFORM_MOD3)
958/** ModR/M: r/m only (memory). */
959#define IEMOPFORM_M_MEM (IEMOPFORM_M | IEMOPFORM_NOT_MOD3)
960/** ModR/M: reg only */
961#define IEMOPFORM_R 4
962
963/** VEX+ModR/M: reg, r/m */
964#define IEMOPFORM_VEX_RM 8
965/** VEX+ModR/M: reg, r/m (register) */
966#define IEMOPFORM_VEX_RM_REG (IEMOPFORM_VEX_RM | IEMOPFORM_MOD3)
967/** VEX+ModR/M: reg, r/m (memory) */
968#define IEMOPFORM_VEX_RM_MEM (IEMOPFORM_VEX_RM | IEMOPFORM_NOT_MOD3)
969/** VEX+ModR/M: r/m, reg */
970#define IEMOPFORM_VEX_MR 9
971/** VEX+ModR/M: r/m (register), reg */
972#define IEMOPFORM_VEX_MR_REG (IEMOPFORM_VEX_MR | IEMOPFORM_MOD3)
973/** VEX+ModR/M: r/m (memory), reg */
974#define IEMOPFORM_VEX_MR_MEM (IEMOPFORM_VEX_MR | IEMOPFORM_NOT_MOD3)
975/** VEX+ModR/M: r/m only */
976#define IEMOPFORM_VEX_M 10
977/** VEX+ModR/M: r/m only (register). */
978#define IEMOPFORM_VEX_M_REG (IEMOPFORM_VEX_M | IEMOPFORM_MOD3)
979/** VEX+ModR/M: r/m only (memory). */
980#define IEMOPFORM_VEX_M_MEM (IEMOPFORM_VEX_M | IEMOPFORM_NOT_MOD3)
981/** VEX+ModR/M: reg only */
982#define IEMOPFORM_VEX_R 11
983/** VEX+ModR/M: reg, vvvv, r/m */
984#define IEMOPFORM_VEX_RVM 12
985/** VEX+ModR/M: reg, vvvv, r/m (register). */
986#define IEMOPFORM_VEX_RVM_REG (IEMOPFORM_VEX_RVM | IEMOPFORM_MOD3)
987/** VEX+ModR/M: reg, vvvv, r/m (memory). */
988#define IEMOPFORM_VEX_RVM_MEM (IEMOPFORM_VEX_RVM | IEMOPFORM_NOT_MOD3)
989/** VEX+ModR/M: reg, r/m, vvvv */
990#define IEMOPFORM_VEX_RMV 13
991/** VEX+ModR/M: reg, r/m, vvvv (register). */
992#define IEMOPFORM_VEX_RMV_REG (IEMOPFORM_VEX_RMV | IEMOPFORM_MOD3)
993/** VEX+ModR/M: reg, r/m, vvvv (memory). */
994#define IEMOPFORM_VEX_RMV_MEM (IEMOPFORM_VEX_RMV | IEMOPFORM_NOT_MOD3)
995/** VEX+ModR/M: reg, r/m, imm8 */
996#define IEMOPFORM_VEX_RMI 14
997/** VEX+ModR/M: reg, r/m, imm8 (register). */
998#define IEMOPFORM_VEX_RMI_REG (IEMOPFORM_VEX_RMI | IEMOPFORM_MOD3)
999/** VEX+ModR/M: reg, r/m, imm8 (memory). */
1000#define IEMOPFORM_VEX_RMI_MEM (IEMOPFORM_VEX_RMI | IEMOPFORM_NOT_MOD3)
1001/** VEX+ModR/M: r/m, vvvv, reg */
1002#define IEMOPFORM_VEX_MVR 15
1003/** VEX+ModR/M: r/m, vvvv, reg (register) */
1004#define IEMOPFORM_VEX_MVR_REG (IEMOPFORM_VEX_MVR | IEMOPFORM_MOD3)
1005/** VEX+ModR/M: r/m, vvvv, reg (memory) */
1006#define IEMOPFORM_VEX_MVR_MEM (IEMOPFORM_VEX_MVR | IEMOPFORM_NOT_MOD3)
1007/** VEX+ModR/M+/n: vvvv, r/m */
1008#define IEMOPFORM_VEX_VM 16
1009/** VEX+ModR/M+/n: vvvv, r/m (register) */
1010#define IEMOPFORM_VEX_VM_REG (IEMOPFORM_VEX_VM | IEMOPFORM_MOD3)
1011/** VEX+ModR/M+/n: vvvv, r/m (memory) */
1012#define IEMOPFORM_VEX_VM_MEM (IEMOPFORM_VEX_VM | IEMOPFORM_NOT_MOD3)
1013
1014/** Fixed register instruction, no R/M. */
1015#define IEMOPFORM_FIXED 32
1016
1017/** The r/m is a register. */
1018#define IEMOPFORM_MOD3 RT_BIT_32(8)
1019/** The r/m is a memory access. */
1020#define IEMOPFORM_NOT_MOD3 RT_BIT_32(9)
1021/** @} */
1022
1023/** @name IEMOPHINT_XXX - Additional Opcode Hints
1024 * @note These are ORed together with IEMOPFORM_XXX.
1025 * @{ */
1026/** Ignores the operand size prefix (66h). */
1027#define IEMOPHINT_IGNORES_OZ_PFX RT_BIT_32(10)
1028/** Ignores REX.W (aka WIG). */
1029#define IEMOPHINT_IGNORES_REXW RT_BIT_32(11)
1030/** Both the operand size prefixes (66h + REX.W) are ignored. */
1031#define IEMOPHINT_IGNORES_OP_SIZES (IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW)
1032/** Allowed with the lock prefix. */
1033#define IEMOPHINT_LOCK_ALLOWED RT_BIT_32(11)
1034/** The VEX.L value is ignored (aka LIG). */
1035#define IEMOPHINT_VEX_L_IGNORED RT_BIT_32(12)
1036/** The VEX.L value must be zero (i.e. 128-bit width only). */
1037#define IEMOPHINT_VEX_L_ZERO RT_BIT_32(13)
1038/** The VEX.V value must be zero. */
1039#define IEMOPHINT_VEX_V_ZERO RT_BIT_32(14)
1040
1041/** Hint to IEMAllInstructionPython.py that this macro should be skipped. */
1042#define IEMOPHINT_SKIP_PYTHON RT_BIT_32(31)
1043/** @} */
1044
1045/**
1046 * Possible hardware task switch sources.
1047 */
1048typedef enum IEMTASKSWITCH
1049{
1050 /** Task switch caused by an interrupt/exception. */
1051 IEMTASKSWITCH_INT_XCPT = 1,
1052 /** Task switch caused by a far CALL. */
1053 IEMTASKSWITCH_CALL,
1054 /** Task switch caused by a far JMP. */
1055 IEMTASKSWITCH_JUMP,
1056 /** Task switch caused by an IRET. */
1057 IEMTASKSWITCH_IRET
1058} IEMTASKSWITCH;
1059AssertCompileSize(IEMTASKSWITCH, 4);
1060
1061/**
1062 * Possible CrX load (write) sources.
1063 */
1064typedef enum IEMACCESSCRX
1065{
1066 /** CrX access caused by 'mov crX' instruction. */
1067 IEMACCESSCRX_MOV_CRX,
1068 /** CrX (CR0) write caused by 'lmsw' instruction. */
1069 IEMACCESSCRX_LMSW,
1070 /** CrX (CR0) write caused by 'clts' instruction. */
1071 IEMACCESSCRX_CLTS,
1072 /** CrX (CR0) read caused by 'smsw' instruction. */
1073 IEMACCESSCRX_SMSW
1074} IEMACCESSCRX;
1075
1076#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1077/** @name IEM_SLAT_FAIL_XXX - Second-level address translation failure information.
1078 *
1079 * These flags provide further context to SLAT page-walk failures that could not be
1080 * determined by PGM (e.g, PGM is not privy to memory access permissions).
1081 *
1082 * @{
1083 */
1084/** Translating a nested-guest linear address failed accessing a nested-guest
1085 * physical address. */
1086# define IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR RT_BIT_32(0)
1087/** Translating a nested-guest linear address failed accessing a
1088 * paging-structure entry or updating accessed/dirty bits. */
1089# define IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE RT_BIT_32(1)
1090/** @} */
1091
1092DECLCALLBACK(FNPGMPHYSHANDLER) iemVmxApicAccessPageHandler;
1093# ifndef IN_RING3
1094DECLCALLBACK(FNPGMRZPHYSPFHANDLER) iemVmxApicAccessPagePfHandler;
1095# endif
1096#endif
1097
1098/**
1099 * Indicates to the verifier that the given flag set is undefined.
1100 *
1101 * Can be invoked again to add more flags.
1102 *
1103 * This is a NOOP if the verifier isn't compiled in.
1104 *
1105 * @note We're temporarily keeping this until code is converted to new
1106 * disassembler style opcode handling.
1107 */
1108#define IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fEfl) do { } while (0)
1109
1110
1111/** @def IEM_DECL_IMPL_TYPE
1112 * For typedef'ing an instruction implementation function.
1113 *
1114 * @param a_RetType The return type.
1115 * @param a_Name The name of the type.
1116 * @param a_ArgList The argument list enclosed in parentheses.
1117 */
1118
1119/** @def IEM_DECL_IMPL_DEF
1120 * For defining an instruction implementation function.
1121 *
1122 * @param a_RetType The return type.
1123 * @param a_Name The name of the type.
1124 * @param a_ArgList The argument list enclosed in parentheses.
1125 */
1126
1127#if defined(__GNUC__) && defined(RT_ARCH_X86)
1128# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1129 __attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
1130# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1131 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1132# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1133 __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1134
1135#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
1136# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1137 a_RetType (__fastcall a_Name) a_ArgList
1138# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1139 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1140# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1141 a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1142
1143#elif __cplusplus >= 201700 /* P0012R1 support */
1144# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1145 a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
1146# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1147 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1148# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1149 a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1150
1151#else
1152# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1153 a_RetType (VBOXCALL a_Name) a_ArgList
1154# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1155 a_RetType VBOXCALL a_Name a_ArgList
1156# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1157 a_RetType VBOXCALL a_Name a_ArgList
1158
1159#endif
1160
1161/** Defined in IEMAllAImplC.cpp but also used by IEMAllAImplA.asm. */
1162RT_C_DECLS_BEGIN
1163extern uint8_t const g_afParity[256];
1164RT_C_DECLS_END
1165
1166
1167/** @name Arithmetic assignment operations on bytes (binary).
1168 * @{ */
1169typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU8, (uint8_t *pu8Dst, uint8_t u8Src, uint32_t *pEFlags));
1170typedef FNIEMAIMPLBINU8 *PFNIEMAIMPLBINU8;
1171FNIEMAIMPLBINU8 iemAImpl_add_u8, iemAImpl_add_u8_locked;
1172FNIEMAIMPLBINU8 iemAImpl_adc_u8, iemAImpl_adc_u8_locked;
1173FNIEMAIMPLBINU8 iemAImpl_sub_u8, iemAImpl_sub_u8_locked;
1174FNIEMAIMPLBINU8 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked;
1175FNIEMAIMPLBINU8 iemAImpl_or_u8, iemAImpl_or_u8_locked;
1176FNIEMAIMPLBINU8 iemAImpl_xor_u8, iemAImpl_xor_u8_locked;
1177FNIEMAIMPLBINU8 iemAImpl_and_u8, iemAImpl_and_u8_locked;
1178/** @} */
1179
1180/** @name Arithmetic assignment operations on words (binary).
1181 * @{ */
1182typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU16, (uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pEFlags));
1183typedef FNIEMAIMPLBINU16 *PFNIEMAIMPLBINU16;
1184FNIEMAIMPLBINU16 iemAImpl_add_u16, iemAImpl_add_u16_locked;
1185FNIEMAIMPLBINU16 iemAImpl_adc_u16, iemAImpl_adc_u16_locked;
1186FNIEMAIMPLBINU16 iemAImpl_sub_u16, iemAImpl_sub_u16_locked;
1187FNIEMAIMPLBINU16 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked;
1188FNIEMAIMPLBINU16 iemAImpl_or_u16, iemAImpl_or_u16_locked;
1189FNIEMAIMPLBINU16 iemAImpl_xor_u16, iemAImpl_xor_u16_locked;
1190FNIEMAIMPLBINU16 iemAImpl_and_u16, iemAImpl_and_u16_locked;
1191/** @} */
1192
1193/** @name Arithmetic assignment operations on double words (binary).
1194 * @{ */
1195typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU32, (uint32_t *pu32Dst, uint32_t u32Src, uint32_t *pEFlags));
1196typedef FNIEMAIMPLBINU32 *PFNIEMAIMPLBINU32;
1197FNIEMAIMPLBINU32 iemAImpl_add_u32, iemAImpl_add_u32_locked;
1198FNIEMAIMPLBINU32 iemAImpl_adc_u32, iemAImpl_adc_u32_locked;
1199FNIEMAIMPLBINU32 iemAImpl_sub_u32, iemAImpl_sub_u32_locked;
1200FNIEMAIMPLBINU32 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked;
1201FNIEMAIMPLBINU32 iemAImpl_or_u32, iemAImpl_or_u32_locked;
1202FNIEMAIMPLBINU32 iemAImpl_xor_u32, iemAImpl_xor_u32_locked;
1203FNIEMAIMPLBINU32 iemAImpl_and_u32, iemAImpl_and_u32_locked;
1204FNIEMAIMPLBINU32 iemAImpl_blsi_u32, iemAImpl_blsi_u32_fallback;
1205FNIEMAIMPLBINU32 iemAImpl_blsr_u32, iemAImpl_blsr_u32_fallback;
1206FNIEMAIMPLBINU32 iemAImpl_blsmsk_u32, iemAImpl_blsmsk_u32_fallback;
1207/** @} */
1208
1209/** @name Arithmetic assignment operations on quad words (binary).
1210 * @{ */
1211typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU64, (uint64_t *pu64Dst, uint64_t u64Src, uint32_t *pEFlags));
1212typedef FNIEMAIMPLBINU64 *PFNIEMAIMPLBINU64;
1213FNIEMAIMPLBINU64 iemAImpl_add_u64, iemAImpl_add_u64_locked;
1214FNIEMAIMPLBINU64 iemAImpl_adc_u64, iemAImpl_adc_u64_locked;
1215FNIEMAIMPLBINU64 iemAImpl_sub_u64, iemAImpl_sub_u64_locked;
1216FNIEMAIMPLBINU64 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked;
1217FNIEMAIMPLBINU64 iemAImpl_or_u64, iemAImpl_or_u64_locked;
1218FNIEMAIMPLBINU64 iemAImpl_xor_u64, iemAImpl_xor_u64_locked;
1219FNIEMAIMPLBINU64 iemAImpl_and_u64, iemAImpl_and_u64_locked;
1220FNIEMAIMPLBINU64 iemAImpl_blsi_u64, iemAImpl_blsi_u64_fallback;
1221FNIEMAIMPLBINU64 iemAImpl_blsr_u64, iemAImpl_blsr_u64_fallback;
1222FNIEMAIMPLBINU64 iemAImpl_blsmsk_u64, iemAImpl_blsmsk_u64_fallback;
1223/** @} */
1224
1225/** @name Compare operations (thrown in with the binary ops).
1226 * @{ */
1227FNIEMAIMPLBINU8 iemAImpl_cmp_u8;
1228FNIEMAIMPLBINU16 iemAImpl_cmp_u16;
1229FNIEMAIMPLBINU32 iemAImpl_cmp_u32;
1230FNIEMAIMPLBINU64 iemAImpl_cmp_u64;
1231/** @} */
1232
1233/** @name Test operations (thrown in with the binary ops).
1234 * @{ */
1235FNIEMAIMPLBINU8 iemAImpl_test_u8;
1236FNIEMAIMPLBINU16 iemAImpl_test_u16;
1237FNIEMAIMPLBINU32 iemAImpl_test_u32;
1238FNIEMAIMPLBINU64 iemAImpl_test_u64;
1239/** @} */
1240
1241/** @name Bit operations operations (thrown in with the binary ops).
1242 * @{ */
1243FNIEMAIMPLBINU16 iemAImpl_bt_u16;
1244FNIEMAIMPLBINU32 iemAImpl_bt_u32;
1245FNIEMAIMPLBINU64 iemAImpl_bt_u64;
1246FNIEMAIMPLBINU16 iemAImpl_btc_u16, iemAImpl_btc_u16_locked;
1247FNIEMAIMPLBINU32 iemAImpl_btc_u32, iemAImpl_btc_u32_locked;
1248FNIEMAIMPLBINU64 iemAImpl_btc_u64, iemAImpl_btc_u64_locked;
1249FNIEMAIMPLBINU16 iemAImpl_btr_u16, iemAImpl_btr_u16_locked;
1250FNIEMAIMPLBINU32 iemAImpl_btr_u32, iemAImpl_btr_u32_locked;
1251FNIEMAIMPLBINU64 iemAImpl_btr_u64, iemAImpl_btr_u64_locked;
1252FNIEMAIMPLBINU16 iemAImpl_bts_u16, iemAImpl_bts_u16_locked;
1253FNIEMAIMPLBINU32 iemAImpl_bts_u32, iemAImpl_bts_u32_locked;
1254FNIEMAIMPLBINU64 iemAImpl_bts_u64, iemAImpl_bts_u64_locked;
1255/** @} */
1256
1257/** @name Arithmetic three operand operations on double words (binary).
1258 * @{ */
1259typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2, uint32_t *pEFlags));
1260typedef FNIEMAIMPLBINVEXU32 *PFNIEMAIMPLBINVEXU32;
1261FNIEMAIMPLBINVEXU32 iemAImpl_andn_u32, iemAImpl_andn_u32_fallback;
1262FNIEMAIMPLBINVEXU32 iemAImpl_bextr_u32, iemAImpl_bextr_u32_fallback;
1263FNIEMAIMPLBINVEXU32 iemAImpl_bzhi_u32, iemAImpl_bzhi_u32_fallback;
1264/** @} */
1265
1266/** @name Arithmetic three operand operations on quad words (binary).
1267 * @{ */
1268typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2, uint32_t *pEFlags));
1269typedef FNIEMAIMPLBINVEXU64 *PFNIEMAIMPLBINVEXU64;
1270FNIEMAIMPLBINVEXU64 iemAImpl_andn_u64, iemAImpl_andn_u64_fallback;
1271FNIEMAIMPLBINVEXU64 iemAImpl_bextr_u64, iemAImpl_bextr_u64_fallback;
1272FNIEMAIMPLBINVEXU64 iemAImpl_bzhi_u64, iemAImpl_bzhi_u64_fallback;
1273/** @} */
1274
1275/** @name Arithmetic three operand operations on double words w/o EFLAGS (binary).
1276 * @{ */
1277typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU32NOEFL, (uint32_t *pu32Dst, uint32_t u32Src1, uint32_t u32Src2));
1278typedef FNIEMAIMPLBINVEXU32NOEFL *PFNIEMAIMPLBINVEXU32NOEFL;
1279FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pdep_u32, iemAImpl_pdep_u32_fallback;
1280FNIEMAIMPLBINVEXU32NOEFL iemAImpl_pext_u32, iemAImpl_pext_u32_fallback;
1281FNIEMAIMPLBINVEXU32NOEFL iemAImpl_sarx_u32, iemAImpl_sarx_u32_fallback;
1282FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shlx_u32, iemAImpl_shlx_u32_fallback;
1283FNIEMAIMPLBINVEXU32NOEFL iemAImpl_shrx_u32, iemAImpl_shrx_u32_fallback;
1284FNIEMAIMPLBINVEXU32NOEFL iemAImpl_rorx_u32;
1285/** @} */
1286
1287/** @name Arithmetic three operand operations on quad words w/o EFLAGS (binary).
1288 * @{ */
1289typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINVEXU64NOEFL, (uint64_t *pu64Dst, uint64_t u64Src1, uint64_t u64Src2));
1290typedef FNIEMAIMPLBINVEXU64NOEFL *PFNIEMAIMPLBINVEXU64NOEFL;
1291FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pdep_u64, iemAImpl_pdep_u64_fallback;
1292FNIEMAIMPLBINVEXU64NOEFL iemAImpl_pext_u64, iemAImpl_pext_u64_fallback;
1293FNIEMAIMPLBINVEXU64NOEFL iemAImpl_sarx_u64, iemAImpl_sarx_u64_fallback;
1294FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shlx_u64, iemAImpl_shlx_u64_fallback;
1295FNIEMAIMPLBINVEXU64NOEFL iemAImpl_shrx_u64, iemAImpl_shrx_u64_fallback;
1296FNIEMAIMPLBINVEXU64NOEFL iemAImpl_rorx_u64;
1297/** @} */
1298
1299/** @name MULX 32-bit and 64-bit.
1300 * @{ */
1301typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU32, (uint32_t *puDst1, uint32_t *puDst2, uint32_t uSrc1, uint32_t uSrc2));
1302typedef FNIEMAIMPLMULXVEXU32 *PFNIEMAIMPLMULXVEXU32;
1303FNIEMAIMPLMULXVEXU32 iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback;
1304
1305typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMULXVEXU64, (uint64_t *puDst1, uint64_t *puDst2, uint64_t uSrc1, uint64_t uSrc2));
1306typedef FNIEMAIMPLMULXVEXU64 *PFNIEMAIMPLMULXVEXU64;
1307FNIEMAIMPLMULXVEXU64 iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback;
1308/** @} */
1309
1310
1311/** @name Exchange memory with register operations.
1312 * @{ */
1313IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1314IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1315IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1316IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1317IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked, (uint8_t *pu8Mem, uint8_t *pu8Reg));
1318IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
1319IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
1320IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
1321/** @} */
1322
1323/** @name Exchange and add operations.
1324 * @{ */
1325IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1326IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1327IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1328IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1329IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8_locked, (uint8_t *pu8Dst, uint8_t *pu8Reg, uint32_t *pEFlags));
1330IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16_locked,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
1331IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32_locked,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
1332IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
1333/** @} */
1334
1335/** @name Compare and exchange.
1336 * @{ */
1337IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1338IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1339IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1340IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1341IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1342IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1343#if ARCH_BITS == 32
1344IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1345IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
1346#else
1347IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1348IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1349#endif
1350IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1351 uint32_t *pEFlags));
1352IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1353 uint32_t *pEFlags));
1354IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1355 uint32_t *pEFlags));
1356IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1357 uint32_t *pEFlags));
1358#ifndef RT_ARCH_ARM64
1359IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1360 PRTUINT128U pu128RbxRcx, uint32_t *pEFlags));
1361#endif
1362/** @} */
1363
1364/** @name Memory ordering
1365 * @{ */
1366typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEMFENCE,(void));
1367typedef FNIEMAIMPLMEMFENCE *PFNIEMAIMPLMEMFENCE;
1368IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void));
1369IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void));
1370IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void));
1371#ifndef RT_ARCH_ARM64
1372IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void));
1373#endif
1374/** @} */
1375
1376/** @name Double precision shifts
1377 * @{ */
1378typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU16,(uint16_t *pu16Dst, uint16_t u16Src, uint8_t cShift, uint32_t *pEFlags));
1379typedef FNIEMAIMPLSHIFTDBLU16 *PFNIEMAIMPLSHIFTDBLU16;
1380typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU32,(uint32_t *pu32Dst, uint32_t u32Src, uint8_t cShift, uint32_t *pEFlags));
1381typedef FNIEMAIMPLSHIFTDBLU32 *PFNIEMAIMPLSHIFTDBLU32;
1382typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU64,(uint64_t *pu64Dst, uint64_t u64Src, uint8_t cShift, uint32_t *pEFlags));
1383typedef FNIEMAIMPLSHIFTDBLU64 *PFNIEMAIMPLSHIFTDBLU64;
1384FNIEMAIMPLSHIFTDBLU16 iemAImpl_shld_u16, iemAImpl_shld_u16_amd, iemAImpl_shld_u16_intel;
1385FNIEMAIMPLSHIFTDBLU32 iemAImpl_shld_u32, iemAImpl_shld_u32_amd, iemAImpl_shld_u32_intel;
1386FNIEMAIMPLSHIFTDBLU64 iemAImpl_shld_u64, iemAImpl_shld_u64_amd, iemAImpl_shld_u64_intel;
1387FNIEMAIMPLSHIFTDBLU16 iemAImpl_shrd_u16, iemAImpl_shrd_u16_amd, iemAImpl_shrd_u16_intel;
1388FNIEMAIMPLSHIFTDBLU32 iemAImpl_shrd_u32, iemAImpl_shrd_u32_amd, iemAImpl_shrd_u32_intel;
1389FNIEMAIMPLSHIFTDBLU64 iemAImpl_shrd_u64, iemAImpl_shrd_u64_amd, iemAImpl_shrd_u64_intel;
1390/** @} */
1391
1392
1393/** @name Bit search operations (thrown in with the binary ops).
1394 * @{ */
1395FNIEMAIMPLBINU16 iemAImpl_bsf_u16, iemAImpl_bsf_u16_amd, iemAImpl_bsf_u16_intel;
1396FNIEMAIMPLBINU32 iemAImpl_bsf_u32, iemAImpl_bsf_u32_amd, iemAImpl_bsf_u32_intel;
1397FNIEMAIMPLBINU64 iemAImpl_bsf_u64, iemAImpl_bsf_u64_amd, iemAImpl_bsf_u64_intel;
1398FNIEMAIMPLBINU16 iemAImpl_bsr_u16, iemAImpl_bsr_u16_amd, iemAImpl_bsr_u16_intel;
1399FNIEMAIMPLBINU32 iemAImpl_bsr_u32, iemAImpl_bsr_u32_amd, iemAImpl_bsr_u32_intel;
1400FNIEMAIMPLBINU64 iemAImpl_bsr_u64, iemAImpl_bsr_u64_amd, iemAImpl_bsr_u64_intel;
1401FNIEMAIMPLBINU16 iemAImpl_lzcnt_u16, iemAImpl_lzcnt_u16_amd, iemAImpl_lzcnt_u16_intel;
1402FNIEMAIMPLBINU32 iemAImpl_lzcnt_u32, iemAImpl_lzcnt_u32_amd, iemAImpl_lzcnt_u32_intel;
1403FNIEMAIMPLBINU64 iemAImpl_lzcnt_u64, iemAImpl_lzcnt_u64_amd, iemAImpl_lzcnt_u64_intel;
1404FNIEMAIMPLBINU16 iemAImpl_tzcnt_u16, iemAImpl_tzcnt_u16_amd, iemAImpl_tzcnt_u16_intel;
1405FNIEMAIMPLBINU32 iemAImpl_tzcnt_u32, iemAImpl_tzcnt_u32_amd, iemAImpl_tzcnt_u32_intel;
1406FNIEMAIMPLBINU64 iemAImpl_tzcnt_u64, iemAImpl_tzcnt_u64_amd, iemAImpl_tzcnt_u64_intel;
1407FNIEMAIMPLBINU16 iemAImpl_popcnt_u16, iemAImpl_popcnt_u16_fallback;
1408FNIEMAIMPLBINU32 iemAImpl_popcnt_u32, iemAImpl_popcnt_u32_fallback;
1409FNIEMAIMPLBINU64 iemAImpl_popcnt_u64, iemAImpl_popcnt_u64_fallback;
1410/** @} */
1411
1412/** @name Signed multiplication operations (thrown in with the binary ops).
1413 * @{ */
1414FNIEMAIMPLBINU16 iemAImpl_imul_two_u16, iemAImpl_imul_two_u16_amd, iemAImpl_imul_two_u16_intel;
1415FNIEMAIMPLBINU32 iemAImpl_imul_two_u32, iemAImpl_imul_two_u32_amd, iemAImpl_imul_two_u32_intel;
1416FNIEMAIMPLBINU64 iemAImpl_imul_two_u64, iemAImpl_imul_two_u64_amd, iemAImpl_imul_two_u64_intel;
1417/** @} */
1418
1419/** @name Arithmetic assignment operations on bytes (unary).
1420 * @{ */
1421typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU8, (uint8_t *pu8Dst, uint32_t *pEFlags));
1422typedef FNIEMAIMPLUNARYU8 *PFNIEMAIMPLUNARYU8;
1423FNIEMAIMPLUNARYU8 iemAImpl_inc_u8, iemAImpl_inc_u8_locked;
1424FNIEMAIMPLUNARYU8 iemAImpl_dec_u8, iemAImpl_dec_u8_locked;
1425FNIEMAIMPLUNARYU8 iemAImpl_not_u8, iemAImpl_not_u8_locked;
1426FNIEMAIMPLUNARYU8 iemAImpl_neg_u8, iemAImpl_neg_u8_locked;
1427/** @} */
1428
1429/** @name Arithmetic assignment operations on words (unary).
1430 * @{ */
1431typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU16, (uint16_t *pu16Dst, uint32_t *pEFlags));
1432typedef FNIEMAIMPLUNARYU16 *PFNIEMAIMPLUNARYU16;
1433FNIEMAIMPLUNARYU16 iemAImpl_inc_u16, iemAImpl_inc_u16_locked;
1434FNIEMAIMPLUNARYU16 iemAImpl_dec_u16, iemAImpl_dec_u16_locked;
1435FNIEMAIMPLUNARYU16 iemAImpl_not_u16, iemAImpl_not_u16_locked;
1436FNIEMAIMPLUNARYU16 iemAImpl_neg_u16, iemAImpl_neg_u16_locked;
1437/** @} */
1438
1439/** @name Arithmetic assignment operations on double words (unary).
1440 * @{ */
1441typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU32, (uint32_t *pu32Dst, uint32_t *pEFlags));
1442typedef FNIEMAIMPLUNARYU32 *PFNIEMAIMPLUNARYU32;
1443FNIEMAIMPLUNARYU32 iemAImpl_inc_u32, iemAImpl_inc_u32_locked;
1444FNIEMAIMPLUNARYU32 iemAImpl_dec_u32, iemAImpl_dec_u32_locked;
1445FNIEMAIMPLUNARYU32 iemAImpl_not_u32, iemAImpl_not_u32_locked;
1446FNIEMAIMPLUNARYU32 iemAImpl_neg_u32, iemAImpl_neg_u32_locked;
1447/** @} */
1448
1449/** @name Arithmetic assignment operations on quad words (unary).
1450 * @{ */
1451typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU64, (uint64_t *pu64Dst, uint32_t *pEFlags));
1452typedef FNIEMAIMPLUNARYU64 *PFNIEMAIMPLUNARYU64;
1453FNIEMAIMPLUNARYU64 iemAImpl_inc_u64, iemAImpl_inc_u64_locked;
1454FNIEMAIMPLUNARYU64 iemAImpl_dec_u64, iemAImpl_dec_u64_locked;
1455FNIEMAIMPLUNARYU64 iemAImpl_not_u64, iemAImpl_not_u64_locked;
1456FNIEMAIMPLUNARYU64 iemAImpl_neg_u64, iemAImpl_neg_u64_locked;
1457/** @} */
1458
1459
1460/** @name Shift operations on bytes (Group 2).
1461 * @{ */
1462typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU8,(uint8_t *pu8Dst, uint8_t cShift, uint32_t *pEFlags));
1463typedef FNIEMAIMPLSHIFTU8 *PFNIEMAIMPLSHIFTU8;
1464FNIEMAIMPLSHIFTU8 iemAImpl_rol_u8, iemAImpl_rol_u8_amd, iemAImpl_rol_u8_intel;
1465FNIEMAIMPLSHIFTU8 iemAImpl_ror_u8, iemAImpl_ror_u8_amd, iemAImpl_ror_u8_intel;
1466FNIEMAIMPLSHIFTU8 iemAImpl_rcl_u8, iemAImpl_rcl_u8_amd, iemAImpl_rcl_u8_intel;
1467FNIEMAIMPLSHIFTU8 iemAImpl_rcr_u8, iemAImpl_rcr_u8_amd, iemAImpl_rcr_u8_intel;
1468FNIEMAIMPLSHIFTU8 iemAImpl_shl_u8, iemAImpl_shl_u8_amd, iemAImpl_shl_u8_intel;
1469FNIEMAIMPLSHIFTU8 iemAImpl_shr_u8, iemAImpl_shr_u8_amd, iemAImpl_shr_u8_intel;
1470FNIEMAIMPLSHIFTU8 iemAImpl_sar_u8, iemAImpl_sar_u8_amd, iemAImpl_sar_u8_intel;
1471/** @} */
1472
1473/** @name Shift operations on words (Group 2).
1474 * @{ */
1475typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU16,(uint16_t *pu16Dst, uint8_t cShift, uint32_t *pEFlags));
1476typedef FNIEMAIMPLSHIFTU16 *PFNIEMAIMPLSHIFTU16;
1477FNIEMAIMPLSHIFTU16 iemAImpl_rol_u16, iemAImpl_rol_u16_amd, iemAImpl_rol_u16_intel;
1478FNIEMAIMPLSHIFTU16 iemAImpl_ror_u16, iemAImpl_ror_u16_amd, iemAImpl_ror_u16_intel;
1479FNIEMAIMPLSHIFTU16 iemAImpl_rcl_u16, iemAImpl_rcl_u16_amd, iemAImpl_rcl_u16_intel;
1480FNIEMAIMPLSHIFTU16 iemAImpl_rcr_u16, iemAImpl_rcr_u16_amd, iemAImpl_rcr_u16_intel;
1481FNIEMAIMPLSHIFTU16 iemAImpl_shl_u16, iemAImpl_shl_u16_amd, iemAImpl_shl_u16_intel;
1482FNIEMAIMPLSHIFTU16 iemAImpl_shr_u16, iemAImpl_shr_u16_amd, iemAImpl_shr_u16_intel;
1483FNIEMAIMPLSHIFTU16 iemAImpl_sar_u16, iemAImpl_sar_u16_amd, iemAImpl_sar_u16_intel;
1484/** @} */
1485
1486/** @name Shift operations on double words (Group 2).
1487 * @{ */
1488typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU32,(uint32_t *pu32Dst, uint8_t cShift, uint32_t *pEFlags));
1489typedef FNIEMAIMPLSHIFTU32 *PFNIEMAIMPLSHIFTU32;
1490FNIEMAIMPLSHIFTU32 iemAImpl_rol_u32, iemAImpl_rol_u32_amd, iemAImpl_rol_u32_intel;
1491FNIEMAIMPLSHIFTU32 iemAImpl_ror_u32, iemAImpl_ror_u32_amd, iemAImpl_ror_u32_intel;
1492FNIEMAIMPLSHIFTU32 iemAImpl_rcl_u32, iemAImpl_rcl_u32_amd, iemAImpl_rcl_u32_intel;
1493FNIEMAIMPLSHIFTU32 iemAImpl_rcr_u32, iemAImpl_rcr_u32_amd, iemAImpl_rcr_u32_intel;
1494FNIEMAIMPLSHIFTU32 iemAImpl_shl_u32, iemAImpl_shl_u32_amd, iemAImpl_shl_u32_intel;
1495FNIEMAIMPLSHIFTU32 iemAImpl_shr_u32, iemAImpl_shr_u32_amd, iemAImpl_shr_u32_intel;
1496FNIEMAIMPLSHIFTU32 iemAImpl_sar_u32, iemAImpl_sar_u32_amd, iemAImpl_sar_u32_intel;
1497/** @} */
1498
1499/** @name Shift operations on words (Group 2).
1500 * @{ */
1501typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU64,(uint64_t *pu64Dst, uint8_t cShift, uint32_t *pEFlags));
1502typedef FNIEMAIMPLSHIFTU64 *PFNIEMAIMPLSHIFTU64;
1503FNIEMAIMPLSHIFTU64 iemAImpl_rol_u64, iemAImpl_rol_u64_amd, iemAImpl_rol_u64_intel;
1504FNIEMAIMPLSHIFTU64 iemAImpl_ror_u64, iemAImpl_ror_u64_amd, iemAImpl_ror_u64_intel;
1505FNIEMAIMPLSHIFTU64 iemAImpl_rcl_u64, iemAImpl_rcl_u64_amd, iemAImpl_rcl_u64_intel;
1506FNIEMAIMPLSHIFTU64 iemAImpl_rcr_u64, iemAImpl_rcr_u64_amd, iemAImpl_rcr_u64_intel;
1507FNIEMAIMPLSHIFTU64 iemAImpl_shl_u64, iemAImpl_shl_u64_amd, iemAImpl_shl_u64_intel;
1508FNIEMAIMPLSHIFTU64 iemAImpl_shr_u64, iemAImpl_shr_u64_amd, iemAImpl_shr_u64_intel;
1509FNIEMAIMPLSHIFTU64 iemAImpl_sar_u64, iemAImpl_sar_u64_amd, iemAImpl_sar_u64_intel;
1510/** @} */
1511
1512/** @name Multiplication and division operations.
1513 * @{ */
1514typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU8,(uint16_t *pu16AX, uint8_t u8FactorDivisor, uint32_t *pEFlags));
1515typedef FNIEMAIMPLMULDIVU8 *PFNIEMAIMPLMULDIVU8;
1516FNIEMAIMPLMULDIVU8 iemAImpl_mul_u8, iemAImpl_mul_u8_amd, iemAImpl_mul_u8_intel;
1517FNIEMAIMPLMULDIVU8 iemAImpl_imul_u8, iemAImpl_imul_u8_amd, iemAImpl_imul_u8_intel;
1518FNIEMAIMPLMULDIVU8 iemAImpl_div_u8, iemAImpl_div_u8_amd, iemAImpl_div_u8_intel;
1519FNIEMAIMPLMULDIVU8 iemAImpl_idiv_u8, iemAImpl_idiv_u8_amd, iemAImpl_idiv_u8_intel;
1520
1521typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU16,(uint16_t *pu16AX, uint16_t *pu16DX, uint16_t u16FactorDivisor, uint32_t *pEFlags));
1522typedef FNIEMAIMPLMULDIVU16 *PFNIEMAIMPLMULDIVU16;
1523FNIEMAIMPLMULDIVU16 iemAImpl_mul_u16, iemAImpl_mul_u16_amd, iemAImpl_mul_u16_intel;
1524FNIEMAIMPLMULDIVU16 iemAImpl_imul_u16, iemAImpl_imul_u16_amd, iemAImpl_imul_u16_intel;
1525FNIEMAIMPLMULDIVU16 iemAImpl_div_u16, iemAImpl_div_u16_amd, iemAImpl_div_u16_intel;
1526FNIEMAIMPLMULDIVU16 iemAImpl_idiv_u16, iemAImpl_idiv_u16_amd, iemAImpl_idiv_u16_intel;
1527
1528typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU32,(uint32_t *pu32EAX, uint32_t *pu32EDX, uint32_t u32FactorDivisor, uint32_t *pEFlags));
1529typedef FNIEMAIMPLMULDIVU32 *PFNIEMAIMPLMULDIVU32;
1530FNIEMAIMPLMULDIVU32 iemAImpl_mul_u32, iemAImpl_mul_u32_amd, iemAImpl_mul_u32_intel;
1531FNIEMAIMPLMULDIVU32 iemAImpl_imul_u32, iemAImpl_imul_u32_amd, iemAImpl_imul_u32_intel;
1532FNIEMAIMPLMULDIVU32 iemAImpl_div_u32, iemAImpl_div_u32_amd, iemAImpl_div_u32_intel;
1533FNIEMAIMPLMULDIVU32 iemAImpl_idiv_u32, iemAImpl_idiv_u32_amd, iemAImpl_idiv_u32_intel;
1534
1535typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64FactorDivisor, uint32_t *pEFlags));
1536typedef FNIEMAIMPLMULDIVU64 *PFNIEMAIMPLMULDIVU64;
1537FNIEMAIMPLMULDIVU64 iemAImpl_mul_u64, iemAImpl_mul_u64_amd, iemAImpl_mul_u64_intel;
1538FNIEMAIMPLMULDIVU64 iemAImpl_imul_u64, iemAImpl_imul_u64_amd, iemAImpl_imul_u64_intel;
1539FNIEMAIMPLMULDIVU64 iemAImpl_div_u64, iemAImpl_div_u64_amd, iemAImpl_div_u64_intel;
1540FNIEMAIMPLMULDIVU64 iemAImpl_idiv_u64, iemAImpl_idiv_u64_amd, iemAImpl_idiv_u64_intel;
1541/** @} */
1542
1543/** @name Byte Swap.
1544 * @{ */
1545IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u16,(uint32_t *pu32Dst)); /* Yes, 32-bit register access. */
1546IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u32,(uint32_t *pu32Dst));
1547IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u64,(uint64_t *pu64Dst));
1548/** @} */
1549
1550/** @name Misc.
1551 * @{ */
1552FNIEMAIMPLBINU16 iemAImpl_arpl;
1553/** @} */
1554
1555
1556/** @name FPU operations taking a 32-bit float argument
1557 * @{ */
1558typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1559 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1560typedef FNIEMAIMPLFPUR32FSW *PFNIEMAIMPLFPUR32FSW;
1561
1562typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1563 PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1564typedef FNIEMAIMPLFPUR32 *PFNIEMAIMPLFPUR32;
1565
1566FNIEMAIMPLFPUR32FSW iemAImpl_fcom_r80_by_r32;
1567FNIEMAIMPLFPUR32 iemAImpl_fadd_r80_by_r32;
1568FNIEMAIMPLFPUR32 iemAImpl_fmul_r80_by_r32;
1569FNIEMAIMPLFPUR32 iemAImpl_fsub_r80_by_r32;
1570FNIEMAIMPLFPUR32 iemAImpl_fsubr_r80_by_r32;
1571FNIEMAIMPLFPUR32 iemAImpl_fdiv_r80_by_r32;
1572FNIEMAIMPLFPUR32 iemAImpl_fdivr_r80_by_r32;
1573
1574IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val));
1575IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1576 PRTFLOAT32U pr32Val, PCRTFLOAT80U pr80Val));
1577/** @} */
1578
1579/** @name FPU operations taking a 64-bit float argument
1580 * @{ */
1581typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1582 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1583typedef FNIEMAIMPLFPUR64FSW *PFNIEMAIMPLFPUR64FSW;
1584
1585typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1586 PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1587typedef FNIEMAIMPLFPUR64 *PFNIEMAIMPLFPUR64;
1588
1589FNIEMAIMPLFPUR64FSW iemAImpl_fcom_r80_by_r64;
1590FNIEMAIMPLFPUR64 iemAImpl_fadd_r80_by_r64;
1591FNIEMAIMPLFPUR64 iemAImpl_fmul_r80_by_r64;
1592FNIEMAIMPLFPUR64 iemAImpl_fsub_r80_by_r64;
1593FNIEMAIMPLFPUR64 iemAImpl_fsubr_r80_by_r64;
1594FNIEMAIMPLFPUR64 iemAImpl_fdiv_r80_by_r64;
1595FNIEMAIMPLFPUR64 iemAImpl_fdivr_r80_by_r64;
1596
1597IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val));
1598IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1599 PRTFLOAT64U pr32Val, PCRTFLOAT80U pr80Val));
1600/** @} */
1601
1602/** @name FPU operations taking a 80-bit float argument
1603 * @{ */
1604typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1605 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1606typedef FNIEMAIMPLFPUR80 *PFNIEMAIMPLFPUR80;
1607FNIEMAIMPLFPUR80 iemAImpl_fadd_r80_by_r80;
1608FNIEMAIMPLFPUR80 iemAImpl_fmul_r80_by_r80;
1609FNIEMAIMPLFPUR80 iemAImpl_fsub_r80_by_r80;
1610FNIEMAIMPLFPUR80 iemAImpl_fsubr_r80_by_r80;
1611FNIEMAIMPLFPUR80 iemAImpl_fdiv_r80_by_r80;
1612FNIEMAIMPLFPUR80 iemAImpl_fdivr_r80_by_r80;
1613FNIEMAIMPLFPUR80 iemAImpl_fprem_r80_by_r80;
1614FNIEMAIMPLFPUR80 iemAImpl_fprem1_r80_by_r80;
1615FNIEMAIMPLFPUR80 iemAImpl_fscale_r80_by_r80;
1616
1617FNIEMAIMPLFPUR80 iemAImpl_fpatan_r80_by_r80, iemAImpl_fpatan_r80_by_r80_amd, iemAImpl_fpatan_r80_by_r80_intel;
1618FNIEMAIMPLFPUR80 iemAImpl_fyl2x_r80_by_r80, iemAImpl_fyl2x_r80_by_r80_amd, iemAImpl_fyl2x_r80_by_r80_intel;
1619FNIEMAIMPLFPUR80 iemAImpl_fyl2xp1_r80_by_r80, iemAImpl_fyl2xp1_r80_by_r80_amd, iemAImpl_fyl2xp1_r80_by_r80_intel;
1620
1621typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1622 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1623typedef FNIEMAIMPLFPUR80FSW *PFNIEMAIMPLFPUR80FSW;
1624FNIEMAIMPLFPUR80FSW iemAImpl_fcom_r80_by_r80;
1625FNIEMAIMPLFPUR80FSW iemAImpl_fucom_r80_by_r80;
1626
1627typedef IEM_DECL_IMPL_TYPE(uint32_t, FNIEMAIMPLFPUR80EFL,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
1628 PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1629typedef FNIEMAIMPLFPUR80EFL *PFNIEMAIMPLFPUR80EFL;
1630FNIEMAIMPLFPUR80EFL iemAImpl_fcomi_r80_by_r80;
1631FNIEMAIMPLFPUR80EFL iemAImpl_fucomi_r80_by_r80;
1632
1633typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARY,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1634typedef FNIEMAIMPLFPUR80UNARY *PFNIEMAIMPLFPUR80UNARY;
1635FNIEMAIMPLFPUR80UNARY iemAImpl_fabs_r80;
1636FNIEMAIMPLFPUR80UNARY iemAImpl_fchs_r80;
1637FNIEMAIMPLFPUR80UNARY iemAImpl_f2xm1_r80, iemAImpl_f2xm1_r80_amd, iemAImpl_f2xm1_r80_intel;
1638FNIEMAIMPLFPUR80UNARY iemAImpl_fsqrt_r80;
1639FNIEMAIMPLFPUR80UNARY iemAImpl_frndint_r80;
1640FNIEMAIMPLFPUR80UNARY iemAImpl_fsin_r80, iemAImpl_fsin_r80_amd, iemAImpl_fsin_r80_intel;
1641FNIEMAIMPLFPUR80UNARY iemAImpl_fcos_r80, iemAImpl_fcos_r80_amd, iemAImpl_fcos_r80_intel;
1642
1643typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYFSW,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val));
1644typedef FNIEMAIMPLFPUR80UNARYFSW *PFNIEMAIMPLFPUR80UNARYFSW;
1645FNIEMAIMPLFPUR80UNARYFSW iemAImpl_ftst_r80;
1646FNIEMAIMPLFPUR80UNARYFSW iemAImpl_fxam_r80;
1647
1648typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80LDCONST,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes));
1649typedef FNIEMAIMPLFPUR80LDCONST *PFNIEMAIMPLFPUR80LDCONST;
1650FNIEMAIMPLFPUR80LDCONST iemAImpl_fld1;
1651FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2t;
1652FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2e;
1653FNIEMAIMPLFPUR80LDCONST iemAImpl_fldpi;
1654FNIEMAIMPLFPUR80LDCONST iemAImpl_fldlg2;
1655FNIEMAIMPLFPUR80LDCONST iemAImpl_fldln2;
1656FNIEMAIMPLFPUR80LDCONST iemAImpl_fldz;
1657
1658typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYTWO,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo,
1659 PCRTFLOAT80U pr80Val));
1660typedef FNIEMAIMPLFPUR80UNARYTWO *PFNIEMAIMPLFPUR80UNARYTWO;
1661FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fptan_r80_r80, iemAImpl_fptan_r80_r80_amd, iemAImpl_fptan_r80_r80_intel;
1662FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fxtract_r80_r80;
1663FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fsincos_r80_r80, iemAImpl_fsincos_r80_r80_amd, iemAImpl_fsincos_r80_r80_intel;
1664
1665IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1666IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1667 PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src));
1668
1669IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val));
1670IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1671 PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src));
1672
1673/** @} */
1674
1675/** @name FPU operations taking a 16-bit signed integer argument
1676 * @{ */
1677typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1678 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1679typedef FNIEMAIMPLFPUI16 *PFNIEMAIMPLFPUI16;
1680typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI16,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1681 int16_t *pi16Dst, PCRTFLOAT80U pr80Src));
1682typedef FNIEMAIMPLFPUSTR80TOI16 *PFNIEMAIMPLFPUSTR80TOI16;
1683
1684FNIEMAIMPLFPUI16 iemAImpl_fiadd_r80_by_i16;
1685FNIEMAIMPLFPUI16 iemAImpl_fimul_r80_by_i16;
1686FNIEMAIMPLFPUI16 iemAImpl_fisub_r80_by_i16;
1687FNIEMAIMPLFPUI16 iemAImpl_fisubr_r80_by_i16;
1688FNIEMAIMPLFPUI16 iemAImpl_fidiv_r80_by_i16;
1689FNIEMAIMPLFPUI16 iemAImpl_fidivr_r80_by_i16;
1690
1691typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1692 PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1693typedef FNIEMAIMPLFPUI16FSW *PFNIEMAIMPLFPUI16FSW;
1694FNIEMAIMPLFPUI16FSW iemAImpl_ficom_r80_by_i16;
1695
1696IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val));
1697FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fist_r80_to_i16;
1698FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fistt_r80_to_i16, iemAImpl_fistt_r80_to_i16_amd, iemAImpl_fistt_r80_to_i16_intel;
1699/** @} */
1700
1701/** @name FPU operations taking a 32-bit signed integer argument
1702 * @{ */
1703typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1704 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1705typedef FNIEMAIMPLFPUI32 *PFNIEMAIMPLFPUI32;
1706typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI32,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1707 int32_t *pi32Dst, PCRTFLOAT80U pr80Src));
1708typedef FNIEMAIMPLFPUSTR80TOI32 *PFNIEMAIMPLFPUSTR80TOI32;
1709
1710FNIEMAIMPLFPUI32 iemAImpl_fiadd_r80_by_i32;
1711FNIEMAIMPLFPUI32 iemAImpl_fimul_r80_by_i32;
1712FNIEMAIMPLFPUI32 iemAImpl_fisub_r80_by_i32;
1713FNIEMAIMPLFPUI32 iemAImpl_fisubr_r80_by_i32;
1714FNIEMAIMPLFPUI32 iemAImpl_fidiv_r80_by_i32;
1715FNIEMAIMPLFPUI32 iemAImpl_fidivr_r80_by_i32;
1716
1717typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1718 PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1719typedef FNIEMAIMPLFPUI32FSW *PFNIEMAIMPLFPUI32FSW;
1720FNIEMAIMPLFPUI32FSW iemAImpl_ficom_r80_by_i32;
1721
1722IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val));
1723FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fist_r80_to_i32;
1724FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fistt_r80_to_i32;
1725/** @} */
1726
1727/** @name FPU operations taking a 64-bit signed integer argument
1728 * @{ */
1729typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI64,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1730 int64_t *pi64Dst, PCRTFLOAT80U pr80Src));
1731typedef FNIEMAIMPLFPUSTR80TOI64 *PFNIEMAIMPLFPUSTR80TOI64;
1732
1733IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val));
1734FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fist_r80_to_i64;
1735FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fistt_r80_to_i64;
1736/** @} */
1737
1738
1739/** Temporary type representing a 256-bit vector register. */
1740typedef struct { uint64_t au64[4]; } IEMVMM256;
1741/** Temporary type pointing to a 256-bit vector register. */
1742typedef IEMVMM256 *PIEMVMM256;
1743/** Temporary type pointing to a const 256-bit vector register. */
1744typedef IEMVMM256 *PCIEMVMM256;
1745
1746
1747/** @name Media (SSE/MMX/AVX) operations: full1 + full2 -> full1.
1748 * @{ */
1749typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc));
1750typedef FNIEMAIMPLMEDIAF2U64 *PFNIEMAIMPLMEDIAF2U64;
1751typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
1752typedef FNIEMAIMPLMEDIAF2U128 *PFNIEMAIMPLMEDIAF2U128;
1753typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U128,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1754typedef FNIEMAIMPLMEDIAF3U128 *PFNIEMAIMPLMEDIAF3U128;
1755typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF3U256,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1756typedef FNIEMAIMPLMEDIAF3U256 *PFNIEMAIMPLMEDIAF3U256;
1757typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U64,(uint64_t *puDst, uint64_t const *puSrc));
1758typedef FNIEMAIMPLMEDIAOPTF2U64 *PFNIEMAIMPLMEDIAOPTF2U64;
1759typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF2U128,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1760typedef FNIEMAIMPLMEDIAOPTF2U128 *PFNIEMAIMPLMEDIAOPTF2U128;
1761typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U128,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2));
1762typedef FNIEMAIMPLMEDIAOPTF3U128 *PFNIEMAIMPLMEDIAOPTF3U128;
1763typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAOPTF3U256,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2));
1764typedef FNIEMAIMPLMEDIAOPTF3U256 *PFNIEMAIMPLMEDIAOPTF3U256;
1765FNIEMAIMPLMEDIAF2U64 iemAImpl_pshufb_u64, iemAImpl_pshufb_u64_fallback;
1766FNIEMAIMPLMEDIAF2U64 iemAImpl_pand_u64, iemAImpl_pandn_u64, iemAImpl_por_u64, iemAImpl_pxor_u64;
1767FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqd_u64;
1768FNIEMAIMPLMEDIAF2U64 iemAImpl_pcmpgtb_u64, iemAImpl_pcmpgtw_u64, iemAImpl_pcmpgtd_u64;
1769FNIEMAIMPLMEDIAF2U64 iemAImpl_paddb_u64, iemAImpl_paddsb_u64, iemAImpl_paddusb_u64;
1770FNIEMAIMPLMEDIAF2U64 iemAImpl_paddw_u64, iemAImpl_paddsw_u64, iemAImpl_paddusw_u64;
1771FNIEMAIMPLMEDIAF2U64 iemAImpl_paddd_u64;
1772FNIEMAIMPLMEDIAF2U64 iemAImpl_paddq_u64;
1773FNIEMAIMPLMEDIAF2U64 iemAImpl_psubb_u64, iemAImpl_psubsb_u64, iemAImpl_psubusb_u64;
1774FNIEMAIMPLMEDIAF2U64 iemAImpl_psubw_u64, iemAImpl_psubsw_u64, iemAImpl_psubusw_u64;
1775FNIEMAIMPLMEDIAF2U64 iemAImpl_psubd_u64;
1776FNIEMAIMPLMEDIAF2U64 iemAImpl_psubq_u64;
1777FNIEMAIMPLMEDIAF2U64 iemAImpl_pmaddwd_u64;
1778FNIEMAIMPLMEDIAF2U64 iemAImpl_pmullw_u64, iemAImpl_pmulhw_u64;
1779FNIEMAIMPLMEDIAF2U64 iemAImpl_pminub_u64, iemAImpl_pmaxub_u64;
1780FNIEMAIMPLMEDIAF2U64 iemAImpl_pminsw_u64, iemAImpl_pmaxsw_u64;
1781FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_psllw_u64, iemAImpl_psrlw_u64, iemAImpl_psraw_u64;
1782FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_pslld_u64, iemAImpl_psrld_u64, iemAImpl_psrad_u64;
1783FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_psllq_u64, iemAImpl_psrlq_u64;
1784FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packsswb_u64, iemAImpl_packuswb_u64;
1785FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_packssdw_u64;
1786FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_pmulhuw_u64;
1787
1788FNIEMAIMPLMEDIAF2U128 iemAImpl_pshufb_u128, iemAImpl_pshufb_u128_fallback;
1789FNIEMAIMPLMEDIAF2U128 iemAImpl_pand_u128, iemAImpl_pandn_u128, iemAImpl_por_u128, iemAImpl_pxor_u128;
1790FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqb_u128, iemAImpl_pcmpeqw_u128, iemAImpl_pcmpeqd_u128;
1791FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpeqq_u128, iemAImpl_pcmpeqq_u128_fallback;
1792FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtb_u128, iemAImpl_pcmpgtw_u128, iemAImpl_pcmpgtd_u128;
1793FNIEMAIMPLMEDIAF2U128 iemAImpl_pcmpgtq_u128, iemAImpl_pcmpgtq_u128_fallback;
1794FNIEMAIMPLMEDIAF2U128 iemAImpl_paddb_u128, iemAImpl_paddsb_u128, iemAImpl_paddusb_u128;
1795FNIEMAIMPLMEDIAF2U128 iemAImpl_paddw_u128, iemAImpl_paddsw_u128, iemAImpl_paddusw_u128;
1796FNIEMAIMPLMEDIAF2U128 iemAImpl_paddd_u128;
1797FNIEMAIMPLMEDIAF2U128 iemAImpl_paddq_u128;
1798FNIEMAIMPLMEDIAF2U128 iemAImpl_psubb_u128, iemAImpl_psubsb_u128, iemAImpl_psubusb_u128;
1799FNIEMAIMPLMEDIAF2U128 iemAImpl_psubw_u128, iemAImpl_psubsw_u128, iemAImpl_psubusw_u128;
1800FNIEMAIMPLMEDIAF2U128 iemAImpl_psubd_u128;
1801FNIEMAIMPLMEDIAF2U128 iemAImpl_psubq_u128;
1802FNIEMAIMPLMEDIAF2U128 iemAImpl_pmullw_u128, iemAImpl_pmullw_u128_fallback;
1803FNIEMAIMPLMEDIAF2U128 iemAImpl_pmulhw_u128;
1804FNIEMAIMPLMEDIAF2U128 iemAImpl_pmulld_u128, iemAImpl_pmulld_u128_fallback;
1805FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaddwd_u128;
1806FNIEMAIMPLMEDIAF2U128 iemAImpl_pminub_u128;
1807FNIEMAIMPLMEDIAF2U128 iemAImpl_pminud_u128, iemAImpl_pminud_u128_fallback;
1808FNIEMAIMPLMEDIAF2U128 iemAImpl_pminuw_u128, iemAImpl_pminuw_u128_fallback;
1809FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsb_u128, iemAImpl_pminsb_u128_fallback;
1810FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsd_u128, iemAImpl_pminsd_u128_fallback;
1811FNIEMAIMPLMEDIAF2U128 iemAImpl_pminsw_u128, iemAImpl_pminsw_u128_fallback;
1812FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxub_u128;
1813FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxud_u128, iemAImpl_pmaxud_u128_fallback;
1814FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxuw_u128, iemAImpl_pmaxuw_u128_fallback;
1815FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsb_u128, iemAImpl_pmaxsb_u128_fallback;
1816FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsw_u128;
1817FNIEMAIMPLMEDIAF2U128 iemAImpl_pmaxsd_u128, iemAImpl_pmaxsd_u128_fallback;
1818FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packsswb_u128, iemAImpl_packuswb_u128;
1819FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_packssdw_u128, iemAImpl_packusdw_u128;
1820FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_psllw_u128, iemAImpl_psrlw_u128, iemAImpl_psraw_u128;
1821FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pslld_u128, iemAImpl_psrld_u128, iemAImpl_psrad_u128;
1822FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_psllq_u128, iemAImpl_psrlq_u128;
1823FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_pmulhuw_u128;
1824
1825FNIEMAIMPLMEDIAF3U128 iemAImpl_vpshufb_u128, iemAImpl_vpshufb_u128_fallback;
1826FNIEMAIMPLMEDIAF3U128 iemAImpl_vpand_u128, iemAImpl_vpand_u128_fallback;
1827FNIEMAIMPLMEDIAF3U128 iemAImpl_vpandn_u128, iemAImpl_vpandn_u128_fallback;
1828FNIEMAIMPLMEDIAF3U128 iemAImpl_vpor_u128, iemAImpl_vpor_u128_fallback;
1829FNIEMAIMPLMEDIAF3U128 iemAImpl_vpxor_u128, iemAImpl_vpxor_u128_fallback;
1830FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqb_u128, iemAImpl_vpcmpeqb_u128_fallback;
1831FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqw_u128, iemAImpl_vpcmpeqw_u128_fallback;
1832FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqd_u128, iemAImpl_vpcmpeqd_u128_fallback;
1833FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpeqq_u128, iemAImpl_vpcmpeqq_u128_fallback;
1834FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtb_u128, iemAImpl_vpcmpgtb_u128_fallback;
1835FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtw_u128, iemAImpl_vpcmpgtw_u128_fallback;
1836FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtd_u128, iemAImpl_vpcmpgtd_u128_fallback;
1837FNIEMAIMPLMEDIAF3U128 iemAImpl_vpcmpgtq_u128, iemAImpl_vpcmpgtq_u128_fallback;
1838FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddb_u128, iemAImpl_vpaddb_u128_fallback;
1839FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddw_u128, iemAImpl_vpaddw_u128_fallback;
1840FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddd_u128, iemAImpl_vpaddd_u128_fallback;
1841FNIEMAIMPLMEDIAF3U128 iemAImpl_vpaddq_u128, iemAImpl_vpaddq_u128_fallback;
1842FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubb_u128, iemAImpl_vpsubb_u128_fallback;
1843FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubw_u128, iemAImpl_vpsubw_u128_fallback;
1844FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubd_u128, iemAImpl_vpsubd_u128_fallback;
1845FNIEMAIMPLMEDIAF3U128 iemAImpl_vpsubq_u128, iemAImpl_vpsubq_u128_fallback;
1846FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminub_u128, iemAImpl_vpminub_u128_fallback;
1847FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminuw_u128, iemAImpl_vpminuw_u128_fallback;
1848FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminud_u128, iemAImpl_vpminud_u128_fallback;
1849FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsb_u128, iemAImpl_vpminsb_u128_fallback;
1850FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsw_u128, iemAImpl_vpminsw_u128_fallback;
1851FNIEMAIMPLMEDIAF3U128 iemAImpl_vpminsd_u128, iemAImpl_vpminsd_u128_fallback;
1852FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxub_u128, iemAImpl_vpmaxub_u128_fallback;
1853FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxuw_u128, iemAImpl_vpmaxuw_u128_fallback;
1854FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxud_u128, iemAImpl_vpmaxud_u128_fallback;
1855FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsb_u128, iemAImpl_vpmaxsb_u128_fallback;
1856FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsw_u128, iemAImpl_vpmaxsw_u128_fallback;
1857FNIEMAIMPLMEDIAF3U128 iemAImpl_vpmaxsd_u128, iemAImpl_vpmaxsd_u128_fallback;
1858FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpacksswb_u128, iemAImpl_vpacksswb_u128_fallback;
1859FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackssdw_u128, iemAImpl_vpackssdw_u128_fallback;
1860FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackuswb_u128, iemAImpl_vpackuswb_u128_fallback;
1861FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpackusdw_u128, iemAImpl_vpackusdw_u128_fallback;
1862FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmullw_u128, iemAImpl_vpmullw_u128_fallback;
1863FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulld_u128, iemAImpl_vpmulld_u128_fallback;
1864FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulhw_u128, iemAImpl_vpmulhw_u128_fallback;
1865FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpmulhuw_u128, iemAImpl_vpmulhuw_u128_fallback;
1866
1867FNIEMAIMPLMEDIAF3U256 iemAImpl_vpshufb_u256, iemAImpl_vpshufb_u256_fallback;
1868FNIEMAIMPLMEDIAF3U256 iemAImpl_vpand_u256, iemAImpl_vpand_u256_fallback;
1869FNIEMAIMPLMEDIAF3U256 iemAImpl_vpandn_u256, iemAImpl_vpandn_u256_fallback;
1870FNIEMAIMPLMEDIAF3U256 iemAImpl_vpor_u256, iemAImpl_vpor_u256_fallback;
1871FNIEMAIMPLMEDIAF3U256 iemAImpl_vpxor_u256, iemAImpl_vpxor_u256_fallback;
1872FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqb_u256, iemAImpl_vpcmpeqb_u256_fallback;
1873FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqw_u256, iemAImpl_vpcmpeqw_u256_fallback;
1874FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqd_u256, iemAImpl_vpcmpeqd_u256_fallback;
1875FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpeqq_u256, iemAImpl_vpcmpeqq_u256_fallback;
1876FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtb_u256, iemAImpl_vpcmpgtb_u256_fallback;
1877FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtw_u256, iemAImpl_vpcmpgtw_u256_fallback;
1878FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtd_u256, iemAImpl_vpcmpgtd_u256_fallback;
1879FNIEMAIMPLMEDIAF3U256 iemAImpl_vpcmpgtq_u256, iemAImpl_vpcmpgtq_u256_fallback;
1880FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddb_u256, iemAImpl_vpaddb_u256_fallback;
1881FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddw_u256, iemAImpl_vpaddw_u256_fallback;
1882FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddd_u256, iemAImpl_vpaddd_u256_fallback;
1883FNIEMAIMPLMEDIAF3U256 iemAImpl_vpaddq_u256, iemAImpl_vpaddq_u256_fallback;
1884FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubb_u256, iemAImpl_vpsubb_u256_fallback;
1885FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubw_u256, iemAImpl_vpsubw_u256_fallback;
1886FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubd_u256, iemAImpl_vpsubd_u256_fallback;
1887FNIEMAIMPLMEDIAF3U256 iemAImpl_vpsubq_u256, iemAImpl_vpsubq_u256_fallback;
1888FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminub_u256, iemAImpl_vpminub_u256_fallback;
1889FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminuw_u256, iemAImpl_vpminuw_u256_fallback;
1890FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminud_u256, iemAImpl_vpminud_u256_fallback;
1891FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsb_u256, iemAImpl_vpminsb_u256_fallback;
1892FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsw_u256, iemAImpl_vpminsw_u256_fallback;
1893FNIEMAIMPLMEDIAF3U256 iemAImpl_vpminsd_u256, iemAImpl_vpminsd_u256_fallback;
1894FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxub_u256, iemAImpl_vpmaxub_u256_fallback;
1895FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxuw_u256, iemAImpl_vpmaxuw_u256_fallback;
1896FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxud_u256, iemAImpl_vpmaxud_u256_fallback;
1897FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsb_u256, iemAImpl_vpmaxsb_u256_fallback;
1898FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsw_u256, iemAImpl_vpmaxsw_u256_fallback;
1899FNIEMAIMPLMEDIAF3U256 iemAImpl_vpmaxsd_u256, iemAImpl_vpmaxsd_u256_fallback;
1900FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpacksswb_u256, iemAImpl_vpacksswb_u256_fallback;
1901FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackssdw_u256, iemAImpl_vpackssdw_u256_fallback;
1902FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackuswb_u256, iemAImpl_vpackuswb_u256_fallback;
1903FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpackusdw_u256, iemAImpl_vpackusdw_u256_fallback;
1904FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmullw_u256, iemAImpl_vpmullw_u256_fallback;
1905FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulld_u256, iemAImpl_vpmulld_u256_fallback;
1906FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulhw_u256, iemAImpl_vpmulhw_u256_fallback;
1907FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpmulhuw_u256, iemAImpl_vpmulhuw_u256_fallback;
1908/** @} */
1909
1910/** @name Media (SSE/MMX/AVX) operations: lowhalf1 + lowhalf1 -> full1.
1911 * @{ */
1912FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpcklbw_u64, iemAImpl_punpcklwd_u64, iemAImpl_punpckldq_u64;
1913FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpcklbw_u128, iemAImpl_punpcklwd_u128, iemAImpl_punpckldq_u128, iemAImpl_punpcklqdq_u128;
1914FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpcklbw_u128, iemAImpl_vpunpcklbw_u128_fallback,
1915 iemAImpl_vpunpcklwd_u128, iemAImpl_vpunpcklwd_u128_fallback,
1916 iemAImpl_vpunpckldq_u128, iemAImpl_vpunpckldq_u128_fallback,
1917 iemAImpl_vpunpcklqdq_u128, iemAImpl_vpunpcklqdq_u128_fallback;
1918FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpcklbw_u256, iemAImpl_vpunpcklbw_u256_fallback,
1919 iemAImpl_vpunpcklwd_u256, iemAImpl_vpunpcklwd_u256_fallback,
1920 iemAImpl_vpunpckldq_u256, iemAImpl_vpunpckldq_u256_fallback,
1921 iemAImpl_vpunpcklqdq_u256, iemAImpl_vpunpcklqdq_u256_fallback;
1922/** @} */
1923
1924/** @name Media (SSE/MMX/AVX) operations: hihalf1 + hihalf2 -> full1.
1925 * @{ */
1926FNIEMAIMPLMEDIAOPTF2U64 iemAImpl_punpckhbw_u64, iemAImpl_punpckhwd_u64, iemAImpl_punpckhdq_u64;
1927FNIEMAIMPLMEDIAOPTF2U128 iemAImpl_punpckhbw_u128, iemAImpl_punpckhwd_u128, iemAImpl_punpckhdq_u128, iemAImpl_punpckhqdq_u128;
1928FNIEMAIMPLMEDIAOPTF3U128 iemAImpl_vpunpckhbw_u128, iemAImpl_vpunpckhbw_u128_fallback,
1929 iemAImpl_vpunpckhwd_u128, iemAImpl_vpunpckhwd_u128_fallback,
1930 iemAImpl_vpunpckhdq_u128, iemAImpl_vpunpckhdq_u128_fallback,
1931 iemAImpl_vpunpckhqdq_u128, iemAImpl_vpunpckhqdq_u128_fallback;
1932FNIEMAIMPLMEDIAOPTF3U256 iemAImpl_vpunpckhbw_u256, iemAImpl_vpunpckhbw_u256_fallback,
1933 iemAImpl_vpunpckhwd_u256, iemAImpl_vpunpckhwd_u256_fallback,
1934 iemAImpl_vpunpckhdq_u256, iemAImpl_vpunpckhdq_u256_fallback,
1935 iemAImpl_vpunpckhqdq_u256, iemAImpl_vpunpckhqdq_u256_fallback;
1936/** @} */
1937
1938/** @name Media (SSE/MMX/AVX) operation: Packed Shuffle Stuff (evil)
1939 * @{ */
1940typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil));
1941typedef FNIEMAIMPLMEDIAPSHUFU128 *PFNIEMAIMPLMEDIAPSHUFU128;
1942typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUFU256,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil));
1943typedef FNIEMAIMPLMEDIAPSHUFU256 *PFNIEMAIMPLMEDIAPSHUFU256;
1944IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw_u64,(uint64_t *puDst, uint64_t const *puSrc, uint8_t bEvil));
1945FNIEMAIMPLMEDIAPSHUFU128 iemAImpl_pshufhw_u128, iemAImpl_pshuflw_u128, iemAImpl_pshufd_u128;
1946#ifndef IEM_WITHOUT_ASSEMBLY
1947FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256, iemAImpl_vpshuflw_u256, iemAImpl_vpshufd_u256;
1948#endif
1949FNIEMAIMPLMEDIAPSHUFU256 iemAImpl_vpshufhw_u256_fallback, iemAImpl_vpshuflw_u256_fallback, iemAImpl_vpshufd_u256_fallback;
1950/** @} */
1951
1952/** @name Media (SSE/MMX/AVX) operation: Shift Immediate Stuff (evil)
1953 * @{ */
1954typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU64,(uint64_t *puDst, uint8_t bShift));
1955typedef FNIEMAIMPLMEDIAPSHIFTU64 *PFNIEMAIMPLMEDIAPSHIFTU64;
1956typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU128,(PRTUINT128U puDst, uint8_t bShift));
1957typedef FNIEMAIMPLMEDIAPSHIFTU128 *PFNIEMAIMPLMEDIAPSHIFTU128;
1958typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHIFTU256,(PRTUINT256U puDst, uint8_t bShift));
1959typedef FNIEMAIMPLMEDIAPSHIFTU256 *PFNIEMAIMPLMEDIAPSHIFTU256;
1960FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psllw_imm_u64, iemAImpl_pslld_imm_u64, iemAImpl_psllq_imm_u64;
1961FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psrlw_imm_u64, iemAImpl_psrld_imm_u64, iemAImpl_psrlq_imm_u64;
1962FNIEMAIMPLMEDIAPSHIFTU64 iemAImpl_psraw_imm_u64, iemAImpl_psrad_imm_u64;
1963FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psllw_imm_u128, iemAImpl_pslld_imm_u128, iemAImpl_psllq_imm_u128;
1964FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psrlw_imm_u128, iemAImpl_psrld_imm_u128, iemAImpl_psrlq_imm_u128;
1965FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_psraw_imm_u128, iemAImpl_psrad_imm_u128;
1966FNIEMAIMPLMEDIAPSHIFTU128 iemAImpl_pslldq_imm_u128, iemAImpl_psrldq_imm_u128;
1967/** @} */
1968
1969/** @name Media (SSE/MMX/AVX) operation: Move Byte Mask
1970 * @{ */
1971IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(uint64_t *pu64Dst, uint64_t const *puSrc));
1972IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(uint64_t *pu64Dst, PCRTUINT128U puSrc));
1973#ifndef IEM_WITHOUT_ASSEMBLY
1974IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
1975#endif
1976IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256_fallback,(uint64_t *pu64Dst, PCRTUINT256U puSrc));
1977/** @} */
1978
1979/** @name Media (SSE/MMX/AVX) operation: Sort this later
1980 * @{ */
1981IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1982IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PRTUINT128U puDst, PCRTUINT128U puSrc));
1983IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PRTUINT128U puDst, uint64_t uSrc));
1984
1985IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1986IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1987IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1988IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1989IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1990IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1991
1992/** @} */
1993
1994/** @name Media Odds and Ends
1995 * @{ */
1996typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U8,(uint32_t *puDst, uint8_t uSrc));
1997typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U16,(uint32_t *puDst, uint16_t uSrc));
1998typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U32,(uint32_t *puDst, uint32_t uSrc));
1999typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCR32U64,(uint32_t *puDst, uint64_t uSrc));
2000FNIEMAIMPLCR32U8 iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback;
2001FNIEMAIMPLCR32U16 iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback;
2002FNIEMAIMPLCR32U32 iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback;
2003FNIEMAIMPLCR32U64 iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback;
2004
2005typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLF2EFL128,(PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint32_t *pEFlags));
2006typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLF2EFL256,(PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint32_t *pEFlags));
2007FNIEMAIMPLF2EFL128 iemAImpl_ptest_u128;
2008FNIEMAIMPLF2EFL256 iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback;
2009/** @} */
2010
2011
2012/** @name Function tables.
2013 * @{
2014 */
2015
2016/**
2017 * Function table for a binary operator providing implementation based on
2018 * operand size.
2019 */
2020typedef struct IEMOPBINSIZES
2021{
2022 PFNIEMAIMPLBINU8 pfnNormalU8, pfnLockedU8;
2023 PFNIEMAIMPLBINU16 pfnNormalU16, pfnLockedU16;
2024 PFNIEMAIMPLBINU32 pfnNormalU32, pfnLockedU32;
2025 PFNIEMAIMPLBINU64 pfnNormalU64, pfnLockedU64;
2026} IEMOPBINSIZES;
2027/** Pointer to a binary operator function table. */
2028typedef IEMOPBINSIZES const *PCIEMOPBINSIZES;
2029
2030
2031/**
2032 * Function table for a unary operator providing implementation based on
2033 * operand size.
2034 */
2035typedef struct IEMOPUNARYSIZES
2036{
2037 PFNIEMAIMPLUNARYU8 pfnNormalU8, pfnLockedU8;
2038 PFNIEMAIMPLUNARYU16 pfnNormalU16, pfnLockedU16;
2039 PFNIEMAIMPLUNARYU32 pfnNormalU32, pfnLockedU32;
2040 PFNIEMAIMPLUNARYU64 pfnNormalU64, pfnLockedU64;
2041} IEMOPUNARYSIZES;
2042/** Pointer to a unary operator function table. */
2043typedef IEMOPUNARYSIZES const *PCIEMOPUNARYSIZES;
2044
2045
2046/**
2047 * Function table for a shift operator providing implementation based on
2048 * operand size.
2049 */
2050typedef struct IEMOPSHIFTSIZES
2051{
2052 PFNIEMAIMPLSHIFTU8 pfnNormalU8;
2053 PFNIEMAIMPLSHIFTU16 pfnNormalU16;
2054 PFNIEMAIMPLSHIFTU32 pfnNormalU32;
2055 PFNIEMAIMPLSHIFTU64 pfnNormalU64;
2056} IEMOPSHIFTSIZES;
2057/** Pointer to a shift operator function table. */
2058typedef IEMOPSHIFTSIZES const *PCIEMOPSHIFTSIZES;
2059
2060
2061/**
2062 * Function table for a multiplication or division operation.
2063 */
2064typedef struct IEMOPMULDIVSIZES
2065{
2066 PFNIEMAIMPLMULDIVU8 pfnU8;
2067 PFNIEMAIMPLMULDIVU16 pfnU16;
2068 PFNIEMAIMPLMULDIVU32 pfnU32;
2069 PFNIEMAIMPLMULDIVU64 pfnU64;
2070} IEMOPMULDIVSIZES;
2071/** Pointer to a multiplication or division operation function table. */
2072typedef IEMOPMULDIVSIZES const *PCIEMOPMULDIVSIZES;
2073
2074
2075/**
2076 * Function table for a double precision shift operator providing implementation
2077 * based on operand size.
2078 */
2079typedef struct IEMOPSHIFTDBLSIZES
2080{
2081 PFNIEMAIMPLSHIFTDBLU16 pfnNormalU16;
2082 PFNIEMAIMPLSHIFTDBLU32 pfnNormalU32;
2083 PFNIEMAIMPLSHIFTDBLU64 pfnNormalU64;
2084} IEMOPSHIFTDBLSIZES;
2085/** Pointer to a double precision shift function table. */
2086typedef IEMOPSHIFTDBLSIZES const *PCIEMOPSHIFTDBLSIZES;
2087
2088
2089/**
2090 * Function table for media instruction taking two full sized media source
2091 * registers and one full sized destination register (AVX).
2092 */
2093typedef struct IEMOPMEDIAF3
2094{
2095 PFNIEMAIMPLMEDIAF3U128 pfnU128;
2096 PFNIEMAIMPLMEDIAF3U256 pfnU256;
2097} IEMOPMEDIAF3;
2098/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2099typedef IEMOPMEDIAF3 const *PCIEMOPMEDIAF3;
2100
2101/** @def IEMOPMEDIAF3_INIT_VARS_EX
2102 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2103 * given functions as initializers. For use in AVX functions where a pair of
2104 * functions are only used once and the function table need not be public. */
2105#ifndef TST_IEM_CHECK_MC
2106# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2107# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2108 static IEMOPMEDIAF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2109 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2110# else
2111# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2112 static IEMOPMEDIAF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2113# endif
2114#else
2115# define IEMOPMEDIAF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2116#endif
2117/** @def IEMOPMEDIAF3_INIT_VARS
2118 * Generate AVX function tables for the @a a_InstrNm instruction.
2119 * @sa IEMOPMEDIAF3_INIT_VARS_EX */
2120#define IEMOPMEDIAF3_INIT_VARS(a_InstrNm) \
2121 IEMOPMEDIAF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2122 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2123
2124/**
2125 * Function table for media instruction taking two full sized media source
2126 * registers and one full sized destination register, but no additional state
2127 * (AVX).
2128 */
2129typedef struct IEMOPMEDIAOPTF3
2130{
2131 PFNIEMAIMPLMEDIAOPTF3U128 pfnU128;
2132 PFNIEMAIMPLMEDIAOPTF3U256 pfnU256;
2133} IEMOPMEDIAOPTF3;
2134/** Pointer to a media operation function table for 3 full sized ops (AVX). */
2135typedef IEMOPMEDIAOPTF3 const *PCIEMOPMEDIAOPTF3;
2136
2137/** @def IEMOPMEDIAOPTF3_INIT_VARS_EX
2138 * Declares a s_Host (x86 & amd64 only) and a s_Fallback variable with the
2139 * given functions as initializers. For use in AVX functions where a pair of
2140 * functions are only used once and the function table need not be public. */
2141#ifndef TST_IEM_CHECK_MC
2142# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
2143# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnHostU128, a_pfnHostU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2144 static IEMOPMEDIAOPTF3 const s_Host = { a_pfnHostU128, a_pfnHostU256 }; \
2145 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2146# else
2147# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) \
2148 static IEMOPMEDIAOPTF3 const s_Fallback = { a_pfnFallbackU128, a_pfnFallbackU256 }
2149# endif
2150#else
2151# define IEMOPMEDIAOPTF3_INIT_VARS_EX(a_pfnU128, a_pfnU256, a_pfnFallbackU128, a_pfnFallbackU256) (void)0
2152#endif
2153/** @def IEMOPMEDIAOPTF3_INIT_VARS
2154 * Generate AVX function tables for the @a a_InstrNm instruction.
2155 * @sa IEMOPMEDIAOPTF3_INIT_VARS_EX */
2156#define IEMOPMEDIAOPTF3_INIT_VARS(a_InstrNm) \
2157 IEMOPMEDIAOPTF3_INIT_VARS_EX(RT_CONCAT3(iemAImpl_,a_InstrNm,_u128), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256),\
2158 RT_CONCAT3(iemAImpl_,a_InstrNm,_u128_fallback), RT_CONCAT3(iemAImpl_,a_InstrNm,_u256_fallback))
2159
2160
2161/** @} */
2162
2163
2164/** @name C instruction implementations for anything slightly complicated.
2165 * @{ */
2166
2167/**
2168 * For typedef'ing or declaring a C instruction implementation function taking
2169 * no extra arguments.
2170 *
2171 * @param a_Name The name of the type.
2172 */
2173# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
2174 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2175/**
2176 * For defining a C instruction implementation function taking no extra
2177 * arguments.
2178 *
2179 * @param a_Name The name of the function
2180 */
2181# define IEM_CIMPL_DEF_0(a_Name) \
2182 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2183/**
2184 * Prototype version of IEM_CIMPL_DEF_0.
2185 */
2186# define IEM_CIMPL_PROTO_0(a_Name) \
2187 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
2188/**
2189 * For calling a C instruction implementation function taking no extra
2190 * arguments.
2191 *
2192 * This special call macro adds default arguments to the call and allow us to
2193 * change these later.
2194 *
2195 * @param a_fn The name of the function.
2196 */
2197# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
2198
2199/**
2200 * For typedef'ing or declaring a C instruction implementation function taking
2201 * one extra argument.
2202 *
2203 * @param a_Name The name of the type.
2204 * @param a_Type0 The argument type.
2205 * @param a_Arg0 The argument name.
2206 */
2207# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
2208 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2209/**
2210 * For defining a C instruction implementation function taking one extra
2211 * argument.
2212 *
2213 * @param a_Name The name of the function
2214 * @param a_Type0 The argument type.
2215 * @param a_Arg0 The argument name.
2216 */
2217# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
2218 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2219/**
2220 * Prototype version of IEM_CIMPL_DEF_1.
2221 */
2222# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
2223 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
2224/**
2225 * For calling a C instruction implementation function taking one extra
2226 * argument.
2227 *
2228 * This special call macro adds default arguments to the call and allow us to
2229 * change these later.
2230 *
2231 * @param a_fn The name of the function.
2232 * @param a0 The name of the 1st argument.
2233 */
2234# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
2235
2236/**
2237 * For typedef'ing or declaring a C instruction implementation function taking
2238 * two extra arguments.
2239 *
2240 * @param a_Name The name of the type.
2241 * @param a_Type0 The type of the 1st argument
2242 * @param a_Arg0 The name of the 1st argument.
2243 * @param a_Type1 The type of the 2nd argument.
2244 * @param a_Arg1 The name of the 2nd argument.
2245 */
2246# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2247 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2248/**
2249 * For defining a C instruction implementation function taking two extra
2250 * arguments.
2251 *
2252 * @param a_Name The name of the function.
2253 * @param a_Type0 The type of the 1st argument
2254 * @param a_Arg0 The name of the 1st argument.
2255 * @param a_Type1 The type of the 2nd argument.
2256 * @param a_Arg1 The name of the 2nd argument.
2257 */
2258# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2259 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2260/**
2261 * Prototype version of IEM_CIMPL_DEF_2.
2262 */
2263# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
2264 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
2265/**
2266 * For calling a C instruction implementation function taking two extra
2267 * arguments.
2268 *
2269 * This special call macro adds default arguments to the call and allow us to
2270 * change these later.
2271 *
2272 * @param a_fn The name of the function.
2273 * @param a0 The name of the 1st argument.
2274 * @param a1 The name of the 2nd argument.
2275 */
2276# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
2277
2278/**
2279 * For typedef'ing or declaring a C instruction implementation function taking
2280 * three extra arguments.
2281 *
2282 * @param a_Name The name of the type.
2283 * @param a_Type0 The type of the 1st argument
2284 * @param a_Arg0 The name of the 1st argument.
2285 * @param a_Type1 The type of the 2nd argument.
2286 * @param a_Arg1 The name of the 2nd argument.
2287 * @param a_Type2 The type of the 3rd argument.
2288 * @param a_Arg2 The name of the 3rd argument.
2289 */
2290# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2291 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2292/**
2293 * For defining a C instruction implementation function taking three extra
2294 * arguments.
2295 *
2296 * @param a_Name The name of the function.
2297 * @param a_Type0 The type of the 1st argument
2298 * @param a_Arg0 The name of the 1st argument.
2299 * @param a_Type1 The type of the 2nd argument.
2300 * @param a_Arg1 The name of the 2nd argument.
2301 * @param a_Type2 The type of the 3rd argument.
2302 * @param a_Arg2 The name of the 3rd argument.
2303 */
2304# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2305 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2306/**
2307 * Prototype version of IEM_CIMPL_DEF_3.
2308 */
2309# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
2310 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
2311/**
2312 * For calling a C instruction implementation function taking three extra
2313 * arguments.
2314 *
2315 * This special call macro adds default arguments to the call and allow us to
2316 * change these later.
2317 *
2318 * @param a_fn The name of the function.
2319 * @param a0 The name of the 1st argument.
2320 * @param a1 The name of the 2nd argument.
2321 * @param a2 The name of the 3rd argument.
2322 */
2323# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
2324
2325
2326/**
2327 * For typedef'ing or declaring a C instruction implementation function taking
2328 * four extra arguments.
2329 *
2330 * @param a_Name The name of the type.
2331 * @param a_Type0 The type of the 1st argument
2332 * @param a_Arg0 The name of the 1st argument.
2333 * @param a_Type1 The type of the 2nd argument.
2334 * @param a_Arg1 The name of the 2nd argument.
2335 * @param a_Type2 The type of the 3rd argument.
2336 * @param a_Arg2 The name of the 3rd argument.
2337 * @param a_Type3 The type of the 4th argument.
2338 * @param a_Arg3 The name of the 4th argument.
2339 */
2340# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2341 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
2342/**
2343 * For defining a C instruction implementation function taking four extra
2344 * arguments.
2345 *
2346 * @param a_Name The name of the function.
2347 * @param a_Type0 The type of the 1st argument
2348 * @param a_Arg0 The name of the 1st argument.
2349 * @param a_Type1 The type of the 2nd argument.
2350 * @param a_Arg1 The name of the 2nd argument.
2351 * @param a_Type2 The type of the 3rd argument.
2352 * @param a_Arg2 The name of the 3rd argument.
2353 * @param a_Type3 The type of the 4th argument.
2354 * @param a_Arg3 The name of the 4th argument.
2355 */
2356# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2357 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2358 a_Type2 a_Arg2, a_Type3 a_Arg3))
2359/**
2360 * Prototype version of IEM_CIMPL_DEF_4.
2361 */
2362# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
2363 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2364 a_Type2 a_Arg2, a_Type3 a_Arg3))
2365/**
2366 * For calling a C instruction implementation function taking four extra
2367 * arguments.
2368 *
2369 * This special call macro adds default arguments to the call and allow us to
2370 * change these later.
2371 *
2372 * @param a_fn The name of the function.
2373 * @param a0 The name of the 1st argument.
2374 * @param a1 The name of the 2nd argument.
2375 * @param a2 The name of the 3rd argument.
2376 * @param a3 The name of the 4th argument.
2377 */
2378# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
2379
2380
2381/**
2382 * For typedef'ing or declaring a C instruction implementation function taking
2383 * five extra arguments.
2384 *
2385 * @param a_Name The name of the type.
2386 * @param a_Type0 The type of the 1st argument
2387 * @param a_Arg0 The name of the 1st argument.
2388 * @param a_Type1 The type of the 2nd argument.
2389 * @param a_Arg1 The name of the 2nd argument.
2390 * @param a_Type2 The type of the 3rd argument.
2391 * @param a_Arg2 The name of the 3rd argument.
2392 * @param a_Type3 The type of the 4th argument.
2393 * @param a_Arg3 The name of the 4th argument.
2394 * @param a_Type4 The type of the 5th argument.
2395 * @param a_Arg4 The name of the 5th argument.
2396 */
2397# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2398 IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
2399 a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
2400 a_Type3 a_Arg3, a_Type4 a_Arg4))
2401/**
2402 * For defining a C instruction implementation function taking five extra
2403 * arguments.
2404 *
2405 * @param a_Name The name of the function.
2406 * @param a_Type0 The type of the 1st argument
2407 * @param a_Arg0 The name of the 1st argument.
2408 * @param a_Type1 The type of the 2nd argument.
2409 * @param a_Arg1 The name of the 2nd argument.
2410 * @param a_Type2 The type of the 3rd argument.
2411 * @param a_Arg2 The name of the 3rd argument.
2412 * @param a_Type3 The type of the 4th argument.
2413 * @param a_Arg3 The name of the 4th argument.
2414 * @param a_Type4 The type of the 5th argument.
2415 * @param a_Arg4 The name of the 5th argument.
2416 */
2417# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2418 IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2419 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2420/**
2421 * Prototype version of IEM_CIMPL_DEF_5.
2422 */
2423# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2424 IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2425 a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2426/**
2427 * For calling a C instruction implementation function taking five extra
2428 * arguments.
2429 *
2430 * This special call macro adds default arguments to the call and allow us to
2431 * change these later.
2432 *
2433 * @param a_fn The name of the function.
2434 * @param a0 The name of the 1st argument.
2435 * @param a1 The name of the 2nd argument.
2436 * @param a2 The name of the 3rd argument.
2437 * @param a3 The name of the 4th argument.
2438 * @param a4 The name of the 5th argument.
2439 */
2440# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2441
2442/** @} */
2443
2444
2445/** @name Opcode Decoder Function Types.
2446 * @{ */
2447
2448/** @typedef PFNIEMOP
2449 * Pointer to an opcode decoder function.
2450 */
2451
2452/** @def FNIEMOP_DEF
2453 * Define an opcode decoder function.
2454 *
2455 * We're using macors for this so that adding and removing parameters as well as
2456 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
2457 *
2458 * @param a_Name The function name.
2459 */
2460
2461/** @typedef PFNIEMOPRM
2462 * Pointer to an opcode decoder function with RM byte.
2463 */
2464
2465/** @def FNIEMOPRM_DEF
2466 * Define an opcode decoder function with RM byte.
2467 *
2468 * We're using macors for this so that adding and removing parameters as well as
2469 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
2470 *
2471 * @param a_Name The function name.
2472 */
2473
2474#if defined(__GNUC__) && defined(RT_ARCH_X86)
2475typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
2476typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2477# define FNIEMOP_DEF(a_Name) \
2478 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
2479# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2480 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2481# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2482 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2483
2484#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2485typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
2486typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2487# define FNIEMOP_DEF(a_Name) \
2488 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2489# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2490 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2491# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2492 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2493
2494#elif defined(__GNUC__)
2495typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2496typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2497# define FNIEMOP_DEF(a_Name) \
2498 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
2499# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2500 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2501# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2502 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2503
2504#else
2505typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2506typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2507# define FNIEMOP_DEF(a_Name) \
2508 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2509# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2510 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2511# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2512 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2513
2514#endif
2515#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
2516
2517/**
2518 * Call an opcode decoder function.
2519 *
2520 * We're using macors for this so that adding and removing parameters can be
2521 * done as we please. See FNIEMOP_DEF.
2522 */
2523#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
2524
2525/**
2526 * Call a common opcode decoder function taking one extra argument.
2527 *
2528 * We're using macors for this so that adding and removing parameters can be
2529 * done as we please. See FNIEMOP_DEF_1.
2530 */
2531#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
2532
2533/**
2534 * Call a common opcode decoder function taking one extra argument.
2535 *
2536 * We're using macors for this so that adding and removing parameters can be
2537 * done as we please. See FNIEMOP_DEF_1.
2538 */
2539#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
2540/** @} */
2541
2542
2543/** @name Misc Helpers
2544 * @{ */
2545
2546/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
2547 * due to GCC lacking knowledge about the value range of a switch. */
2548#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
2549
2550/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
2551#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
2552
2553/**
2554 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2555 * occation.
2556 */
2557#ifdef LOG_ENABLED
2558# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2559 do { \
2560 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
2561 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2562 } while (0)
2563#else
2564# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2565 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2566#endif
2567
2568/**
2569 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2570 * occation using the supplied logger statement.
2571 *
2572 * @param a_LoggerArgs What to log on failure.
2573 */
2574#ifdef LOG_ENABLED
2575# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2576 do { \
2577 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
2578 /*LogFunc(a_LoggerArgs);*/ \
2579 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2580 } while (0)
2581#else
2582# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2583 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2584#endif
2585
2586/**
2587 * Check if we're currently executing in real or virtual 8086 mode.
2588 *
2589 * @returns @c true if it is, @c false if not.
2590 * @param a_pVCpu The IEM state of the current CPU.
2591 */
2592#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2593
2594/**
2595 * Check if we're currently executing in virtual 8086 mode.
2596 *
2597 * @returns @c true if it is, @c false if not.
2598 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2599 */
2600#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2601
2602/**
2603 * Check if we're currently executing in long mode.
2604 *
2605 * @returns @c true if it is, @c false if not.
2606 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2607 */
2608#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
2609
2610/**
2611 * Check if we're currently executing in a 64-bit code segment.
2612 *
2613 * @returns @c true if it is, @c false if not.
2614 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2615 */
2616#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
2617
2618/**
2619 * Check if we're currently executing in real mode.
2620 *
2621 * @returns @c true if it is, @c false if not.
2622 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2623 */
2624#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
2625
2626/**
2627 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
2628 * @returns PCCPUMFEATURES
2629 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2630 */
2631#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
2632
2633/**
2634 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
2635 * @returns PCCPUMFEATURES
2636 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2637 */
2638#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&g_CpumHostFeatures.s)
2639
2640/**
2641 * Evaluates to true if we're presenting an Intel CPU to the guest.
2642 */
2643#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
2644
2645/**
2646 * Evaluates to true if we're presenting an AMD CPU to the guest.
2647 */
2648#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
2649
2650/**
2651 * Check if the address is canonical.
2652 */
2653#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
2654
2655/** Checks if the ModR/M byte is in register mode or not. */
2656#define IEM_IS_MODRM_REG_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT) )
2657/** Checks if the ModR/M byte is in memory mode or not. */
2658#define IEM_IS_MODRM_MEM_MODE(a_bRm) ( ((a_bRm) & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT) )
2659
2660/**
2661 * Gets the register (reg) part of a ModR/M encoding, with REX.R added in.
2662 *
2663 * For use during decoding.
2664 */
2665#define IEM_GET_MODRM_REG(a_pVCpu, a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | (a_pVCpu)->iem.s.uRexReg )
2666/**
2667 * Gets the r/m part of a ModR/M encoding as a register index, with REX.B added in.
2668 *
2669 * For use during decoding.
2670 */
2671#define IEM_GET_MODRM_RM(a_pVCpu, a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) | (a_pVCpu)->iem.s.uRexB )
2672
2673/**
2674 * Gets the register (reg) part of a ModR/M encoding, without REX.R.
2675 *
2676 * For use during decoding.
2677 */
2678#define IEM_GET_MODRM_REG_8(a_bRm) ( (((a_bRm) >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) )
2679/**
2680 * Gets the r/m part of a ModR/M encoding as a register index, without REX.B.
2681 *
2682 * For use during decoding.
2683 */
2684#define IEM_GET_MODRM_RM_8(a_bRm) ( ((a_bRm) & X86_MODRM_RM_MASK) )
2685
2686/**
2687 * Gets the effective VEX.VVVV value.
2688 *
2689 * The 4th bit is ignored if not 64-bit code.
2690 * @returns effective V-register value.
2691 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2692 */
2693#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
2694 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
2695
2696
2697#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2698
2699/**
2700 * Check if the guest has entered VMX root operation.
2701 */
2702# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
2703
2704/**
2705 * Check if the guest has entered VMX non-root operation.
2706 */
2707# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
2708
2709/**
2710 * Check if the nested-guest has the given Pin-based VM-execution control set.
2711 */
2712# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
2713 (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
2714
2715/**
2716 * Check if the nested-guest has the given Processor-based VM-execution control set.
2717 */
2718# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
2719 (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
2720
2721/**
2722 * Check if the nested-guest has the given Secondary Processor-based VM-execution
2723 * control set.
2724 */
2725# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
2726 (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
2727
2728/** Gets the guest-physical address of the shadows VMCS for the given VCPU. */
2729# define IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs)
2730
2731/** Whether a shadow VMCS is present for the given VCPU. */
2732# define IEM_VMX_HAS_SHADOW_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2733
2734/** Gets the VMXON region pointer. */
2735# define IEM_VMX_GET_VMXON_PTR(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
2736
2737/** Gets the guest-physical address of the current VMCS for the given VCPU. */
2738# define IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs)
2739
2740/** Whether a current VMCS is present for the given VCPU. */
2741# define IEM_VMX_HAS_CURRENT_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2742
2743/** Assigns the guest-physical address of the current VMCS for the given VCPU. */
2744# define IEM_VMX_SET_CURRENT_VMCS(a_pVCpu, a_GCPhysVmcs) \
2745 do \
2746 { \
2747 Assert((a_GCPhysVmcs) != NIL_RTGCPHYS); \
2748 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = (a_GCPhysVmcs); \
2749 } while (0)
2750
2751/** Clears any current VMCS for the given VCPU. */
2752# define IEM_VMX_CLEAR_CURRENT_VMCS(a_pVCpu) \
2753 do \
2754 { \
2755 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = NIL_RTGCPHYS; \
2756 } while (0)
2757
2758/**
2759 * Invokes the VMX VM-exit handler for an instruction intercept.
2760 */
2761# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
2762 do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
2763
2764/**
2765 * Invokes the VMX VM-exit handler for an instruction intercept where the
2766 * instruction provides additional VM-exit information.
2767 */
2768# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
2769 do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
2770
2771/**
2772 * Invokes the VMX VM-exit handler for a task switch.
2773 */
2774# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
2775 do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
2776
2777/**
2778 * Invokes the VMX VM-exit handler for MWAIT.
2779 */
2780# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
2781 do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
2782
2783/**
2784 * Invokes the VMX VM-exit handler for EPT faults.
2785 */
2786# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) \
2787 do { return iemVmxVmexitEpt(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr); } while (0)
2788
2789/**
2790 * Invokes the VMX VM-exit handler.
2791 */
2792# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
2793 do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
2794
2795#else
2796# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
2797# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
2798# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
2799# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
2800# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
2801# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2802# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2803# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2804# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2805# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2806# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
2807
2808#endif
2809
2810#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2811/**
2812 * Check if an SVM control/instruction intercept is set.
2813 */
2814# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
2815 (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
2816
2817/**
2818 * Check if an SVM read CRx intercept is set.
2819 */
2820# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2821 (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2822
2823/**
2824 * Check if an SVM write CRx intercept is set.
2825 */
2826# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2827 (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2828
2829/**
2830 * Check if an SVM read DRx intercept is set.
2831 */
2832# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2833 (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2834
2835/**
2836 * Check if an SVM write DRx intercept is set.
2837 */
2838# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2839 (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2840
2841/**
2842 * Check if an SVM exception intercept is set.
2843 */
2844# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
2845 (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
2846
2847/**
2848 * Invokes the SVM \#VMEXIT handler for the nested-guest.
2849 */
2850# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2851 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
2852
2853/**
2854 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
2855 * corresponding decode assist information.
2856 */
2857# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
2858 do \
2859 { \
2860 uint64_t uExitInfo1; \
2861 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
2862 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
2863 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
2864 else \
2865 uExitInfo1 = 0; \
2866 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
2867 } while (0)
2868
2869/** Check and handles SVM nested-guest instruction intercept and updates
2870 * NRIP if needed.
2871 */
2872# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2873 do \
2874 { \
2875 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
2876 { \
2877 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2878 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
2879 } \
2880 } while (0)
2881
2882/** Checks and handles SVM nested-guest CR0 read intercept. */
2883# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
2884 do \
2885 { \
2886 if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
2887 { /* probably likely */ } \
2888 else \
2889 { \
2890 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2891 IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
2892 } \
2893 } while (0)
2894
2895/**
2896 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
2897 */
2898# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
2899 do { \
2900 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
2901 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
2902 } while (0)
2903
2904#else
2905# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
2906# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2907# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2908# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2909# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2910# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
2911# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
2912# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
2913# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2914# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2915# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
2916
2917#endif
2918
2919/** @} */
2920
2921
2922
2923/**
2924 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
2925 */
2926typedef union IEMSELDESC
2927{
2928 /** The legacy view. */
2929 X86DESC Legacy;
2930 /** The long mode view. */
2931 X86DESC64 Long;
2932} IEMSELDESC;
2933/** Pointer to a selector descriptor table entry. */
2934typedef IEMSELDESC *PIEMSELDESC;
2935
2936/** @name Raising Exceptions.
2937 * @{ */
2938VBOXSTRICTRC iemTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, uint32_t uNextEip, uint32_t fFlags,
2939 uint16_t uErr, uint64_t uCr2, RTSEL SelTSS, PIEMSELDESC pNewDescTSS) RT_NOEXCEPT;
2940
2941VBOXSTRICTRC iemRaiseXcptOrInt(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector, uint32_t fFlags,
2942 uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2943#ifdef IEM_WITH_SETJMP
2944DECL_NO_RETURN(void) iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector,
2945 uint32_t fFlags, uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2946#endif
2947VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT;
2948VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT;
2949VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT;
2950VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT;
2951VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT;
2952VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2953VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT;
2954VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2955VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2956/*VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;*/
2957VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2958VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2959VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2960VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2961VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2962VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2963#ifdef IEM_WITH_SETJMP
2964DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2965#endif
2966VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2967VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT;
2968VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2969#ifdef IEM_WITH_SETJMP
2970DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2971#endif
2972VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2973#ifdef IEM_WITH_SETJMP
2974DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2975#endif
2976VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2977#ifdef IEM_WITH_SETJMP
2978DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2979#endif
2980VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2981#ifdef IEM_WITH_SETJMP
2982DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2983#endif
2984VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu);
2985VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
2986#ifdef IEM_WITH_SETJMP
2987DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2988#endif
2989
2990IEM_CIMPL_DEF_0(iemCImplRaiseDivideError);
2991IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix);
2992IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode);
2993
2994/**
2995 * Macro for calling iemCImplRaiseDivideError().
2996 *
2997 * This enables us to add/remove arguments and force different levels of
2998 * inlining as we wish.
2999 *
3000 * @return Strict VBox status code.
3001 */
3002#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
3003
3004/**
3005 * Macro for calling iemCImplRaiseInvalidLockPrefix().
3006 *
3007 * This enables us to add/remove arguments and force different levels of
3008 * inlining as we wish.
3009 *
3010 * @return Strict VBox status code.
3011 */
3012#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
3013
3014/**
3015 * Macro for calling iemCImplRaiseInvalidOpcode().
3016 *
3017 * This enables us to add/remove arguments and force different levels of
3018 * inlining as we wish.
3019 *
3020 * @return Strict VBox status code.
3021 */
3022#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
3023/** @} */
3024
3025/** @name Register Access.
3026 * @{ */
3027VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT;
3028VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT;
3029VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT;
3030VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT;
3031/** @} */
3032
3033/** @name FPU access and helpers.
3034 * @{ */
3035void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT;
3036void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3037void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT;
3038void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
3039void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
3040void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
3041 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3042void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
3043 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3044void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3045void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3046void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3047void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3048void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
3049void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3050void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
3051void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3052void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
3053void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3054void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT;
3055void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3056void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT;
3057void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
3058void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
3059/** @} */
3060
3061/** @name Memory access.
3062 * @{ */
3063
3064/** Report a \#GP instead of \#AC and do not restrict to ring-3 */
3065#define IEM_MEMMAP_F_ALIGN_GP RT_BIT_32(16)
3066/** SSE access that should report a \#GP instead of \#AC, unless MXCSR.MM=1
3067 * when it works like normal \#AC. Always used with IEM_MEMMAP_F_ALIGN_GP. */
3068#define IEM_MEMMAP_F_ALIGN_SSE RT_BIT_32(17)
3069/** If \#AC is applicable, raise it. Always used with IEM_MEMMAP_F_ALIGN_GP.
3070 * Users include FXSAVE & FXRSTOR. */
3071#define IEM_MEMMAP_F_ALIGN_GP_OR_AC RT_BIT_32(18)
3072
3073VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
3074 uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT;
3075VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
3076#ifndef IN_RING3
3077VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
3078#endif
3079void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
3080VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT;
3081VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
3082VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
3083
3084#ifdef IEM_WITH_CODE_TLB
3085void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT;
3086#else
3087VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT;
3088#endif
3089#ifdef IEM_WITH_SETJMP
3090uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3091uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3092uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3093uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
3094#else
3095VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT;
3096VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3097VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3098VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3099VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
3100VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3101VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3102VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
3103VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3104VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3105VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
3106#endif
3107
3108VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3109VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3110VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3111VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3112VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3113VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3114VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3115VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3116VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3117VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3118VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3119VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3120VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
3121 RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT;
3122#ifdef IEM_WITH_SETJMP
3123uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3124uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3125uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3126uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3127uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3128void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3129void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3130void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3131void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3132void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3133void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3134#endif
3135
3136VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3137VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3138VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3139VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3140VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT;
3141
3142VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3143VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3144VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3145VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3146VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3147VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3148VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3149VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3150VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
3151#ifdef IEM_WITH_SETJMP
3152void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
3153void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
3154void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
3155void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
3156void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3157void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
3158void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3159void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
3160#endif
3161
3162VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3163 void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3164VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT;
3165VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT;
3166VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3167VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT;
3168VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3169VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3170VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3171VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
3172VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
3173 void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3174VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT;
3175VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT;
3176VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT;
3177VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT;
3178VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT;
3179VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3180VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3181VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
3182/** @} */
3183
3184/** @name IEMAllCImpl.cpp
3185 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/'
3186 * @{ */
3187IEM_CIMPL_PROTO_0(iemCImpl_popa_16);
3188IEM_CIMPL_PROTO_0(iemCImpl_popa_32);
3189IEM_CIMPL_PROTO_0(iemCImpl_pusha_16);
3190IEM_CIMPL_PROTO_0(iemCImpl_pusha_32);
3191IEM_CIMPL_PROTO_1(iemCImpl_pushf, IEMMODE, enmEffOpSize);
3192IEM_CIMPL_PROTO_1(iemCImpl_popf, IEMMODE, enmEffOpSize);
3193IEM_CIMPL_PROTO_1(iemCImpl_call_16, uint16_t, uNewPC);
3194IEM_CIMPL_PROTO_1(iemCImpl_call_rel_16, int16_t, offDisp);
3195IEM_CIMPL_PROTO_1(iemCImpl_call_32, uint32_t, uNewPC);
3196IEM_CIMPL_PROTO_1(iemCImpl_call_rel_32, int32_t, offDisp);
3197IEM_CIMPL_PROTO_1(iemCImpl_call_64, uint64_t, uNewPC);
3198IEM_CIMPL_PROTO_1(iemCImpl_call_rel_64, int64_t, offDisp);
3199IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3200IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3201IEM_CIMPL_PROTO_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3202IEM_CIMPL_PROTO_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
3203IEM_CIMPL_PROTO_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3204IEM_CIMPL_PROTO_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
3205IEM_CIMPL_PROTO_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3206IEM_CIMPL_PROTO_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop);
3207IEM_CIMPL_PROTO_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters);
3208IEM_CIMPL_PROTO_1(iemCImpl_leave, IEMMODE, enmEffOpSize);
3209IEM_CIMPL_PROTO_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt);
3210IEM_CIMPL_PROTO_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize);
3211IEM_CIMPL_PROTO_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp);
3212IEM_CIMPL_PROTO_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize);
3213IEM_CIMPL_PROTO_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize);
3214IEM_CIMPL_PROTO_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize);
3215IEM_CIMPL_PROTO_1(iemCImpl_iret, IEMMODE, enmEffOpSize);
3216IEM_CIMPL_PROTO_0(iemCImpl_loadall286);
3217IEM_CIMPL_PROTO_0(iemCImpl_syscall);
3218IEM_CIMPL_PROTO_0(iemCImpl_sysret);
3219IEM_CIMPL_PROTO_0(iemCImpl_sysenter);
3220IEM_CIMPL_PROTO_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize);
3221IEM_CIMPL_PROTO_2(iemCImpl_LoadSReg, uint8_t, iSegReg, uint16_t, uSel);
3222IEM_CIMPL_PROTO_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel);
3223IEM_CIMPL_PROTO_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize);
3224IEM_CIMPL_PROTO_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize);
3225IEM_CIMPL_PROTO_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite);
3226IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar);
3227IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar);
3228IEM_CIMPL_PROTO_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3229IEM_CIMPL_PROTO_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3230IEM_CIMPL_PROTO_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
3231IEM_CIMPL_PROTO_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3232IEM_CIMPL_PROTO_1(iemCImpl_lldt, uint16_t, uNewLdt);
3233IEM_CIMPL_PROTO_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3234IEM_CIMPL_PROTO_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3235IEM_CIMPL_PROTO_1(iemCImpl_ltr, uint16_t, uNewTr);
3236IEM_CIMPL_PROTO_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3237IEM_CIMPL_PROTO_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3238IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg);
3239IEM_CIMPL_PROTO_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
3240IEM_CIMPL_PROTO_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3241IEM_CIMPL_PROTO_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg);
3242IEM_CIMPL_PROTO_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg);
3243IEM_CIMPL_PROTO_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst);
3244IEM_CIMPL_PROTO_0(iemCImpl_clts);
3245IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg);
3246IEM_CIMPL_PROTO_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg);
3247IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg);
3248IEM_CIMPL_PROTO_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg);
3249IEM_CIMPL_PROTO_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage);
3250IEM_CIMPL_PROTO_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType);
3251IEM_CIMPL_PROTO_0(iemCImpl_invd);
3252IEM_CIMPL_PROTO_0(iemCImpl_wbinvd);
3253IEM_CIMPL_PROTO_0(iemCImpl_rsm);
3254IEM_CIMPL_PROTO_0(iemCImpl_rdtsc);
3255IEM_CIMPL_PROTO_0(iemCImpl_rdtscp);
3256IEM_CIMPL_PROTO_0(iemCImpl_rdpmc);
3257IEM_CIMPL_PROTO_0(iemCImpl_rdmsr);
3258IEM_CIMPL_PROTO_0(iemCImpl_wrmsr);
3259IEM_CIMPL_PROTO_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3260IEM_CIMPL_PROTO_1(iemCImpl_in_eAX_DX, uint8_t, cbReg);
3261IEM_CIMPL_PROTO_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
3262IEM_CIMPL_PROTO_1(iemCImpl_out_DX_eAX, uint8_t, cbReg);
3263IEM_CIMPL_PROTO_0(iemCImpl_cli);
3264IEM_CIMPL_PROTO_0(iemCImpl_sti);
3265IEM_CIMPL_PROTO_0(iemCImpl_hlt);
3266IEM_CIMPL_PROTO_1(iemCImpl_monitor, uint8_t, iEffSeg);
3267IEM_CIMPL_PROTO_0(iemCImpl_mwait);
3268IEM_CIMPL_PROTO_0(iemCImpl_swapgs);
3269IEM_CIMPL_PROTO_0(iemCImpl_cpuid);
3270IEM_CIMPL_PROTO_1(iemCImpl_aad, uint8_t, bImm);
3271IEM_CIMPL_PROTO_1(iemCImpl_aam, uint8_t, bImm);
3272IEM_CIMPL_PROTO_0(iemCImpl_daa);
3273IEM_CIMPL_PROTO_0(iemCImpl_das);
3274IEM_CIMPL_PROTO_0(iemCImpl_aaa);
3275IEM_CIMPL_PROTO_0(iemCImpl_aas);
3276IEM_CIMPL_PROTO_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound);
3277IEM_CIMPL_PROTO_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound);
3278IEM_CIMPL_PROTO_0(iemCImpl_xgetbv);
3279IEM_CIMPL_PROTO_0(iemCImpl_xsetbv);
3280IEM_CIMPL_PROTO_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
3281 PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags);
3282IEM_CIMPL_PROTO_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3283IEM_CIMPL_PROTO_1(iemCImpl_finit, bool, fCheckXcpts);
3284IEM_CIMPL_PROTO_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3285IEM_CIMPL_PROTO_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3286IEM_CIMPL_PROTO_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3287IEM_CIMPL_PROTO_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
3288IEM_CIMPL_PROTO_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3289IEM_CIMPL_PROTO_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3290IEM_CIMPL_PROTO_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
3291IEM_CIMPL_PROTO_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3292IEM_CIMPL_PROTO_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
3293IEM_CIMPL_PROTO_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3294IEM_CIMPL_PROTO_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
3295IEM_CIMPL_PROTO_1(iemCImpl_fldcw, uint16_t, u16Fcw);
3296IEM_CIMPL_PROTO_1(iemCImpl_fxch_underflow, uint8_t, iStReg);
3297IEM_CIMPL_PROTO_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop);
3298/** @} */
3299
3300/** @name IEMAllCImplStrInstr.cpp.h
3301 * @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/' -e 's/RT_CONCAT4(//' \
3302 * -e 's/,ADDR_SIZE)/64/g' -e 's/,OP_SIZE,/64/g' -e 's/,OP_rAX,/rax/g' IEMAllCImplStrInstr.cpp.h
3303 * @{ */
3304IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr16, uint8_t, iEffSeg);
3305IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr16, uint8_t, iEffSeg);
3306IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m16);
3307IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m16);
3308IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr16, uint8_t, iEffSeg);
3309IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m16);
3310IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m16, int8_t, iEffSeg);
3311IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr16, bool, fIoChecked);
3312IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr16, bool, fIoChecked);
3313IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3314IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3315
3316IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr16, uint8_t, iEffSeg);
3317IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr16, uint8_t, iEffSeg);
3318IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m16);
3319IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m16);
3320IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr16, uint8_t, iEffSeg);
3321IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m16);
3322IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m16, int8_t, iEffSeg);
3323IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr16, bool, fIoChecked);
3324IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr16, bool, fIoChecked);
3325IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3326IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3327
3328IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr16, uint8_t, iEffSeg);
3329IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr16, uint8_t, iEffSeg);
3330IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m16);
3331IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m16);
3332IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr16, uint8_t, iEffSeg);
3333IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m16);
3334IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m16, int8_t, iEffSeg);
3335IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr16, bool, fIoChecked);
3336IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr16, bool, fIoChecked);
3337IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3338IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
3339
3340
3341IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr32, uint8_t, iEffSeg);
3342IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr32, uint8_t, iEffSeg);
3343IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m32);
3344IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m32);
3345IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr32, uint8_t, iEffSeg);
3346IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m32);
3347IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m32, int8_t, iEffSeg);
3348IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr32, bool, fIoChecked);
3349IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr32, bool, fIoChecked);
3350IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3351IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3352
3353IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr32, uint8_t, iEffSeg);
3354IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr32, uint8_t, iEffSeg);
3355IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m32);
3356IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m32);
3357IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr32, uint8_t, iEffSeg);
3358IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m32);
3359IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m32, int8_t, iEffSeg);
3360IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr32, bool, fIoChecked);
3361IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr32, bool, fIoChecked);
3362IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3363IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3364
3365IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr32, uint8_t, iEffSeg);
3366IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr32, uint8_t, iEffSeg);
3367IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m32);
3368IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m32);
3369IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr32, uint8_t, iEffSeg);
3370IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m32);
3371IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m32, int8_t, iEffSeg);
3372IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr32, bool, fIoChecked);
3373IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr32, bool, fIoChecked);
3374IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3375IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3376
3377IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr32, uint8_t, iEffSeg);
3378IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr32, uint8_t, iEffSeg);
3379IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m32);
3380IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m32);
3381IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr32, uint8_t, iEffSeg);
3382IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m32);
3383IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m32, int8_t, iEffSeg);
3384IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr32, bool, fIoChecked);
3385IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr32, bool, fIoChecked);
3386IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3387IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
3388
3389
3390IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr64, uint8_t, iEffSeg);
3391IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr64, uint8_t, iEffSeg);
3392IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m64);
3393IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m64);
3394IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr64, uint8_t, iEffSeg);
3395IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m64);
3396IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m64, int8_t, iEffSeg);
3397IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr64, bool, fIoChecked);
3398IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr64, bool, fIoChecked);
3399IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3400IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3401
3402IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr64, uint8_t, iEffSeg);
3403IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr64, uint8_t, iEffSeg);
3404IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m64);
3405IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m64);
3406IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr64, uint8_t, iEffSeg);
3407IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m64);
3408IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m64, int8_t, iEffSeg);
3409IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr64, bool, fIoChecked);
3410IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr64, bool, fIoChecked);
3411IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3412IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3413
3414IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr64, uint8_t, iEffSeg);
3415IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr64, uint8_t, iEffSeg);
3416IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m64);
3417IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m64);
3418IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr64, uint8_t, iEffSeg);
3419IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m64);
3420IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m64, int8_t, iEffSeg);
3421IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr64, bool, fIoChecked);
3422IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr64, bool, fIoChecked);
3423IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3424IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3425
3426IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr64, uint8_t, iEffSeg);
3427IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr64, uint8_t, iEffSeg);
3428IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m64);
3429IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m64);
3430IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr64, uint8_t, iEffSeg);
3431IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m64);
3432IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m64, int8_t, iEffSeg);
3433IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr64, bool, fIoChecked);
3434IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr64, bool, fIoChecked);
3435IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3436IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
3437/** @} */
3438
3439#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3440VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT;
3441VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT;
3442VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT;
3443VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT;
3444VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT;
3445VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT;
3446VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT;
3447VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT;
3448VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT;
3449VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port,
3450 bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT;
3451VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess,
3452 bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT;
3453VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3454VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3455VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3456VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3457VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3458VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
3459VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT;
3460VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw,
3461 RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT;
3462VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT;
3463VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT;
3464VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT;
3465uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT;
3466void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT;
3467VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
3468 uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT;
3469bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT;
3470IEM_CIMPL_PROTO_0(iemCImpl_vmxoff);
3471IEM_CIMPL_PROTO_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon);
3472IEM_CIMPL_PROTO_0(iemCImpl_vmlaunch);
3473IEM_CIMPL_PROTO_0(iemCImpl_vmresume);
3474IEM_CIMPL_PROTO_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3475IEM_CIMPL_PROTO_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3476IEM_CIMPL_PROTO_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3477IEM_CIMPL_PROTO_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField);
3478IEM_CIMPL_PROTO_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField);
3479IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField);
3480IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField);
3481IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField);
3482IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField);
3483IEM_CIMPL_PROTO_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType);
3484IEM_CIMPL_PROTO_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType);
3485IEM_CIMPL_PROTO_0(iemCImpl_vmx_pause);
3486#endif
3487
3488#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3489VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT;
3490VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3491VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
3492 uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT;
3493VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT;
3494IEM_CIMPL_PROTO_0(iemCImpl_vmrun);
3495IEM_CIMPL_PROTO_0(iemCImpl_vmload);
3496IEM_CIMPL_PROTO_0(iemCImpl_vmsave);
3497IEM_CIMPL_PROTO_0(iemCImpl_clgi);
3498IEM_CIMPL_PROTO_0(iemCImpl_stgi);
3499IEM_CIMPL_PROTO_0(iemCImpl_invlpga);
3500IEM_CIMPL_PROTO_0(iemCImpl_skinit);
3501IEM_CIMPL_PROTO_0(iemCImpl_svm_pause);
3502#endif
3503
3504IEM_CIMPL_PROTO_0(iemCImpl_vmcall); /* vmx */
3505IEM_CIMPL_PROTO_0(iemCImpl_vmmcall); /* svm */
3506IEM_CIMPL_PROTO_1(iemCImpl_Hypercall, uint16_t, uDisOpcode); /* both */
3507
3508
3509extern const PFNIEMOP g_apfnOneByteMap[256];
3510
3511/** @} */
3512
3513RT_C_DECLS_END
3514
3515#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3516
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette